diff --git a/.travis.yml b/.travis.yml index e99f535..1b18501 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,5 +11,5 @@ install: script: python setup.py test os: - linux - - \ No newline at end of file +# Need trusty+ for tensorflow +dist: trusty diff --git a/mloop/controllers.py b/mloop/controllers.py index ea53553..a1c1e04 100644 --- a/mloop/controllers.py +++ b/mloop/controllers.py @@ -675,14 +675,14 @@ def _optimization_routine(self): ''' #Run the training runs using the standard optimization routine. self.log.debug('Starting training optimization.') - self.log.info('Run:' + str(self.num_in_costs +1)) + self.log.info('Run:' + str(self.num_in_costs +1) + ' (training)') next_params = self._first_params() self._put_params_and_out_dict(next_params) self.save_archive() self._get_cost_and_in_dict() while (self.num_in_costs < self.num_training_runs) and self.check_end_conditions(): - self.log.info('Run:' + str(self.num_in_costs +1)) + self.log.info('Run:' + str(self.num_in_costs +1) + ' (training)') next_params = self._next_params() self._put_params_and_out_dict(next_params) self.save_archive() @@ -690,26 +690,30 @@ def _optimization_routine(self): if self.check_end_conditions(): #Start last training run - self.log.info('Run:' + str(self.num_in_costs +1)) + self.log.info('Run:' + str(self.num_in_costs +1) + ' (training)') next_params = self._next_params() self._put_params_and_out_dict(next_params) self.log.debug('Starting ML optimization.') - self.new_params_event.set() - self.save_archive() + # TODO: This is a race. There's no guarantee that this will be available by the time the + # event is set. self._get_cost_and_in_dict() + self.save_archive() + self.new_params_event.set() self.log.debug('End training runs.') ml_consec = 0 ml_count = 0 while self.check_end_conditions(): - self.log.info('Run:' + str(self.num_in_costs +1)) + run_num = self.num_in_costs + 1 if ml_consec==self.generation_num or (self.no_delay and self.ml_learner_params_queue.empty()): + self.log.info('Run:' + str(run_num) + ' (trainer)') next_params = self._next_params() self._put_params_and_out_dict(next_params) ml_consec = 0 else: + self.log.info('Run:' + str(run_num) + ' (machine learner)') next_params = self.ml_learner_params_queue.get() super(MachineLearnerController,self)._put_params_and_out_dict(next_params, param_type=self.machine_learner_type) ml_consec += 1 diff --git a/mloop/learners.py b/mloop/learners.py index ad6e5b1..1455c21 100644 --- a/mloop/learners.py +++ b/mloop/learners.py @@ -19,6 +19,7 @@ import sklearn.gaussian_process.kernels as skk import sklearn.preprocessing as skp import multiprocessing as mp +import mloop.neuralnet as mlnn learner_thread_count = 0 default_learner_archive_filename = 'learner_archive' @@ -1467,15 +1468,14 @@ def find_local_minima(self): class NeuralNetLearner(Learner, mp.Process): ''' - Shell of Neural Net Learner. + Learner that uses a neural network for function approximation. Args: params_out_queue (queue): Queue for parameters sent to controller. - costs_in_queue (queue): Queue for costs for gaussian process. This must be tuple + costs_in_queue (queue): Queue for costs. end_event (event): Event to trigger end of learner. Keyword Args: - length_scale (Optional [array]): The initial guess for length scale(s) of the gaussian process. The array can either of size one or the number of parameters or None. If it is size one, it is assumed all the correlation lengths are the same. If it is the number of the parameters then all the parameters have their own independent length scale. If it is None, it is assumed all the length scales should be independent and they are all given an initial value of 1. Default None. trust_region (Optional [float or array]): The trust region defines the maximum distance the learner will travel from the current best set of parameters. If None, the learner will search everywhere. If a float, this number must be between 0 and 1 and defines maximum distance the learner will venture as a percentage of the boundaries. If it is an array, it must have the same size as the number of parameters and the numbers define the maximum absolute distance that can be moved along each direction. default_bad_cost (Optional [float]): If a run is reported as bad and default_bad_cost is provided, the cost for the bad run is set to this default value. If default_bad_cost is None, then the worst cost received is set to all the bad runs. Default None. default_bad_uncertainty (Optional [float]): If a run is reported as bad and default_bad_uncertainty is provided, the uncertainty for the bad run is set to this default value. If default_bad_uncertainty is None, then the uncertainty is set to a tenth of the best to worst cost range. Default None. @@ -1484,11 +1484,10 @@ class NeuralNetLearner(Learner, mp.Process): predict_local_minima_at_end (Optional [bool]): If True finds all minima when the learner is ended. Does not if False. Default False. Attributes: - TODO: Update these. all_params (array): Array containing all parameters sent to learner. all_costs (array): Array containing all costs sent to learner. all_uncers (array): Array containing all uncertainties sent to learner. - scaled_costs (array): Array contaning all the costs scaled to have zero mean and a standard deviation of 1. Needed for training the gaussian process. + scaled_costs (array): Array contaning all the costs scaled to have zero mean and a standard deviation of 1. bad_run_indexs (list): list of indexes to all runs that were marked as bad. best_cost (float): Minimum received cost, updated during execution. best_params (array): Parameters of best run. (reference to element in params array). @@ -1497,23 +1496,22 @@ class NeuralNetLearner(Learner, mp.Process): worst_index (int): index to run with worst cost. cost_range (float): Difference between worst_cost and best_cost generation_num (int): Number of sets of parameters to generate each generation. Set to 5. - length_scale_history (list): List of length scales found after each fit. noise_level_history (list): List of noise levels found after each fit. - fit_count (int): Counter for the number of times the gaussian process has been fit. cost_count (int): Counter for the number of costs, parameters and uncertainties added to learner. params_count (int): Counter for the number of parameters asked to be evaluated by the learner. - gaussian_process (GaussianProcessRegressor): Gaussian process that is fitted to data and used to make predictions + neural_net (NeuralNet): Neural net that is fitted to data and used to make predictions. cost_scaler (StandardScaler): Scaler used to normalize the provided costs. + cost_scaler_init_index (int): The number of params to use to initialise cost_scaler. has_trust_region (bool): Whether the learner has a trust region. ''' def __init__(self, - update_hyperparameters = True, trust_region=None, default_bad_cost = None, default_bad_uncertainty = None, nn_training_filename =None, nn_training_file_type ='txt', + minimum_uncertainty = 1e-8, predict_global_minima_at_end = True, predict_local_minima_at_end = False, **kwargs): @@ -1523,7 +1521,7 @@ def __init__(self, nn_training_filename = str(nn_training_filename) nn_training_file_type = str(nn_training_file_type) if not mlu.check_file_type_supported(nn_training_file_type): - self.log.error('GP training file type not supported' + repr(nn_training_file_type)) + self.log.error('NN training file type not supported' + repr(nn_training_file_type)) self.training_dict = mlu.get_dict_from_file(nn_training_filename, nn_training_file_type) @@ -1534,7 +1532,6 @@ def __init__(self, #Counters self.costs_count = int(self.training_dict['costs_count']) - self.fit_count = int(self.training_dict['fit_count']) self.params_count = int(self.training_dict['params_count']) #Data from previous experiment @@ -1555,7 +1552,10 @@ def __init__(self, #Configuration of the fake neural net learner self.length_scale = mlu.safe_squeeze(self.training_dict['length_scale']) self.noise_level = float(self.training_dict['noise_level']) - + + self.cost_scaler_init_index = self.training_dict['cost_scaler_init_index'] + if not self.cost_scaler_init_index is None: + self._init_cost_scaler() try: self.predicted_best_parameters = mlu.safe_squeeze(self.training_dict['predicted_best_parameters']) @@ -1599,15 +1599,17 @@ def __init__(self, self.worst_cost = float('-inf') self.worst_index = 0 self.cost_range = float('inf') - self.length_scale_history = [] self.noise_level_history = [] self.costs_count = 0 - self.fit_count = 0 self.params_count = 0 self.has_local_minima = False self.has_global_minima = False + + # The scaler will be initialised when we're ready to fit it + self.cost_scaler = None + self.cost_scaler_init_index = None #Multiprocessor controls self.new_params_event = mp.Event() @@ -1617,16 +1619,17 @@ def __init__(self, self.scaled_costs = None #Constants, limits and tolerances - self.generation_num = 1 + self.num_nets = 3 + self.generation_num = 3 self.search_precision = 1.0e-6 self.parameter_searches = max(10,self.num_params) self.hyperparameter_searches = max(10,self.num_params) self.bad_uncer_frac = 0.1 #Fraction of cost range to set a bad run uncertainty #Optional user set variables - self.update_hyperparameters = bool(update_hyperparameters) self.predict_global_minima_at_end = bool(predict_global_minima_at_end) self.predict_local_minima_at_end = bool(predict_local_minima_at_end) + self.minimum_uncertainty = float(minimum_uncertainty) if default_bad_cost is not None: self.default_bad_cost = float(default_bad_cost) else: @@ -1642,6 +1645,9 @@ def __init__(self, else: self.log.error('Both the default cost and uncertainty must be set for a bad run or they must both be set to None.') raise ValueError + if self.minimum_uncertainty <= 0: + self.log.error('Minimum uncertainty must be larger than zero for the learner.') + raise ValueError self._set_trust_region(trust_region) @@ -1655,13 +1661,7 @@ def __init__(self, self.cost_has_noise = True self.noise_level = 1 - # Set up the scaler to do nothing. - # TODO: Figure out how to use scaling for the NN (it's a bit difficult because we don't - # completely re-train each time, and don't want the scaling changing without doing a complete - # re-train). - self.cost_scaler = skp.StandardScaler(with_mean=False, with_std=False) - - self.archive_dict.update({'archive_type':'nerual_net_learner', + self.archive_dict.update({'archive_type':'neural_net_learner', 'bad_run_indexs':self.bad_run_indexs, 'generation_num':self.generation_num, 'search_precision':self.search_precision, @@ -1673,48 +1673,72 @@ def __init__(self, 'predict_global_minima_at_end':self.predict_global_minima_at_end, 'predict_local_minima_at_end':self.predict_local_minima_at_end}) - #Remove logger so gaussian process can be safely picked for multiprocessing on Windows + #Remove logger so neural net can be safely picked for multiprocessing on Windows self.log = None + def _construct_net(self): + self.neural_net = [mlnn.NeuralNet(self.num_params) for _ in range(self.num_nets)] + + def _init_cost_scaler(self): + ''' + Initialises the cost scaler. cost_scaler_init_index must be set. + ''' + self.cost_scaler = skp.StandardScaler(with_mean=False, with_std=False) + self.cost_scaler.fit(self.all_costs[:self.cost_scaler_init_index,np.newaxis]) + def create_neural_net(self): ''' Creates the neural net. Must be called from the same process as fit_neural_net, predict_cost and predict_costs_from_param_array. ''' - import mloop.nnlearner as mlnn - self.neural_net_impl = mlnn.NeuralNetImpl(self.num_params) + self._construct_net() + for n in self.neural_net: + n.init() - def fit_neural_net(self): + def import_neural_net(self): ''' - Determine the appropriate number of layers for the NN given the data. + Imports neural net parameters from the training dictionary provided at construction. Must be called from the same process as fit_neural_net, predict_cost and predict_costs_from_param_array. You must call exactly one of this and create_neural_net before calling other methods. + ''' + if not self.training_dict: + raise ValueError + self._construct_net() + for i, n in enumerate(self.neural_net): + n.load(self.training_dict['net_' + str(i)]) - Fit the Neural Net with the appropriate topology to the data + def _fit_neural_net(self,index): + ''' + Fits a neural net to the data. + cost_scaler must have been fitted before calling this method. ''' - self.scaled_costs = self.cost_scaler.fit_transform(self.all_costs[:,np.newaxis])[:,0] + self.scaled_costs = self.cost_scaler.transform(self.all_costs[:,np.newaxis])[:,0] - self.neural_net_impl.fit_neural_net(self.all_params, self.scaled_costs) + self.neural_net[index].fit_neural_net(self.all_params, self.scaled_costs) - def predict_cost(self,params): + def predict_cost(self,params,net_index=None): ''' - Produces a prediction of cost from the gaussian process at params. + Produces a prediction of cost from the neural net at params. Returns: float : Predicted cost at paramters ''' - return self.neural_net_impl.predict_cost(params) + if net_index is None: + net_index = nr.randint(self.num_nets) + return self.neural_net[net_index].predict_cost(params) - def predict_cost_gradient(self,params): + def predict_cost_gradient(self,params,net_index=None): ''' Produces a prediction of the gradient of the cost function at params. Returns: float : Predicted gradient at paramters ''' + if net_index is None: + net_index = nr.randint(self.num_nets) # scipy.optimize.minimize doesn't seem to like a 32-bit Jacobian, so we convert to 64 - return np.float64(self.neural_net_impl.predict_cost_gradient(params)) + return self.neural_net[net_index].predict_cost_gradient(params).astype(np.float64) - def predict_costs_from_param_array(self,params): + def predict_costs_from_param_array(self,params,net_index=None): ''' Produces a prediction of costs from an array of params. @@ -1722,7 +1746,7 @@ def predict_costs_from_param_array(self,params): float : Predicted cost at paramters ''' # TODO: Can do this more efficiently. - return [self.predict_cost(param) for param in params] + return [self.predict_cost(param,net_index) for param in params] def wait_for_new_params_event(self): @@ -1744,18 +1768,28 @@ def get_params_and_costs(self): ''' Get the parameters and costs from the queue and place in their appropriate all_[type] arrays. Also updates bad costs, best parameters, and search boundaries given trust region. ''' - if self.costs_in_queue.empty(): - self.log.error('Neural network asked for new parameters but no new costs were provided.') - raise ValueError - new_params = [] new_costs = [] new_uncers = [] new_bads = [] update_bads_flag = False - while not self.costs_in_queue.empty(): - (param, cost, uncer, bad) = self.costs_in_queue.get_nowait() + first_dequeue = True + while True: + if first_dequeue: + try: + # Block for 1s, because there might be a race with the event being set. + (param, cost, uncer, bad) = self.costs_in_queue.get(block=True, timeout=1) + first_dequeue = False + except mlu.empty_exception: + self.log.error('Neural network asked for new parameters but no new costs were provided after 1s.') + raise ValueError + else: + try: + (param, cost, uncer, bad) = self.costs_in_queue.get_nowait() + except mlu.empty_exception: + break + self.costs_count +=1 if bad: @@ -1861,34 +1895,42 @@ def update_archive(self): 'worst_cost':self.worst_cost, 'worst_index':self.worst_index, 'cost_range':self.cost_range, - 'fit_count':self.fit_count, 'costs_count':self.costs_count, 'params_count':self.params_count, - 'update_hyperparameters':self.update_hyperparameters, 'length_scale':self.length_scale, - 'noise_level':self.noise_level}) + 'noise_level':self.noise_level, + 'cost_scaler_init_index':self.cost_scaler_init_index}) + if self.neural_net: + for i,n in enumerate(self.neural_net): + self.archive_dict.update({'net_'+str(i):n.save()}) - def find_next_parameters(self): + def find_next_parameters(self, net_index=None): ''' Returns next parameters to find. Increments counters appropriately. Return: next_params (array): Returns next parameters from cost search. ''' - # TODO: We could implement some other type of biasing. + if net_index is None: + net_index = nr.randint(self.num_nets) + self.params_count += 1 self.update_search_params() next_params = None next_cost = float('inf') + self.neural_net[net_index].start_opt() for start_params in self.search_params: - result = so.minimize(fun = self.predict_cost, + result = so.minimize(fun = lambda x: self.predict_cost(x, net_index), x0 = start_params, - jac = self.predict_cost_gradient, + jac = lambda x: self.predict_cost_gradient(x, net_index), bounds = self.search_region, tol = self.search_precision) if result.fun < next_cost: next_params = result.x next_cost = result.fun + self.neural_net[net_index].stop_opt() + self.log.debug("Suggesting params " + str(next_params) + " with predicted cost: " + + str(next_cost)) return next_params def run(self): @@ -1902,20 +1944,42 @@ def run(self): # The network needs to be created in the same process in which it runs self.create_neural_net() + # We cycle through our different nets to generate each new set of params. This keeps track + # of the current net. + net_index = 0 + try: while not self.end_event.is_set(): self.log.debug('Learner waiting for new params event') - self.save_archive() + # TODO: Not doing this because it's slow. Is it necessary? + #self.save_archive() self.wait_for_new_params_event() self.log.debug('NN learner reading costs') self.get_params_and_costs() - self.fit_neural_net() + if self.cost_scaler_init_index is None: + self.cost_scaler_init_index = len(self.all_costs) + self._init_cost_scaler() + # Now we need to generate generation_num new param sets, by iterating over our + # nets. We want to fire off new params as quickly as possible, so we don't train a + # net until we actually need to use it. But we need to make sure that each net gets + # trained exactly once, regardless of how many times it's used to generate new + # params. + num_nets_trained = 0 for _ in range(self.generation_num): + if num_nets_trained < self.num_nets: + self._fit_neural_net(net_index) + num_nets_trained += 1 + self.log.debug('Neural network learner generating parameter:'+ str(self.params_count+1)) - next_params = self.find_next_parameters() + next_params = self.find_next_parameters(net_index) + net_index = (net_index + 1) % self.num_nets self.params_out_queue.put(next_params) if self.end_event.is_set(): raise LearnerInterrupt() + # Train any nets that haven't been trained yet. + for i in range(self.num_nets - num_nets_trained): + self._fit_neural_net((net_index + i) % self.num_nets) + except LearnerInterrupt: pass # TODO: Fix this. We can't just do what's here because the costs queue might be empty, but @@ -1934,6 +1998,8 @@ def run(self): 'local_minima_costs':self.local_minima_costs}) self.params_out_queue.put(end_dict) self._shut_down() + for n in self.neural_net: + n.destroy() self.log.debug('Ended neural network learner') def find_global_minima(self): @@ -1967,7 +2033,7 @@ def find_global_minima(self): self.predicted_best_parameters = curr_best_params self.predicted_best_scaled_cost = curr_best_cost - self.predicted_best_cost = float(self.cost_scaler.inverse_transform(self.predicted_best_scaled_cost)) + self.predicted_best_cost = float(self.cost_scaler.inverse_transform([self.predicted_best_scaled_cost])) self.archive_dict.update({'predicted_best_parameters':self.predicted_best_parameters, 'predicted_best_scaled_cost':self.predicted_best_scaled_cost, 'predicted_best_cost':self.predicted_best_cost}) @@ -2014,9 +2080,10 @@ def find_local_minima(self): self.has_local_minima = True self.log.info('Search completed') + # Methods for debugging/analysis. - - - - - + def get_losses(self): + all_losses = [] + for n in self.neural_net: + all_losses += n.get_losses() + return all_losses diff --git a/mloop/neuralnet.py b/mloop/neuralnet.py new file mode 100644 index 0000000..0e17925 --- /dev/null +++ b/mloop/neuralnet.py @@ -0,0 +1,644 @@ +import datetime +import logging +import math +import time +import base64 + +import mloop.utilities as mlu +import numpy as np +import numpy.random as nr +import sklearn.preprocessing as skp +import tensorflow as tf + +class SingleNeuralNet(): + ''' + A single neural network with fixed hyperparameters/topology. + + This must run in the same process in which it's created. + + This class should be considered private to this module. + + Args: + num_params: The number of params. + layer_dims: The number of nodes in each layer. + layer_activations: The activation function for each layer. + train_threshold_ratio: (Relative) loss improvement per train under which training should + terminate. E.g. 0.1 means we will train (train_epochs at a time) until the improvement + in loss is less than 0.1 of the loss when that train started (so lower values mean we + will train for longer). Alternatively, you can think of this as the smallest gradient + we'll allow before deciding that the loss isn't improving any more. + batch_size: The training batch size. + keep_prob: The dropoout keep probability. + regularisation_coefficient: The regularisation coefficient. + losses_list: A list to which this object will append training losses. + ''' + + def __init__(self, + num_params, + layer_dims, + layer_activations, + train_threshold_ratio, + batch_size, + keep_prob, + regularisation_coefficient, + losses_list): + self.log = logging.getLogger(__name__) + start = time.time() + + self.save_archive_filename = ( + mlu.archive_foldername + + "neural_net_archive_" + + mlu.datetime_to_string(datetime.datetime.now()) + + "_" + # We include 6 random bytes for deduplication in case multiple nets + # are created at the same time. + + base64.urlsafe_b64encode(nr.bytes(6)).decode() + + ".ckpt") + + self.log.info("Constructing net") + self.graph = tf.Graph() + self.tf_session = tf.Session(graph=self.graph) + + if not len(layer_dims) == len(layer_activations): + self.log.error('len(layer_dims) != len(layer_activations)') + raise ValueError + + # Hyperparameters for the net. These are all constant. + self.num_params = num_params + self.train_threshold_ratio = train_threshold_ratio + self.batch_size = batch_size + self.keep_prob = keep_prob + self.regularisation_coefficient = regularisation_coefficient + + self.losses_list = losses_list + + with self.graph.as_default(): + ## Inputs + self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params]) + self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1]) + self.keep_prob_placeholder = tf.placeholder_with_default(1., shape=[]) + self.regularisation_coefficient_placeholder = tf.placeholder_with_default(0., shape=[]) + + ## Initialise the network + + weights = [] + biases = [] + + # Input + internal nodes + prev_layer_dim = self.num_params + bias_stddev=0.5 + for (i, dim) in enumerate(layer_dims): + weights.append(tf.Variable( + tf.random_normal([prev_layer_dim, dim], stddev=1.4/np.sqrt(prev_layer_dim)), + name="weight_"+str(i))) + biases.append(tf.Variable( + tf.random_normal([dim], stddev=bias_stddev), + name="bias_"+str(i))) + prev_layer_dim = dim + + # Output node + weights.append(tf.Variable( + tf.random_normal([prev_layer_dim, 1], stddev=1.4/np.sqrt(prev_layer_dim)), + name="weight_out")) + biases.append(tf.Variable( + tf.random_normal([1], stddev=bias_stddev), + name="bias_out")) + + # Get the output var given an input var + def get_output_var(input_var): + prev_h = input_var + for w, b, act in zip(weights[:-1], biases[:-1], layer_activations): + prev_h = tf.nn.dropout( + act(tf.matmul(prev_h, w) + b), + keep_prob=self.keep_prob_placeholder) + return tf.matmul(prev_h, weights[-1]) + biases[-1] + + ## Define tensors for evaluating the output var and gradient on the full input + self.output_var = get_output_var(self.input_placeholder) + self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder) + + ## Declare common loss functions + + # Get the raw loss given the expected and actual output vars + def get_loss_raw(expected, actual): + return tf.reduce_mean(tf.reduce_sum( + tf.square(expected - actual), + reduction_indices=[1])) + + # Regularisation component of the loss. + loss_reg = (self.regularisation_coefficient_placeholder + * tf.reduce_mean([tf.nn.l2_loss(W) for W in weights])) + + ## Define tensors for evaluating the loss on the full input + self.loss_raw = get_loss_raw(self.output_placeholder, self.output_var) + self.loss_total = self.loss_raw + loss_reg + + ## Training + self.train_step = tf.train.AdamOptimizer().minimize(self.loss_total) + + # Initialiser for ... initialising + self.initialiser = tf.global_variables_initializer() + + # Saver for saving and restoring params + self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2) + self.log.debug("Finished constructing net in: " + str(time.time() - start)) + + def destroy(self): + self.tf_session.close() + + def init(self): + ''' + Initializes the net. + ''' + self.tf_session.run(self.initialiser) + + def load(self, archive): + ''' + Imports the net from an archive dictionary. You must call exactly one of this and init() before calling any other methods. + ''' + self.log.info("Loading neural network") + self.saver.restore(self.tf_session, "./" + str(archive['saver_path'])) + + def save(self): + ''' + Exports the net to an archive dictionary. + ''' + path = self.saver.save(self.tf_session, self.save_archive_filename) + self.log.info("Saving neural network to: " + path) + return {'saver_path': path} + + def _loss(self, params, costs): + ''' + Returns the loss and unregularised loss for the given params and costs. + ''' + return self.tf_session.run( + [self.loss_total, self.loss_raw], + feed_dict={self.input_placeholder: params, + self.output_placeholder: [[c] for c in costs], + self.regularisation_coefficient_placeholder: self.regularisation_coefficient, + }) + + def fit(self, params, costs, epochs): + ''' + Fit the neural net to the provided data + + Args: + params (array): array of parameter arrays + costs (array): array of costs (associated with the corresponding parameters) + ''' + self.log.info('Fitting neural network') + if len(params) == 0: + self.log.error('No data provided.') + raise ValueError + if not len(params) == len(costs): + self.log.error("Params and costs must have the same length") + raise ValueError + + lparams = np.array(params) + lcosts = np.expand_dims(np.array(costs), axis=1) + + # The general training procedure is as follows: + # - set a threshold based on the current loss + # - train for train_epochs epochs + # - if the new loss is greater than the threshold then we haven't improved much, so stop + # - else start from the top + start = time.time() + while True: + threshold = (1 - self.train_threshold_ratio) * self._loss(params, costs)[0] + self.log.debug("Training with threshold " + str(threshold)) + if threshold == 0: + break + tot = 0 + run_start = time.time() + for i in range(epochs): + # Split the data into random batches, and train on each batch + indices = np.random.permutation(len(params)) + for j in range(int(math.ceil(len(params) / self.batch_size))): + batch_indices = indices[j * self.batch_size : (j + 1) * self.batch_size] + batch_input = lparams[batch_indices] + batch_output = lcosts[batch_indices] + self.tf_session.run(self.train_step, + feed_dict={self.input_placeholder: batch_input, + self.output_placeholder: batch_output, + self.regularisation_coefficient_placeholder: self.regularisation_coefficient, + self.keep_prob_placeholder: self.keep_prob, + }) + if i % 10 == 0: + (l, ul) = self._loss(params, costs) + self.losses_list.append(l) + self.log.info('Fit neural network with total training cost ' + str(l) + + ', with unregularized cost ' + str(ul)) + self.log.debug("Run trained for: " + str(time.time() - run_start)) + + (l, ul) = self._loss(params, costs) + al = tot / float(epochs) + self.log.debug('Loss ' + str(l) + ', average loss ' + str(al)) + if l > threshold: + break + self.log.debug("Total trained for: " + str(time.time() - start)) + + def cross_validation_loss(self, params, costs): + ''' + Returns the loss of the network on a cross validation set. + + Args: + params (array): array of parameter arrays + costs (array): array of costs (associated with the corresponding parameters) + ''' + return self.tf_session.run(self.loss_total, + feed_dict={self.input_placeholder: params, + self.output_placeholder: [[c] for c in costs], + }) + + def predict_cost(self,params): + ''' + Produces a prediction of cost from the neural net at params. + + Returns: + float : Predicted cost at parameters + ''' + return self.tf_session.run(self.output_var, feed_dict={self.input_placeholder: [params]})[0][0] + #runs = 100 + ## Do some runs with dropout, and return the smallest. This is kind of LCB. + #results = [y[0] for y in self.tf_session.run(self.output_var, feed_dict={ + # self.input_placeholder: [params] * runs, + # self.keep_prob_placeholder: 0.99})] + #results.sort() + #return results[int(runs * 0.2)] + + def predict_cost_gradient(self,params): + ''' + Produces a prediction of the gradient of the cost function at params. + + Returns: + float : Predicted gradient at parameters + ''' + return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0] + + +class SampledNeuralNet(): + ''' + A "neural network" that tracks a collection of SingleNeuralNet objects, and predicts the landscape + by sampling from that collection. + + This must run in the same process in which it's created. + + This class should be considered private to this module. + + Args: + net_creator: Callable that creates and returns a new SingleNeuralNet. + count: The number of individual networks to track. + ''' + + def __init__(self, + net_creator, + count): + self.log = logging.getLogger(__name__) + self.net_creator = net_creator + self.nets = [self.net_creator() for _ in range(count)] + self.fit_count = 0 + self.opt_net = None + + def _random_net(self): + return self.nets[np.random.randint(0, len(self.nets))] + + def destroy(self): + for n in self.nets: + n.destroy() + + def init(self): + for n in self.nets: + n.init() + + def load(self, archive): + for i, n in enumerate(self.nets): + #n.load(archive[str(i)]) + n.load(archive) + + def save(self): + return self.nets[0].save() + #ret = {} + #for i, n in enumerate(self.nets): + # ret[str(i)] = n.save() + #return ret + + def fit(self, params, costs, epochs): + self.fit_count += 1 + # Every per'th fit we clear out a net and re-train it. + #per = 2 + #if self.fit_count % per == 0: + # index = int(self.fit_count / per) % len(self.nets) + # self.log.debug("Re-creating net " + str(index)) + # self.nets[index].destroy() + # self.nets[index] = self.net_creator() + # self.nets[index].init() + + for n in self.nets: + n.fit(params, costs, epochs) + + def cross_validation_loss(self, params, costs): + return np.mean([n.cross_validation_loss(params, costs) for n in self.nets]) + + def predict_cost(self,params): + if self.opt_net: + return self.opt_net.predict_cost(params) + else: + return self._random_net().predict_cost(params) + #return np.mean([n.predict_cost(params) for n in self.nets]) + + def predict_cost_gradient(self,params): + if self.opt_net: + return self.opt_net.predict_cost_gradient(params) + else: + return self._random_net().predict_cost_gradient(params) + #return np.mean([n.predict_cost_gradient(params) for n in self.nets]) + + def start_opt(self): + self.opt_net = self._random_net() + + def stop_opt(self): + self.opt_net = None + +class NeuralNet(): + ''' + Neural network implementation. This may actually create multiple neural networks with different + topologies or hyperparameters, and switch between them based on the data. + + This must run in the same process in which it's created. + + This handles scaling of parameters and costs internally, so there is no need to ensure that these + values are scaled or normalised in any way. + + All parameters should be considered private to this class. That is, you should only interact with + this class via the methods documented to be public. + + Args: + num_params (int): The number of params. + fit_hyperparameters (bool): Whether to try to fit the hyperparameters to the data. + ''' + + def __init__(self, + num_params = None, + fit_hyperparameters = False): + + self.log = logging.getLogger(__name__) + self.log.info('Initialising neural network impl') + if num_params is None: + self.log.error("num_params must be provided") + raise ValueError + + # Constants. + self.num_params = num_params + self.fit_hyperparameters = fit_hyperparameters + + self.initial_epochs = 100 + self.subsequent_epochs = 20 + + # Variables for tracking the current state of hyperparameter fitting. + self.last_hyperfit = 0 + self.last_net_reg = 1e-8 + + # The samples used to fit the scalers. When set, this will be a tuple of + # (params samples, cost samples). + self.scaler_samples = None + + # The training losses incurred by the network. This is a concatenation of the losses + # associated with each instance of SingleNeuralNet. + self.losses_list = [] + + self.net = None + + # Private helper methods. + + def _make_net(self, reg): + ''' + Helper method to create a new net with a specified regularisation coefficient. The net is not + initialised, so you must call init() or load() on it before any other method. + + Args: + reg (float): Regularisation coefficient. + ''' + def gelu_fast(_x): + return 0.5 * _x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (_x + 0.044715 * tf.pow(_x, 3)))) + creator = lambda: SingleNeuralNet( + self.num_params, + [64]*5, [gelu_fast]*5, + 0.2, # train_threshold_ratio + 16, # batch_size + 1., # keep_prob + reg, + self.losses_list) + return SampledNeuralNet(creator, 1) + + def _fit_scaler(self): + ''' + Fits the cost and param scalers based on the scaler_samples member variable. + ''' + if self.scaler_samples is None: + self.log.error("_fit_scaler() called before samples set") + raise ValueError + self._cost_scaler = skp.StandardScaler(with_mean=True, with_std=True) + self._param_scaler = skp.StandardScaler(with_mean=True, with_std=True) + + self._param_scaler.fit(self.scaler_samples[0]) + # Cost is scalar but numpy doesn't like scalars, so reshape to be a 0D vector instead. + self._cost_scaler.fit(np.array(self.scaler_samples[1]).reshape(-1,1)) + + self._mean_offset = 0 + + # Now that the scaler is fitted, calculate the parameters we'll need to unscale gradients. + # We need to know which unscaled gradient would correspond to a scaled gradient of [1,...1], + # which we can calculate as the unscaled gradient associated with a scaled rise of 1 and a + # scaled run of [1,...1]: + rise_unscaled = ( + self._unscale_cost(np.float64(1)) + - self._unscale_cost(np.float64(0))) + run_unscaled = ( + self._unscale_params([np.float64(1)]*self.num_params) + - self._unscale_params([np.float64(0)]*self.num_params)) + self._gradient_unscale = rise_unscaled / run_unscaled + + def _scale_params_and_cost_list(self, params_list_unscaled, cost_list_unscaled): + params_list_scaled = self._param_scaler.transform(params_list_unscaled) + # As above, numpy doesn't like scalars, so we need to do some reshaping. + cost_vector_list_unscaled = np.array(cost_list_unscaled).reshape(-1,1) + cost_vector_list_scaled = (self._cost_scaler.transform(cost_vector_list_unscaled) + + self._mean_offset) + cost_list_scaled = cost_vector_list_scaled[:,0] + return params_list_scaled, cost_list_scaled + + def _scale_params(self, params_unscaled): + return self._param_scaler.transform([params_unscaled])[0] + + def _unscale_params(self, params_scaled): + return self._param_scaler.inverse_transform([params_scaled])[0] + + def _unscale_cost(self, cost_scaled): + return self._cost_scaler.inverse_transform([[cost_scaled - self._mean_offset]])[0][0] + + def _unscale_gradient(self, gradient_scaled): + return np.multiply(gradient_scaled, self._gradient_unscale) + + # Public methods. + + def init(self): + ''' + Initializes the net. You must call exactly one of this and load() before calling any other + methods. + ''' + if not self.net is None: + self.log.error("Called init() when already initialised/loaded") + raise ValueError + + self.net = self._make_net(self.last_net_reg) + self.net.init() + + def load(self, archive): + ''' + Imports the net from an archive dictionary. You must call exactly one of this and init() + before calling any other methods. + + You must only load a net from an archive if that archive corresponds to a net with the same + constructor parameters. + ''' + if not self.net is None: + self.log.error("Called load() when net already initialised/loaded") + raise ValueError + + self.last_hyperfit = int(archive['last_hyperfit']) + self.last_net_reg = float(archive['last_net_reg']) + + self.losses_list = list(archive['losses_list']) + + self.scaler_samples = archive['scaler_samples'] + if not self.scaler_samples is None: + self._fit_scaler() + + self.net = self._make_net(self.last_net_reg) + self.net.load(dict(archive['net'])) + + def save(self): + ''' + Exports the net to an archive dictionary. + ''' + return {'last_hyperfit': self.last_hyperfit, + 'last_net_reg': self.last_net_reg, + 'losses_list': self.losses_list, + 'scaler_samples': self.scaler_samples, + 'net': self.net.save(), + } + + def destroy(self): + ''' + Destroys the net. + ''' + if not self.net is None: + self.net.destroy() + + def fit_neural_net(self, all_params, all_costs): + ''' + Fits the neural net to the data. + + Args: + all_params (array): array of all parameter arrays + all_costs (array): array of costs (associated with the corresponding parameters) + ''' + if len(all_params) == 0: + self.log.error('No data provided.') + raise ValueError + if not len(all_params) == len(all_costs): + self.log.error("Params and costs must have the same length") + raise ValueError + + # If we haven't initialised the scaler yet, do it now. + if self.scaler_samples is None: + first_fit = True + self.scaler_samples = (all_params.copy(), all_costs.copy()) + self._fit_scaler() + else: + first_fit = False + + all_params, all_costs = self._scale_params_and_cost_list(all_params, all_costs) + + if self.fit_hyperparameters: + # Every 20 fits (starting at 5, just because), re-fit the hyperparameters + if int(len(all_params + 5) / 20) > self.last_hyperfit: + self.last_hyperfit = int(len(all_params + 5) / 20) + + # Fit regularisation + + # Split the data into training and cross validation + cv_size = int(len(all_params) / 10) + train_params = all_params[:-cv_size] + train_costs = all_costs[:-cv_size] + cv_params = all_params[cv_size:] + cv_costs = all_costs[cv_size:] + + orig_cv_loss = self.net.cross_validation_loss(cv_params, cv_costs) + best_cv_loss = orig_cv_loss + + self.log.debug("Fitting regularisation, current cv loss=" + str(orig_cv_loss)) + + # Try a bunch of different regularisation parameters, switching to a new one if it + # does significantly better on the cross validation set than the old one. + for r in [0.001, 0.01, 0.1, 1, 10]: + net = self._make_net(r) + net.init() + net.fit(train_params, train_costs, self.initial_epochs) + this_cv_loss = net.cross_validation_loss(cv_params, cv_costs) + if this_cv_loss < best_cv_loss and this_cv_loss < 0.1 * orig_cv_loss: + best_cv_loss = this_cv_loss + self.log.debug("Switching to reg=" + str(r) + ", cv loss=" + str(best_cv_loss)) + self.last_net_reg = r + self.net.destroy() + self.net = net + else: + net.destroy() + + self.net.fit( + all_params, + all_costs, + self.initial_epochs if first_fit else self.subsequent_epochs) + + def predict_cost(self,params): + ''' + Produces a prediction of cost from the neural net at params. + + Must not be called before fit_neural_net(). + + Returns: + float : Predicted cost at parameters + ''' + return self._unscale_cost(self.net.predict_cost(self._scale_params(params))) + + def predict_cost_gradient(self,params): + ''' + Produces a prediction of the gradient of the cost function at params. + + Must not be called before fit_neural_net(). + + Returns: + float : Predicted gradient at parameters + ''' + return self._unscale_gradient(self.net.predict_cost_gradient(self._scale_params(params))) + + def start_opt(self): + ''' + Starts an optimisation run. Until stop_opt() is called, predict_cost() and + predict_cost_gradient() will return consistent values. + ''' + self.net.start_opt() + + def stop_opt(self): + ''' + Stops an optimisation run. + ''' + self.net.stop_opt() + + # Public mmethods to be used only for debugging/analysis. + + def get_losses(self): + ''' + Returns a list of training losses experienced by the network. + ''' + return self.losses_list diff --git a/mloop/nnlearner.py b/mloop/nnlearner.py deleted file mode 100644 index 4674f85..0000000 --- a/mloop/nnlearner.py +++ /dev/null @@ -1,130 +0,0 @@ -import logging -import math -import tensorflow as tf -import numpy as np - -class NeuralNetImpl(): - ''' - Neural network implementation. - - This must run in the same process in which it's created. - - Args: - num_params (int): The number of params. - ''' - - def __init__(self, - num_params = None): - - self.log = logging.getLogger(__name__) - self.log.debug('Initialising neural network impl') - if num_params is None: - self.log.error("num_params must be provided") - raise ValueError - self.num_params = num_params - - self.tf_session = tf.InteractiveSession() - - # Initial hyperparameters - self.num_layers = 1 - self.layer_dim = 128 - self.train_epochs = 300 - self.batch_size = 64 - - # Inputs - self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params]) - self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1]) - self.keep_prob = tf.placeholder_with_default(1., shape=[]) - self.regularisation_coefficient = tf.placeholder_with_default(0., shape=[]) - - self._create_neural_net() - - def _create_neural_net(self): - ''' - Creates the neural net with topology specified by the current hyperparameters. - - ''' - self.log.debug('Creating neural network') - # Forget about any old weights/biases - self.weights = [] - self.biases = [] - - # Input + internal nodes - # TODO: Use length scale for setting initial weights? - prev_layer_dim = self.num_params - prev_h = self.input_placeholder - for dim in [self.layer_dim] * self.num_layers: - self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, dim], stddev=0.1))) - self.biases.append(tf.Variable(tf.random_normal([dim]))) - prev_layer_dim = dim - prev_h = tf.nn.dropout( - tf.nn.sigmoid(tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]), - keep_prob=self.keep_prob) - - # Output node - self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, 1]))) - self.biases.append(tf.Variable(tf.random_normal([1]))) - self.output_var = tf.matmul(prev_h, self.weights[-1]) + self.biases[-1] - - # Loss function and training - loss_func = ( - tf.reduce_mean(tf.reduce_sum(tf.square(self.output_var - self.output_placeholder), - reduction_indices=[1])) - + self.regularisation_coefficient * sum([tf.nn.l2_loss(W) for W in self.weights])) - self.train_step = tf.train.AdamOptimizer(1.0).minimize(loss_func) - - # Gradient - self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder) - - self.tf_session.run(tf.initialize_all_variables()) - - def fit_neural_net(self, all_params, all_costs): - ''' - Determine the appropriate number of layers for the NN given the data. - - Fit the Neural Net with the appropriate topology to the data - - Args: - all_params (array): array of all parameter arrays - all_costs (array): array of costs (associated with the corresponding parameters) - ''' - self.log.debug('Fitting neural network') - if len(all_params) == 0: - self.log.error('No data provided.') - raise ValueError - if not len(all_params) == len(all_costs): - self.log.error("Params and costs must have the same length") - raise ValueError - - # TODO: Fit hyperparameters. - - for i in range(self.train_epochs): - # Split the data into random batches, and train on each batch - all_indices = np.random.permutation(len(all_params)) - for j in range(math.ceil(len(all_params) / self.batch_size)): - batch_indices = all_indices[j * self.batch_size : (j + 1) * self.batch_size] - batch_input = [all_params[index] for index in batch_indices] - batch_output = [[all_costs[index]] for index in batch_indices] - self.tf_session.run(self.train_step, - feed_dict={self.input_placeholder: batch_input, - self.output_placeholder: batch_output, - self.regularisation_coefficient: 0.01, - }) - - def predict_cost(self,params): - ''' - Produces a prediction of cost from the neural net at params. - - Returns: - float : Predicted cost at parameters - ''' - return self.tf_session.run(self.output_var, feed_dict={self.input_placeholder: [params]})[0][0] - - def predict_cost_gradient(self,params): - ''' - Produces a prediction of the gradient of the cost function at params. - - Returns: - float : Predicted gradient at parameters - ''' - return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0] diff --git a/mloop/visualizations.py b/mloop/visualizations.py index 8219fbc..f746e76 100644 --- a/mloop/visualizations.py +++ b/mloop/visualizations.py @@ -11,6 +11,9 @@ import logging import matplotlib.pyplot as plt import matplotlib as mpl +import plotly.plotly as py +import plotly.tools as tls +import plotly.exceptions as pye from mpl_toolkits.mplot3d import Axes3D @@ -38,7 +41,7 @@ def show_all_default_visualizations(controller, show_plots=True): log = logging.getLogger(__name__) configure_plots() log.debug('Creating controller visualizations.') - create_contoller_visualizations(controller.total_archive_filename, + create_controller_visualizations(controller.total_archive_filename, file_type=controller.controller_archive_file_type) if isinstance(controller, mlc.DifferentialEvolutionController): @@ -64,6 +67,28 @@ def show_all_default_visualizations(controller, show_plots=True): if show_plots: plt.show() +def show_all_default_visualizations_from_archive(controller_filename, learner_filename, controller_type, show_plots=True, upload_cross_sections=False): + log = logging.getLogger(__name__) + configure_plots() + log.debug('Creating controller visualizations.') + controller_file_type = controller_filename.split(".")[-1] + learner_file_type = learner_filename.split(".")[-1] + create_controller_visualizations(controller_filename, file_type=controller_file_type) + + if controller_type == 'neural_net': + log.debug('Creating neural net visualizations.') + create_neural_net_learner_visualizations( + learner_filename, + file_type=learner_file_type, + upload_cross_sections=upload_cross_sections) + else: + log.error('show_all_default_visualizations not implemented for type: ' + controller_type) + raise ValueError + + log.info('Showing visualizations, close all to end MLOOP.') + if show_plots: + plt.show() + def _color_from_controller_name(controller_name): ''' Gives a color (as a number betweeen zero an one) corresponding to each controller name string. @@ -91,7 +116,7 @@ def configure_plots(): mpl.rcParams['legend.scatterpoints'] = 1 mpl.rcParams['legend.fontsize']= 'medium' -def create_contoller_visualizations(filename, +def create_controller_visualizations(filename, file_type='pkl', plot_cost_vs_run=True, plot_parameters_vs_run=True, @@ -113,8 +138,8 @@ def create_contoller_visualizations(filename, visualization.plot_cost_vs_run() if plot_parameters_vs_run: visualization.plot_parameters_vs_run() - if plot_parameters_vs_cost: - visualization.plot_parameters_vs_cost() + #if plot_parameters_vs_cost: + # visualization.plot_parameters_vs_cost() class ControllerVisualizer(): ''' @@ -557,7 +582,8 @@ def plot_hyperparameters_vs_run(self): def create_neural_net_learner_visualizations(filename, file_type='pkl', - plot_cross_sections=True): + plot_cross_sections=True, + upload_cross_sections=False): ''' Creates plots from a neural nets learner file. @@ -571,8 +597,10 @@ def create_neural_net_learner_visualizations(filename, ''' visualization = NeuralNetVisualizer(filename, file_type=file_type) if plot_cross_sections: - visualization.plot_cross_sections() - visualization.plot_surface() + visualization.do_cross_sections(upload=upload_cross_sections) + visualization.plot_surface() + visualization.plot_density_surface() + visualization.plot_losses() class NeuralNetVisualizer(mll.NeuralNetLearner): @@ -600,8 +628,7 @@ def __init__(self, filename, file_type = 'pkl', **kwargs): self.has_trust_region = bool(np.array(self.training_dict['has_trust_region'])) self.trust_region = np.squeeze(np.array(self.training_dict['trust_region'], dtype=float)) - self.create_neural_net() - self.fit_neural_net() + self.import_neural_net() if np.all(np.isfinite(self.min_boundary)) and np.all(np.isfinite(self.min_boundary)): self.finite_flag = True @@ -649,46 +676,99 @@ def return_cross_sections(self, points=100, cross_section_center=None): self.log.error('cross_section_center not in boundaries:' + repr(cross_section_center)) raise ValueError - cross_parameter_arrays = [ np.linspace(min_p, max_p, points) for (min_p,max_p) in zip(self.min_boundary,self.max_boundary)] - cost_arrays = [] - for ind in range(self.num_params): - sample_parameters = np.array([cross_section_center for _ in range(points)]) - sample_parameters[:, ind] = cross_parameter_arrays[ind] - costs = self.predict_costs_from_param_array(sample_parameters) - cost_arrays.append(costs) - if self.cost_scaler.scale_: - cross_parameter_arrays = np.array(cross_parameter_arrays)/self.cost_scaler.scale_ - else: - cross_parameter_arrays = np.array(cross_parameter_arrays) - cost_arrays = self.cost_scaler.inverse_transform(np.array(cost_arrays)) - return (cross_parameter_arrays,cost_arrays) + res = [] + for net_index in range(self.num_nets): + cross_parameter_arrays = [ np.linspace(min_p, max_p, points) for (min_p,max_p) in zip(self.min_boundary,self.max_boundary)] + cost_arrays = [] + for ind in range(self.num_params): + sample_parameters = np.array([cross_section_center for _ in range(points)]) + sample_parameters[:, ind] = cross_parameter_arrays[ind] + costs = self.predict_costs_from_param_array(sample_parameters, net_index) + cost_arrays.append(costs) + if self.cost_scaler.scale_: + cross_parameter_arrays = np.array(cross_parameter_arrays)/self.cost_scaler.scale_ + else: + cross_parameter_arrays = np.array(cross_parameter_arrays) + cost_arrays = self.cost_scaler.inverse_transform(np.array(cost_arrays)) + res.append((cross_parameter_arrays, cost_arrays)) + return res - def plot_cross_sections(self): + def do_cross_sections(self, upload): ''' Produce a figure of the cross section about best cost and parameters ''' - global figure_counter, legend_loc - figure_counter += 1 - plt.figure(figure_counter) points = 100 - (_,cost_arrays) = self.return_cross_sections(points=points) rel_params = np.linspace(0,1,points) - for ind in range(self.num_params): - plt.plot(rel_params,cost_arrays[ind,:],'-',color=self.param_colors[ind]) - if self.has_trust_region: - axes = plt.gca() - ymin, ymax = axes.get_ylim() - ytrust = ymin + 0.1*(ymax - ymin) + all_cost_arrays = [a for _,a in self.return_cross_sections(points=points)] + for net_index, cost_arrays in enumerate(all_cost_arrays): + def prepare_plot(): + global figure_counter + figure_counter += 1 + fig = plt.figure(figure_counter) + axes = plt.gca() + for ind in range(self.num_params): + axes.plot(rel_params,cost_arrays[ind,:],'-',color=self.param_colors[ind],label=str(ind)) + if self.has_trust_region: + ymin, ymax = axes.get_ylim() + ytrust = ymin + 0.1*(ymax - ymin) + for ind in range(self.num_params): + axes.plot([self.scaled_trust_min[ind],self.scaled_trust_max[ind]],[ytrust,ytrust],'s', color=self.param_colors[ind]) + axes.set_xlabel(scale_param_label) + axes.set_xlim((0,1)) + axes.set_ylabel(cost_label) + axes.set_title('NN Learner: Predicted landscape' + ('with trust regions.' if self.has_trust_region else '.') + ' (' + str(net_index) + ')') + return fig + if upload: + plf = tls.mpl_to_plotly(prepare_plot()) + plf['layout']['showlegend'] = True + try: + url = py.plot(plf,auto_open=False) + print(url) + except pye.PlotlyRequestError: + print("Failed to upload due to quota restrictions") + prepare_plot() + artists = [] for ind in range(self.num_params): - plt.plot([self.scaled_trust_min[ind],self.scaled_trust_max[ind]],[ytrust,ytrust],'s', color=self.param_colors[ind]) - plt.xlabel(scale_param_label) - plt.xlim((0,1)) - plt.ylabel(cost_label) - plt.title('NN Learner: Predicted landscape' + ('with trust regions.' if self.has_trust_region else '.')) - artists = [] - for ind in range(self.num_params): - artists.append(plt.Line2D((0,1),(0,0), color=self.param_colors[ind], linestyle='-')) - plt.legend(artists,[str(x) for x in range(1,self.num_params+1)],loc=legend_loc) + artists.append(plt.Line2D((0,1),(0,0), color=self.param_colors[ind], linestyle='-')) + plt.legend(artists,[str(x) for x in range(1,self.num_params+1)],loc=legend_loc) + if self.num_nets > 1: + # And now create a plot showing the average, max and min for each cross section. + def prepare_plot(): + global figure_counter + figure_counter += 1 + fig = plt.figure(figure_counter) + axes = plt.gca() + for ind in range(self.num_params): + this_param_cost_array = np.array(all_cost_arrays)[:,ind,:] + mn = np.mean(this_param_cost_array, axis=0) + m = np.min(this_param_cost_array, axis=0) + M = np.max(this_param_cost_array, axis=0) + axes.plot(rel_params,mn,'-',color=self.param_colors[ind],label=str(ind)) + axes.plot(rel_params,m,'--',color=self.param_colors[ind],label=str(ind)) + axes.plot(rel_params,M,'--',color=self.param_colors[ind],label=str(ind)) + axes.set_xlabel(scale_param_label) + axes.set_xlim((0,1)) + axes.set_ylabel(cost_label) + axes.set_title('NN Learner: Average predicted landscape') + return fig + if upload: + plf = tls.mpl_to_plotly(prepare_plot()) + plf['layout']['showlegend'] = True + for i,d in enumerate(plf['data']): + d['legendgroup'] = str(int(i/3)) + if not i % 3 == 0: + d['showlegend'] = False + # Pretty sure this shouldn't be necessary, but it seems to be anyway. + d['line']['dash'] = 'dash' + try: + url = py.plot(plf,auto_open=False) + print(url) + except pye.PlotlyRequestError: + print("Failed to upload due to quota restrictions") + prepare_plot() + for ind in range(self.num_params): + artists.append(plt.Line2D((0,1),(0,0), color=self.param_colors[ind], linestyle='-')) + plt.legend(artists,[str(x) for x in range(1,self.num_params+1)],loc=legend_loc) def plot_surface(self): ''' @@ -706,9 +786,45 @@ def plot_surface(self): params = [(x,y) for x in param_set[0] for y in param_set[1]] costs = self.predict_costs_from_param_array(params) ax.scatter([param[0] for param in params], [param[1] for param in params], costs) - ax.set_zlim(top=100) + ax.set_zlim(top=500,bottom=0) ax.set_xlabel('x') ax.set_ylabel('y') ax.set_zlabel('cost') ax.scatter(self.all_params[:,0], self.all_params[:,1], self.all_costs, c='r') + + def plot_density_surface(self): + ''' + Produce a density plot of the cost surface (only works when there are 2 parameters) + ''' + if self.num_params != 2: + return + global figure_counter + figure_counter += 1 + fig = plt.figure(figure_counter) + + points = 50 + xs, ys = np.meshgrid( + np.linspace(self.min_boundary[0], self.max_boundary[0], points), + np.linspace(self.min_boundary[1], self.max_boundary[1], points)) + zs_list = self.predict_costs_from_param_array(list(zip(xs.flatten(),ys.flatten()))) + zs = np.array(zs_list).reshape(points,points) + plt.pcolormesh(xs,ys,zs) + plt.scatter(self.all_params[:,0], self.all_params[:,1], c=self.all_costs, vmin=np.min(zs), vmax=np.max(zs), s=100) + plt.colorbar() + plt.xlabel("Param 0") + plt.ylabel("Param 1") + + def plot_losses(self): + ''' + Produce a figure of the loss as a function of training run. + ''' + global figure_counter + figure_counter += 1 + fig = plt.figure(figure_counter) + + losses = self.get_losses() + plt.scatter(range(len(losses)), losses) + plt.xlabel("Run") + plt.ylabel("Training cost") + plt.title('Loss vs training run.') diff --git a/requirements.txt b/requirements.txt index 5012a5c..010acc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ docutils>=0.3 numpy>=1.11 scipy>=0.17 matplotlib>=1.5 +plotly>=2.0.8 pytest>=2.9 scikit-learn>=0.18 -setuptools>=26 \ No newline at end of file +setuptools>=26 +tensorflow>=1.1.0 diff --git a/setup.py b/setup.py index c6b6017..3174f97 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,10 @@ def main(): 'numpy>=1.11', 'scipy>=0.17', 'matplotlib>=1.5', + 'plotly>=2.0.8', 'pytest>=2.9', - 'scikit-learn>=0.18'], + 'scikit-learn>=0.18', + 'tensorflow>=1.1.0'], tests_require=['pytest','setuptools>=26'], package_data = { @@ -60,4 +62,4 @@ def main(): if __name__=='__main__': mp.freeze_support() - main() \ No newline at end of file + main() diff --git a/tools/landscape_vis.py b/tools/landscape_vis.py new file mode 100644 index 0000000..a63a21b --- /dev/null +++ b/tools/landscape_vis.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import argparse + +parser = argparse.ArgumentParser(description='Plot cross sections of the predicted landscape, and optionally upload them via plotly. Must be run from the same directory as M-LOOP was run.') +parser.add_argument("controller_filename") +parser.add_argument("learner_filename") +parser.add_argument("learner_type") +parser.add_argument("-u","--upload",action="store_true",help="upload plots to the interwebs") +args = parser.parse_args() + +import mloop.visualizations as mlv +import mloop.utilities as mlu +import logging + +mlu.config_logger(log_filename=None, console_log_level=logging.DEBUG) + +mlv.show_all_default_visualizations_from_archive(args.controller_filename, args.learner_filename, args.learner_type, upload_cross_sections=args.upload)