From 9b968f7580a817510959f68e086561c815ef7bf2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 18 Jun 2019 09:45:26 -0500 Subject: [PATCH 001/644] Initial branch commit for persistent APOSMM work; starting work outside APOSMM --- libensemble/alloc_funcs/start_persistent_local_opt_gens.py | 2 +- libensemble/gen_funcs/uniform_or_localopt.py | 1 + ...hump_camel_uniform_sampling_with_persistent_localopt_gens.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 3a0bbe0a3..3ddc8a969 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -63,7 +63,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per # Start persistent generator for local opt run unless it would use all workers if starting_inds and gen_count + 1 < len(W): # Start at the best possible starting point - ind = starting_inds[np.argmin(H['f'][starting_inds])] + ind = np.array([0,1]) gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']], np.atleast_1d(ind), persis_info[i], persistent=True) diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index 7c6950634..c839c26d1 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -20,6 +20,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): """ if libE_info.get('persistent'): + import ipdb; ipdb.set_trace() x_opt, persis_info_updates, tag_out = try_and_run_nlopt(H, gen_specs, libE_info) O = [] return O, persis_info_updates, tag_out diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index bcaa94c91..c17878827 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -53,7 +53,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 1000, 'elapsed_wallclock_time': 300} +exit_criteria = {'sim_max': 1000} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 968237200d3924eaccfa2d4604d21a4d54b9d6bf Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 28 Jun 2019 14:50:18 -0500 Subject: [PATCH 002/644] WIP: starts a mp implementation of the asynchronous runs; --- libensemble/libE_worker.py | 58 ++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/libensemble/libE_worker.py b/libensemble/libE_worker.py index 9e217fcf0..30a9273b6 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/libE_worker.py @@ -170,7 +170,11 @@ def _handle_calc(self, Work, calc_in): with timer: with self.loc_stack.loc(calc_type): logger.debug("Calling calc {}".format(calc_type)) + # this would be stuck here. + # I would fork here. out = calc(calc_in, Work['persis_info'], Work['libE_info']) + # when do I join. + # So every time I get an inpt. logger.debug("Return from calc call") assert isinstance(out, tuple), \ @@ -251,17 +255,55 @@ def run(self): logger.info("Worker {} initiated on node {}". format(self.workerID, socket.gethostname())) + rank = self.comm.get_Rank() + from multiprocessing import Process + for worker_iter in count(start=1): + # This 'for' loop is like a while loop. logger.debug("Iteration {}".format(worker_iter)) - mtag, Work = self.comm.recv() - if mtag == STOP_TAG: - break - - response = self._handle(Work) - if response is None: - break - self.comm.send(0, response) + if rank == 1: + #FIXME: this will not work always, we need to + # persistent mode operation + mtag, Work = self.comm.recv() + if mtag == STOP_TAG: + break + + # what does this repsonse stand for? + # this is where we fork? + # yep + + p = Process(target=self._handle, args=(Work,)) + p.start() + # Ok.. + # Things to do: append the result of the run to one of our + # variables + + # response = self._handle(Work) + if response is None: + break + + # this is where we send. + self.comm.send(0, response) + p.join() + else: + mtag, Work = self.comm.recv() + if mtag == STOP_TAG: + break + + # what does this repsonse stand for? + # this is where we fork? + # yep + + response = self._handle(Work) + if response is None: + break + + print(response) + 1/0 + + # this is where we send. + self.comm.send(0, response) except Exception as e: self.comm.send(0, WorkerErrMsg(str(e), format_exc())) From ec9425b9b9707a81e72a9ed39903eaa0662181d8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 5 Jul 2019 08:24:07 -0500 Subject: [PATCH 003/644] Reverting some changes --- .../alloc_funcs/start_only_persistent.py | 8 +-- .../start_persistent_local_opt_gens.py | 2 +- .../gen_funcs/persistent_uniform_sampling.py | 1 + libensemble/gen_funcs/uniform_or_localopt.py | 1 - libensemble/libE_worker.py | 58 +++---------------- ..._sampling_with_persistent_localopt_gens.py | 5 +- 6 files changed, 16 insertions(+), 59 deletions(-) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index a96186018..9bef765b3 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -13,11 +13,12 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): :See: ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ + import ipdb; ipdb.set_trace() Work = {} gen_count = count_persis_gens(W) - # If i is idle, but in persistent mode, and its calculated values have + # If i is in persistent mode, and any of its calculated values have # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): gen_inds = (H['gen_worker'] == i) @@ -31,14 +32,13 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): task_avail = ~H['given'] for i in avail_worker_ids(W, persistent=False): if np.any(task_avail): - - # perform sim evaluations from existing runs (if they exist). + # perform sim evaluations (if they exist in History). sim_ids_to_send = np.nonzero(task_avail)[0][0] # oldest point sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), persis_info[i]) task_avail[sim_ids_to_send] = False elif gen_count == 0: - # Finally, generate points since there is nothing else to do. + # Finally, call a persistent generator as there is nothing else to do. gen_count += 1 gen_work(Work, i, gen_specs['in'], [], persis_info[i], persistent=True) diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 3ddc8a969..3a0bbe0a3 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -63,7 +63,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per # Start persistent generator for local opt run unless it would use all workers if starting_inds and gen_count + 1 < len(W): # Start at the best possible starting point - ind = np.array([0,1]) + ind = starting_inds[np.argmin(H['f'][starting_inds])] gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']], np.atleast_1d(ind), persis_info[i], persistent=True) diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index 3d13916a9..66a8ea21e 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -12,6 +12,7 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): :See: ``libensemble/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ + import ipdb; ipdb.set_trace() ub = gen_specs['ub'] lb = gen_specs['lb'] n = len(lb) diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index c839c26d1..7c6950634 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -20,7 +20,6 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): """ if libE_info.get('persistent'): - import ipdb; ipdb.set_trace() x_opt, persis_info_updates, tag_out = try_and_run_nlopt(H, gen_specs, libE_info) O = [] return O, persis_info_updates, tag_out diff --git a/libensemble/libE_worker.py b/libensemble/libE_worker.py index 30a9273b6..9e217fcf0 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/libE_worker.py @@ -170,11 +170,7 @@ def _handle_calc(self, Work, calc_in): with timer: with self.loc_stack.loc(calc_type): logger.debug("Calling calc {}".format(calc_type)) - # this would be stuck here. - # I would fork here. out = calc(calc_in, Work['persis_info'], Work['libE_info']) - # when do I join. - # So every time I get an inpt. logger.debug("Return from calc call") assert isinstance(out, tuple), \ @@ -255,55 +251,17 @@ def run(self): logger.info("Worker {} initiated on node {}". format(self.workerID, socket.gethostname())) - rank = self.comm.get_Rank() - from multiprocessing import Process - for worker_iter in count(start=1): - # This 'for' loop is like a while loop. logger.debug("Iteration {}".format(worker_iter)) - if rank == 1: - #FIXME: this will not work always, we need to - # persistent mode operation - mtag, Work = self.comm.recv() - if mtag == STOP_TAG: - break - - # what does this repsonse stand for? - # this is where we fork? - # yep - - p = Process(target=self._handle, args=(Work,)) - p.start() - # Ok.. - # Things to do: append the result of the run to one of our - # variables - - # response = self._handle(Work) - if response is None: - break - - # this is where we send. - self.comm.send(0, response) - p.join() - else: - mtag, Work = self.comm.recv() - if mtag == STOP_TAG: - break - - # what does this repsonse stand for? - # this is where we fork? - # yep - - response = self._handle(Work) - if response is None: - break - - print(response) - 1/0 - - # this is where we send. - self.comm.send(0, response) + mtag, Work = self.comm.recv() + if mtag == STOP_TAG: + break + + response = self._handle(Work) + if response is None: + break + self.comm.send(0, response) except Exception as e: self.comm.send(0, WorkerErrMsg(str(e), format_exc())) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index c17878827..1f522c336 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -14,8 +14,7 @@ # TESTSUITE_COMMS: mpi local tcp # TESTSUITE_NPROCS: 3 4 -import sys -import numpy as np +import sys import numpy as np # Import libEnsemble main, sim_specs, gen_specs, alloc_specs, and persis_info from libensemble.libE import libE @@ -53,7 +52,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 1000} +exit_criteria = {'sim_max': 1000, 'elapsed_wallclock_time': 300} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From b09c7d0bd224301e3e71b9506d471a23a79d73ba Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 5 Jul 2019 08:26:33 -0500 Subject: [PATCH 004/644] Cleaning up a bit --- libensemble/alloc_funcs/start_only_persistent.py | 1 - libensemble/gen_funcs/persistent_uniform_sampling.py | 1 - ...ump_camel_uniform_sampling_with_persistent_localopt_gens.py | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 9bef765b3..2d2d93fdc 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -13,7 +13,6 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): :See: ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ - import ipdb; ipdb.set_trace() Work = {} gen_count = count_persis_gens(W) diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index 66a8ea21e..3d13916a9 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -12,7 +12,6 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): :See: ``libensemble/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ - import ipdb; ipdb.set_trace() ub = gen_specs['ub'] lb = gen_specs['lb'] n = len(lb) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index 1f522c336..bcaa94c91 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -14,7 +14,8 @@ # TESTSUITE_COMMS: mpi local tcp # TESTSUITE_NPROCS: 3 4 -import sys import numpy as np +import sys +import numpy as np # Import libEnsemble main, sim_specs, gen_specs, alloc_specs, and persis_info from libensemble.libE import libE From 54602bc30615bef35102573776550a3757733c7a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 5 Jul 2019 08:27:26 -0500 Subject: [PATCH 005/644] Adding new files for persistent aposmm --- libensemble/gen_funcs/persistent_aposmm.py | 890 ++++++++++++++++++ .../test_6-hump_camel_persistent_aposmm.py | 55 ++ 2 files changed, 945 insertions(+) create mode 100644 libensemble/gen_funcs/persistent_aposmm.py create mode 100644 libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py new file mode 100644 index 000000000..fe747da27 --- /dev/null +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -0,0 +1,890 @@ +""" +This module contains methods used our implementation of the Asynchronously +Parallel Optimization Solver for finding Multiple Minima (APOSMM) method +described in detail in the paper +`https://doi.org/10.1007/s12532-017-0131-4 `_ +""" +__all__ = ['aposmm_logic', 'initialize_APOSMM', + 'decide_where_to_start_localopt', 'update_history_dist'] + +import sys +import pickle +import traceback +import numpy as np +from scipy.spatial.distance import cdist, pdist, squareform +from scipy import optimize as scipy_optimize + +from mpi4py import MPI +from petsc4py import PETSc + +from numpy.lib.recfunctions import merge_arrays + +from math import log, gamma, pi, sqrt + +import nlopt + + +class APOSMMException(Exception): + "Raised for any exception in APOSMM" + + +def aposmm(H, persis_info, gen_specs, libE_info): + """ + APOSMM coordinates multiple local optimization runs, starting from points + which do not have a better point nearby (within a distance ``r_k``). This + generation function produces/requires the following fields in ``H``: + + - ``'x' [n floats]``: Parameters being optimized over + - ``'x_on_cube' [n floats]``: Parameters scaled to the unit cube + - ``'f' [float]``: Objective function being minimized + - ``'local_pt' [bool]``: True if point from a local optimization run + - ``'dist_to_unit_bounds' [float]``: Distance to domain boundary + - ``'dist_to_better_l' [float]``: Dist to closest better local opt point + - ``'dist_to_better_s' [float]``: Dist to closest better sample point + - ``'ind_of_better_l' [int]``: Index of point ``'dist_to_better_l``' away + - ``'ind_of_better_s' [int]``: Index of point ``'dist_to_better_s``' away + - ``'started_run' [bool]``: True if point has started a local opt run + - ``'num_active_runs' [int]``: Number of active local runs point is in + - ``'local_min' [float]``: True if point has been ruled a local minima + - ``'sim_id' [int]``: Row number of entry in history + + and optionally + + - ``'priority' [float]``: Value quantifying a point's desirability + - ``'f_i' [float]``: Value of ith objective component (if single_component) + - ``'fvec' [m floats]``: All objective components (if calculated together) + - ``'obj_component' [int]``: Index corresponding to value in ``'f_i``' + - ``'pt_id' [int]``: Identify the point (useful when evaluating different + objective components for a given ``'x'``) + + When using libEnsemble to do individual objective component evaluations, + APOSMM will return ``gen_specs['components']`` copies of each point, but + the component=0 entry of each point will only be considered when + + - deciding where to start a run, + - best nearby point, + - storing the order of the points is the run + - storing the combined objective function value + - etc + + Necessary quantities in ``gen_specs`` are: + + - ``'lb' [n floats]``: Lower bound on search domain + - ``'ub' [n floats]``: Upper bound on search domain + - ``'initial_sample_size' [int]``: Number of uniformly sampled points + must be returned (non-nan value) before a local opt run is started + + - ``'localopt_method' [str]``: Name of an NLopt, PETSc/TAO, or SciPy method + (see 'advance_local_run' below for supported methods) + + Optional ``gen_specs`` entries are: + + - ``'sample_points' [numpy array]``: Points to be sampled (original domain) + - ``'combine_component_func' [func]``: Function to combine obj components + - ``'components' [int]``: Number of objective components + - ``'dist_to_bound_multiple' [float in (0,1]]``: What fraction of the + distance to the nearest boundary should the initial step size be in + localopt runs + - ``'high_priority_to_best_localopt_runs': [bool]``: True if localopt runs + with smallest observed function value are given priority + - ``'lhs_divisions' [int]``: Number of Latin hypercube sampling partitions + (0 or 1 results in uniform sampling) + - ``'min_batch_size' [int]``: Lower bound on the number of points given + every time APOSMM is called + - ``'mu' [float]``: Distance from the boundary that all localopt starting + points must satisfy + - ``'nu' [float]``: Distance from identified minima that all starting + points must satisfy + - ``'single_component_at_a_time' [bool]``: True if single objective + components will be evaluated at a time + - ``'rk_const' [float]``: Multiplier in front of the r_k value + - ``'max_active_runs' [int]``: Bound on number of runs APOSMM is advancing + + And ``gen_specs`` convergence tolerances for NLopt, PETSc/TAO, SciPy + + - ``'fatol' [float]``: + - ``'ftol_abs' [float]``: + - ``'ftol_rel' [float]``: + - ``'gatol' [float]``: + - ``'grtol' [float]``: + - ``'xtol_abs' [float]``: + - ``'xtol_rel' [float]``: + - ``'tol' [float]``: + + + As a default, APOSMM starts a local optimization runs from a point that: + + - is not in an active local optimization run, + - is more than ``mu`` from the boundary (in the unit-cube domain), + - is more than ``nu`` from identified minima (in the unit-cube domain), + - does not have a better point within a distance ``r_k`` of it. + + If the above results in more than ``'max_active_runs'`` being advanced, the + best point in each run is determined and the dist_to_better is computed + (with inf being the value for the best run). Then those + ``'max_active_runs'`` runs with largest dist_to_better are advanced + (breaking ties arbitrarily). + + :Note: + ``gen_specs['combine_component_func']`` must be defined when there are + multiple objective components. + + :Note: + APOSMM critically uses ``persis_info`` to store information about + active runs, order of points in each run, etc. The allocation function + must ensure it's always given. + + :See: + ``libensemble/tests/regression_tests/test_branin_aposmm.py`` + for basic APOSMM usage. + :See: + ``libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` + for an example of APOSMM coordinating multiple local optimization runs + for an objective with more than one component. + """ + """ + Description of intermediate variables in aposmm_logic: + + n: domain dimension + c_flag: True if giving libEnsemble individual components of fvec + to evaluate. (Note if c_flag is True, APOSMM will use + only the component to store the function value f) + n_s: the number of complete evaluations of sampled points + updated_inds: indices of H that have been updated (and so all their + information must be sent back to libE manager to update) + O: new points to be sent back to the history + + + When re-running a local opt method to get the next point: + advance_local_run.x_new: stores the first new point requested by + a local optimization method + advance_local_run.pt_in_run: counts function evaluations to know + when a new point is given + + starting_inds: indices where a runs should be started. + active_runs: indices of active local optimization runs + sorted_run_inds: indices of the considered run (in the order they were + requested by the localopt method) + x_opt: the reported minimum from a localopt run (disregarded + unless exit_code isn't 0) + exit_code: 0 if a new localopt point has been found, otherwise it's + the NLopt/TAO/SciPy code + samples_needed: Number of additional uniformly drawn samples needed + + + Description of persistent variables used to maintain the state of APOSMM + + persis_info['total_runs']: Running count of started/completed localopt runs + persis_info['run_order']: Sequence of indices of points in unfinished runs + persis_info['old_runs']: Sequence of indices of points in finished runs + + """ + import ipdb; ipdb.set_trace() + + n, n_s, c_flag, rk_const, mu, nu, comm = initialize_APOSMM(H, gen_specs, libE_info) + + tag = None + + while tag not in [STOP_TAG, PERSIS_STOP]: + n_s = count_samples(H,gen_specs) + + if n_s >= gen_specs['initial_sample_size']: + update_history_dist(H, n, gen_specs, c_flag) + + starting_inds = decide_where_to_start_localopt(H, n_s, rk_const, mu, nu) + + for ind in starting_inds: + # Mark point as having started a run + new_run_num = total_runs + total_runs += 1 + H['started_run'][ind] = 1 + run_order[new_run_num] = [ind] + + # Initialize child process for new localopt run + x_new = start_localopt(H, gen_specs, c_flag) + + # Send x_new to manager for evaluation + O = np.zeros(1, dtype=gen_specs['out']) + add_to_O(O, x_new, H, gen_specs, c_flag, persis_info, local_flag=1) + tag, Work, calc_in = send_mgr_worker_msg(comm, O) + + # Receive from manager (if possible), contact child process to get + # x_new, and send back to manager + get_mgr_worker_msg(comm, status) + + else: + O = np.zeros(gen_specs['initial_sample_size'], dtype=gen_specs['out']) + sampled_points = persis_info['rand_stream'].uniform(0, 1, (len(O), n)) + on_cube = True + add_to_O(O, sampled_points, H, gen_specs, c_flag, persis_info, on_cube=on_cube) + + tag, Work, calc_in = send_mgr_worker_msg(comm, O) + + return O, persis_info, tag + + +def add_to_O(O, pts, H, gen_specs, c_flag, persis_info, local_flag=0, + sorted_run_inds=[], run=[], on_cube=True): + """ + Adds points to O, the numpy structured array to be sent back to the manager + """ + + assert not local_flag or len(pts) == 1, "Can't > 1 local points" + + original_len_O = len(O) + + len_H = len(H) + ub = gen_specs['ub'] + lb = gen_specs['lb'] + if c_flag: + m = gen_specs['components'] + + assert len_H % m == 0, "Number of points in len_H not congruent to 0 mod 'components'" + pt_ids = np.sort(np.tile(np.arange((len_H+original_len_O)/m, (len_H+original_len_O)/m+len(pts)), (1, m))) + pts = np.tile(pts, (m, 1)) + + num_pts = len(pts) + + O.resize(len(O)+num_pts, refcheck=False) # Adds num_pts rows of zeros to O + + if on_cube: + O['x_on_cube'][-num_pts:] = pts + O['x'][-num_pts:] = pts*(ub-lb)+lb + else: + O['x_on_cube'][-num_pts:] = (pts-lb)/(ub-lb) + O['x'][-num_pts:] = pts + + O['sim_id'][-num_pts:] = np.arange(len_H+original_len_O, len_H+original_len_O+num_pts) + O['local_pt'][-num_pts:] = local_flag + + O['dist_to_unit_bounds'][-num_pts:] = np.inf + O['dist_to_better_l'][-num_pts:] = np.inf + O['dist_to_better_s'][-num_pts:] = np.inf + O['ind_of_better_l'][-num_pts:] = -1 + O['ind_of_better_s'][-num_pts:] = -1 + + if c_flag: + O['obj_component'][-num_pts:] = np.tile(range(0, m), (1, num_pts//m)) + O['pt_id'][-num_pts:] = pt_ids + + if local_flag: + O['num_active_runs'][-num_pts] += 1 + # O['priority'][-num_pts:] = 1 + # O['priority'][-num_pts:] = np.random.uniform(0,1,num_pts) + if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: + O['priority'][-num_pts:] = -min(H['f'][persis_info['run_order'][run]]) # Give highest priority to run with lowest function value + else: + O['priority'][-num_pts:] = persis_info['rand_stream'].uniform(0, 1, num_pts) + persis_info['run_order'][run].append(O[-num_pts]['sim_id']) + else: + if c_flag: + # p_tmp = np.sort(np.tile(np.random.uniform(0,1,num_pts/m),(m,1))) # If you want all "duplicate points" to have the same priority (meaning libEnsemble gives them all at once) + # p_tmp = np.random.uniform(0,1,num_pts) + p_tmp = persis_info['rand_stream'].uniform(0, 1, num_pts) + else: + # p_tmp = np.random.uniform(0,1,num_pts) + # persis_info['rand_stream'].uniform(lb,ub,(1,n)) + if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: + p_tmp = -np.inf*np.ones(num_pts) + else: + p_tmp = persis_info['rand_stream'].uniform(0, 1, num_pts) + O['priority'][-num_pts:] = p_tmp + # O['priority'][-num_pts:] = 1 + + return persis_info + + +def update_history_dist(H, n, gen_specs, c_flag): + """ + Updates distances/indices after new points that have been evaluated. + + :See: + ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + """ + + new_inds = np.where(~H['known_to_aposmm'])[0] + + if c_flag: + for v in np.unique(H['pt_id'][new_inds]): + inds = H['pt_id'] == v + H['f'][inds] = np.inf + H['f'][np.where(inds)[0][0]] = gen_specs['combine_component_func'](H['f_i'][inds]) + + p = np.logical_and.reduce((H['returned'], H['obj_component'] == 0, ~np.isnan(H['f']))) + else: + p = np.logical_and.reduce((H['returned'], ~np.isnan(H['f']))) + + for new_ind in new_inds: + # Loop over new returned points and update their distances + if p[new_ind]: + H['known_to_aposmm'][new_ind] = True + + # Compute distance to boundary + H['dist_to_unit_bounds'][new_ind] = min(min(np.ones(n)-H['x_on_cube'][new_ind]), min(H['x_on_cube'][new_ind]-np.zeros(n))) + + dist_to_all = cdist(H['x_on_cube'][[new_ind]], H['x_on_cube'][p], 'euclidean').flatten() + new_better_than = H['f'][new_ind] < H['f'][p] + + # Update any other points if new_ind is closer and better + if H['local_pt'][new_ind]: + inds_of_p = np.logical_and(dist_to_all < H['dist_to_better_l'][p], new_better_than) + updates = np.where(p)[0][inds_of_p] + H['dist_to_better_l'][updates] = dist_to_all[inds_of_p] + H['ind_of_better_l'][updates] = new_ind + else: + inds_of_p = np.logical_and(dist_to_all < H['dist_to_better_s'][p], new_better_than) + updates = np.where(p)[0][inds_of_p] + H['dist_to_better_s'][updates] = dist_to_all[inds_of_p] + H['ind_of_better_s'][updates] = new_ind + + # Since we allow equality when deciding better_than_new_l and + # better_than_new_s, we have to prevent new_ind from being its own + # better point. + better_than_new_l = np.logical_and.reduce((~new_better_than, H['local_pt'][p], H['sim_id'][p] != new_ind)) + better_than_new_s = np.logical_and.reduce((~new_better_than, ~H['local_pt'][p], H['sim_id'][p] != new_ind)) + + # Who is closest to ind and better + if np.any(better_than_new_l): + ind = dist_to_all[better_than_new_l].argmin() + H['ind_of_better_l'][new_ind] = H['sim_id'][p][np.nonzero(better_than_new_l)[0][ind]] + H['dist_to_better_l'][new_ind] = dist_to_all[better_than_new_l][ind] + + if np.any(better_than_new_s): + ind = dist_to_all[better_than_new_s].argmin() + H['ind_of_better_s'][new_ind] = H['sim_id'][p][np.nonzero(better_than_new_s)[0][ind]] + H['dist_to_better_s'][new_ind] = dist_to_all[better_than_new_s][ind] + + # if not ignore_L8: + # r_k = calc_rk(len(H['x_on_cube'][0]), n_s, rk_const, lhs_divisions) + # H['worse_within_rk'][new_ind][p] = np.logical_and.reduce((H['f'][new_ind] <= H['f'][p], dist_to_all <= r_k)) + + # # Add trues if new point is 'worse_within_rk' + # inds_to_change = np.logical_and.reduce((H['dist_to_all'][p,new_ind] <= r_k, H['f'][new_ind] >= H['f'][p], H['sim_id'][p] != new_ind)) + # H['worse_within_rk'][inds_to_change,new_ind] = True + + # if not H['local_pt'][new_ind]: + # H['worse_within_rk'][H['dist_to_all'] > r_k] = False + + + +def update_history_optimal(x_opt, H, run_inds): + """ + Updated the history after any point has been declared a local minimum + """ + + # opt_ind = np.where(np.logical_and(np.equal(x_opt,H['x_on_cube']).all(1),~np.isinf(H['f'])))[0] # This fails on some problems. x_opt is 1e-16 away from the point that was given and opt_ind is empty + run_inds = np.unique(run_inds) + + dists = np.linalg.norm(H['x_on_cube'][run_inds]-x_opt, axis=1) + ind = np.argmin(dists) + opt_ind = run_inds[ind] + + if dists[ind] > 1e-15: + print("Dist from x_opt to closest point is:"+str(dists[ind])) + print("Report this!") + print(x_opt) + print(run_inds) + sys.stdout.flush() + assert dists[ind] <= 1e-15, "Closest point to x_opt not within 1e-15?" + + failsafe = np.logical_and(H['f'][run_inds] < H['f'][opt_ind], dists < 1e-8) + if np.any(failsafe): + # Rare event, but want to not start another run next to a minimum + print('Marking more than 1 point in this run as a min!') + print("Report this!") + sys.stdout.flush() + H['local_min'][run_inds[failsafe]] = 1 + + H['local_min'][opt_ind] = 1 + H['num_active_runs'][run_inds] -= 1 + + +def advance_local_run(H, gen_specs, c_flag, run, persis_info): + """ + Moves a local optimization method one iteration forward. We currently do + this by feeding all past evaluations from a run to the method and then + storing the first new point generated + """ + + while 1: + sorted_run_inds = persis_info['run_order'][run] + advance_local_run.x_new = np.ones((1, len(gen_specs['ub'])))*np.inf + advance_local_run.pt_in_run = 0 + + if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', + 'LN_COBYLA', 'LN_NELDERMEAD', + 'LD_MMA']: + + if gen_specs['localopt_method'] in ['LD_MMA']: + fields_to_pass = ['x_on_cube', 'f', 'grad'] + else: + fields_to_pass = ['x_on_cube', 'f'] + + try: + x_opt, exit_code = set_up_and_run_nlopt(H[fields_to_pass][sorted_run_inds], gen_specs) + except Exception as e: + x_opt = 0 + exit_code = 0 + display_exception(e) + + elif gen_specs['localopt_method'] in ['pounders']: + + if c_flag: + Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, gen_specs['components'])]) + for i, ind in enumerate(sorted_run_inds): + a1 = H['pt_id'] == H['pt_id'][ind] + Run_H_F['fvec'][i, :] = H['f_i'][a1] + Run_H = merge_arrays([H[['x_on_cube']][sorted_run_inds], Run_H_F], flatten=True) + else: + Run_H = H[['x_on_cube', 'fvec']][sorted_run_inds] + + try: + x_opt, exit_code = set_up_and_run_tao(Run_H, gen_specs) + except Exception as e: + x_opt = 0 + exit_code = 0 + display_exception(e) + + elif gen_specs['localopt_method'] == 'scipy_COBYLA': + + fields_to_pass = ['x_on_cube', 'f'] + + try: + x_opt, exit_code = set_up_and_run_scipy_minimize(H[fields_to_pass][sorted_run_inds], gen_specs) + except Exception as e: + x_opt = 0 + exit_code = 0 + display_exception(e) + + else: + raise APOSMMException("Unknown localopt method. Exiting") + + match_ind = np.equal(advance_local_run.x_new, H['x_on_cube']).all(1) + if ~match_ind.any(): + # Generated a new point + break + else: + # We need to add a previously evaluated point into this run + persis_info['run_order'][run].append(np.nonzero(match_ind)[0][0]) + + return x_opt, exit_code, persis_info, sorted_run_inds, advance_local_run.x_new + + +def set_up_and_run_scipy_minimize(Run_H, gen_specs): + """ Set up objective and runs scipy + + Declares the appropriate syntax for our special objective function to read + through Run_H, sets the parameters and starting points for the run. + """ + + def scipy_obj_fun(x, Run_H): + out = look_in_history(x, Run_H) + + return out + + obj = lambda x: scipy_obj_fun(x, Run_H) + x0 = Run_H['x_on_cube'][0] + + # Construct the bounds in the form of constraints + cons = [] + for factor in range(len(x0)): + lo = {'type': 'ineq', + 'fun': lambda x, lb=gen_specs['lb'][factor], i=factor: x[i]-lb} + up = {'type': 'ineq', + 'fun': lambda x, ub=gen_specs['ub'][factor], i=factor: ub-x[i]} + cons.append(lo) + cons.append(up) + + method = gen_specs['localopt_method'][6:] + res = scipy_optimize.minimize(obj, x0, method=method, options={'maxiter': len(Run_H['x_on_cube'])+1, 'tol': gen_specs['tol']}) + + if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum + exit_code = 0 + else: + if method == 'COBYLA': + assert res['status'] == 1, "Unknown status for COBYLA" + exit_code = 1 + + x_opt = res['x'] + return x_opt, exit_code + + +def set_up_and_run_nlopt(Run_H, gen_specs): + """ Set up objective and runs nlopt + + Declares the appropriate syntax for our special objective function to read + through Run_H, sets the parameters and starting points for the run. + """ + + assert 'xtol_rel' or 'xtol_abs' or 'ftol_rel' or 'ftol_abs' in gen_specs, "NLopt can cycle if xtol_rel, xtol_abs, ftol_rel, or ftol_abs are not set" + + def nlopt_obj_fun(x, grad, Run_H): + out = look_in_history(x, Run_H) + + if gen_specs['localopt_method'] in ['LD_MMA']: + grad[:] = out[1] + out = out[0] + + return out + + n = len(gen_specs['ub']) + + opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) + + lb = np.zeros(n) + ub = np.ones(n) + opt.set_lower_bounds(lb) + opt.set_upper_bounds(ub) + x0 = Run_H['x_on_cube'][0] + + # Care must be taken here because a too-large initial step causes nlopt to move the starting point! + dist_to_bound = min(min(ub-x0), min(x0-lb)) + assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" + + if 'dist_to_bound_multiple' in gen_specs: + opt.set_initial_step(dist_to_bound*gen_specs['dist_to_bound_multiple']) + else: + opt.set_initial_step(dist_to_bound) + + opt.set_maxeval(len(Run_H)+1) # evaluate one more point + opt.set_min_objective(lambda x, grad: nlopt_obj_fun(x, grad, Run_H)) + if 'xtol_rel' in gen_specs: + opt.set_xtol_rel(gen_specs['xtol_rel']) + if 'ftol_rel' in gen_specs: + opt.set_ftol_rel(gen_specs['ftol_rel']) + if 'xtol_abs' in gen_specs: + opt.set_xtol_abs(gen_specs['xtol_abs']) + if 'ftol_abs' in gen_specs: + opt.set_ftol_abs(gen_specs['ftol_abs']) + + x_opt = opt.optimize(x0) + exit_code = opt.last_optimize_result() + + if exit_code == 5: # NLOPT code for exhausting budget of evaluations, so not at a minimum + exit_code = 0 + + return x_opt, exit_code + + +def set_up_and_run_tao(Run_H, gen_specs): + """ Set up objective and runs PETSc on the comm_self communicator + + Declares the appropriate syntax for our special objective function to read + through Run_H, sets the parameters and starting points for the run. + """ + tao_comm = MPI.COMM_SELF + n = len(gen_specs['ub']) + m = len(Run_H['fvec'][0]) + + def pounders_obj_func(tao, X, F, Run_H): + F.array = look_in_history(X.array, Run_H, vector_return=True) + return F + + # def blmvm_obj_func(tao, X, G, Run_H): + # (f, grad) = look_in_history_fd_grad(X.array, Run_H) + # G.array = grad + # return f + + # Create starting point, bounds, and tao object + x = PETSc.Vec().create(tao_comm) + x.setSizes(n) + x.setFromOptions() + x.array = Run_H['x_on_cube'][0] + lb = x.duplicate() + ub = x.duplicate() + lb.array = 0*np.ones(n) + ub.array = 1*np.ones(n) + tao = PETSc.TAO().create(tao_comm) + tao.setType(gen_specs['localopt_method']) + + # if gen_specs['localopt_method'] == 'pounders': + f = PETSc.Vec().create(tao_comm) + f.setSizes(m) + f.setFromOptions() + + delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + + PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) + # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') + if hasattr(tao, 'setResidual'): + tao.setResidual(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) + else: + tao.setSeparableObjective(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) + # elif gen_specs['localopt_method'] == 'blmvm': + # g = PETSc.Vec().create(tao_comm) + # g.setSizes(n) + # g.setFromOptions() + # tao.setObjectiveGradient(lambda tao, x, g: blmvm_obj_func(tao, x, g, Run_H)) + + # Set everything for tao before solving + PETSc.Options().setValue('-tao_max_funcs', str(len(Run_H)+1)) + tao.setFromOptions() + tao.setVariableBounds((lb, ub)) + # tao.setObjectiveTolerances(fatol=gen_specs['fatol'], frtol=gen_specs['frtol']) + # tao.setGradientTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + tao.setInitial(x) + + tao.solve(x) + + x_opt = tao.getSolution().getArray() + exit_code = tao.getConvergedReason() + # print(exit_code) + # print(tao.view()) + # print(x_opt) + + # if gen_specs['localopt_method'] == 'pounders': + f.destroy() + # elif gen_specs['localopt_method'] == 'blmvm': + # g.destroy() + + lb.destroy() + ub.destroy() + x.destroy() + tao.destroy() + + return x_opt, exit_code + + +def decide_where_to_start_localopt(H, n_s, rk_const, mu=0, nu=0): + """ + Finds points in the history that satisfy the conditions (S1-S5 and L1-L8) in + Table 1 of the `APOSMM paper `_ + This method first identifies sample points satisfying S2-S5, and then + identifies all localopt points that satisfy L1-L7. + We then start from any sample point also satisfying S1. + We do not check condition L8 currently. + + We don't consider points in the history that have not returned from + computation, or that have a ``nan`` value. Also, note that ``mu`` and ``nu`` + implicitly depend on the scaling that is happening with the domain. That + is, adjusting the initial domain can make a run start (or not start) at + a point that didn't (or did) previously. + + Parameters + ---------- + H: numpy structured array + History array storing rows for each point. + r_k_const: float + Radius for deciding when to start runs + lhs_divisions: integer + Number of Latin hypercube sampling divisions (0 or 1 means uniform + random sampling over the domain) + mu: nonnegative float + Distance from the boundary that all starting points must satisfy + nu: nonnegative float + Distance from identified minima that all starting points must satisfy + gamma_quantile: float in (0,1] + Only sample points whose function values are in the lower + gamma_quantile can start localopt runs + + Returns + ---------- + start_inds: list + Indices where a local opt run should be started + + + :See: + ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + """ + + r_k = calc_rk(n, n_s, rk_const) + + if nu > 0: + test_2_through_5 = np.logical_and.reduce(( + H['returned'] == 1, # have a returned function value + H['dist_to_better_s'] > + r_k, # no better sample point within r_k (L2) + ~H['started_run'], # have not started a run (L3) + H['dist_to_unit_bounds'] >= + mu, # have all components at least mu away from bounds (L4) + np.all( + cdist(H['x_on_cube'], H['x_on_cube'][H['local_min']]) >= nu, + axis=1) # distance nu away from known local mins (L5) + )) + else: + test_2_through_5 = np.logical_and.reduce(( + H['returned'] == 1, # have a returned function value + H['dist_to_better_s'] > + r_k, # no better sample point within r_k (L2) + ~H['started_run'], # have not started a run (L3) + H['dist_to_unit_bounds'] >= + mu, # have all components at least mu away from bounds (L4) + )) # (L5) is always true when nu = 0 + + assert gamma_quantile == 1, "This is not supported yet. What is the best way to decide this when there are NaNs present in H['f']?" + # if gamma_quantile < 1: + # cut_off_value = np.sort(H['f'][~H['local_pt']])[np.floor(gamma_quantile*(sum(~H['local_pt'])-1)).astype(int)] + # else: + # cut_off_value = np.inf + + # Find the indices of points that... + sample_seeds = np.logical_and.reduce(( + ~H['local_pt'], # are not localopt points + # H['f'] <= cut_off_value, # have a small enough objective value + ~np.isinf(H['f']), # have a non-infinity objective value + ~np.isnan(H['f']), # have a non-NaN objective value + test_2_through_5, # satisfy tests 2 through 5 + )) + + # Uncomment the following to test the effect of ignorning LocalOpt points + # in APOSMM. This allows us to test a parallel MLSL. + # return list(np.ix_(sample_seeds)[0]) + + those_satisfying_S1 = H['dist_to_better_l'][sample_seeds] > r_k # no better localopt point within r_k + sample_start_inds = np.ix_(sample_seeds)[0][those_satisfying_S1] + + # Find the indices of points that... + local_seeds = np.logical_and.reduce(( + H['local_pt'], # are localopt points + H['dist_to_better_l'] > r_k, # no better local point within r_k (L1) + ~np.isinf(H['f']), # have a non-infinity objective value + ~np.isnan(H['f']), # have a non-NaN objective value + test_2_through_5, + H['num_active_runs'] == 0, # are not in an active run (L6) + ~H['local_min'] # are not a local min (L7) + )) + + local_start_inds2 = list(np.ix_(local_seeds)[0]) + + # If paused is a field in H, don't start from paused points. + if 'paused' in H.dtype.names: + sample_start_inds = sample_start_inds[~H[sample_start_inds]['paused']] + start_inds = list(sample_start_inds)+local_start_inds2 + else: + start_inds = list(sample_start_inds)+local_start_inds2 + + return start_inds + + +def look_in_history(x, Run_H, vector_return=False): + """ See if Run['x_on_cube'][advance_local_run.pt_in_run] matches x, + returning f or fvec, or saves x to advance_local_run.x_new if every point in Run_H has been + checked. + """ + + if vector_return: + to_return = 'fvec' + else: + if 'grad' in Run_H.dtype.names: + to_return = ['f', 'grad'] + else: + to_return = 'f' + + if advance_local_run.pt_in_run < len(Run_H): + # Return the value in history to the localopt algorithm. + assert np.allclose(x, Run_H['x_on_cube'][advance_local_run.pt_in_run], rtol=1e-08, atol=1e-08), \ + "History point does not match Localopt point" + f_out = Run_H[to_return][advance_local_run.pt_in_run] + else: + if advance_local_run.pt_in_run == len(Run_H): + # The history of points is exhausted. Save the requested point x to + # x_new. x_new will be returned to the manager. + advance_local_run.x_new[:] = x + + # Just in case the local opt method requests more points after a new + # point has been identified. + f_out = Run_H[to_return][-1] + + advance_local_run.pt_in_run += 1 + + return f_out + + +def calc_rk(n, n_s, rk_const, lhs_divisions=0): + """ Calculate the critical distance r_k """ + + if lhs_divisions == 0: + r_k = rk_const*(log(n_s)/n_s)**(1/n) + else: + k = np.floor(n_s/lhs_divisions).astype(int) + if k <= 1: # to prevent r_k=0 + r_k = np.inf + else: + r_k = rk_const*(log(k)/k)**(1/n) + + return r_k + +def count_samples(H,gen_specs): + if 'single_component_at_a_time' in gen_specs and gen_specs['single_component_at_a_time']: + # Get the pt_id for non-nan, returned points + pt_ids = H['pt_id'][np.logical_and(H['returned'], ~np.isnan(H['f_i']))] + _, counts = np.unique(pt_ids, return_counts=True) + n_s = np.sum(counts == gen_specs['components']) + + else: + # Number of returned sampled points (excluding nans) + n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['returned']))) + + return n_s + +def initialize_APOSMM(H, gen_specs, libE_info): + """ + Computes common values every time that APOSMM is reinvoked + + :See: + ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + """ + n = len(gen_specs['ub']) + + n_s = 0 + + if 'single_component_at_a_time' in gen_specs and gen_specs['single_component_at_a_time']: + assert gen_specs['batch_mode'], ("Must be in batch mode when using " + "'single_component_at_a_time'") + c_flag = True + else: + c_flag = False + + if 'rk_const' in gen_specs: + rk_c = gen_specs['rk_const'] + else: + rk_c = ((gamma(1+(n/2.0))*5.0)**(1.0/n))/sqrt(pi) + + if 'lhs_divisions' in gen_specs: + ld = gen_specs['lhs_divisions'] + else: + ld = 0 + + if 'mu' in gen_specs: + mu = gen_specs['mu'] + else: + mu = 1e-4 + + if 'nu' in gen_specs: + nu = gen_specs['nu'] + else: + nu = 0 + + total_runs = 0 + + + comm = libE_info['comm'] + + + return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm + + +def display_exception(e): + print(e.__doc__) + print(e.args) + _, _, tb = sys.exc_info() + traceback.print_tb(tb) # Fixed format + tb_info = traceback.extract_tb(tb) + filename, line, func, text = tb_info[-1] + print('An error occurred on line {} of function {} with statement {}'.format(line, func, text)) + + # PETSc/TAO errors are printed in the following manner: + if hasattr(e, '_traceback_'): + print('The error was:') + for i in e._traceback_: + print(i) + sys.stdout.flush() + + +# if __name__ == "__main__": +# [H,gen_specs,persis_info] = [np.load('H20.npz')[i] for i in ['H','gen_specs','persis_info']] +# gen_specs = gen_specs.item() +# persis_info = persis_info.item() +# import ipdb; ipdb.set_trace() +# aposmm_logic(H,persis_info,gen_specs,{}) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py new file mode 100644 index 000000000..e6b7f433a --- /dev/null +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -0,0 +1,55 @@ +# """ +# Runs libEnsemble on the 6-hump camel problem. Documented here: +# https://www.sfu.ca/~ssurjano/camel6.html +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_6-hump_camel_persistent_uniform_sampling.py +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms local +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_NPROCS: 3 4 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f +from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream + +nworkers, is_master, libE_specs, _ = parse_args() + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('grad', float, n)]} + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'gen_batch_size': 20, + 'out': [('x', float, (n,))], + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + +alloc_specs = {'alloc_f': alloc_f, 'out': []} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'sim_max': 40} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_master: + save_libE_output(H, persis_info, __file__, nworkers) From 56be7c6d68ab50b3d0a7396d0e70d5b1ea88e7e4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 5 Jul 2019 11:14:09 -0500 Subject: [PATCH 006/644] Example of persistent APOSMM ready for Kaushik --- libensemble/gen_funcs/persistent_aposmm.py | 160 ++++++++++-------- .../test_6-hump_camel_aposmm_LD_MMA.py | 1 - .../test_6-hump_camel_persistent_aposmm.py | 8 +- 3 files changed, 96 insertions(+), 73 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index fe747da27..d0f6592ef 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -23,6 +23,10 @@ import nlopt +from libensemble.message_numbers import STOP_TAG, PERSIS_STOP +from libensemble.gen_funcs.support import send_mgr_worker_msg +from libensemble.gen_funcs.support import get_mgr_worker_msg + class APOSMMException(Exception): "Raised for any exception in APOSMM" @@ -179,119 +183,108 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - import ipdb; ipdb.set_trace() - n, n_s, c_flag, rk_const, mu, nu, comm = initialize_APOSMM(H, gen_specs, libE_info) + n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + + send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H) tag = None + while 1: + # Receive from manager + tag, Work, calc_in = get_mgr_worker_msg(comm) - while tag not in [STOP_TAG, PERSIS_STOP]: - n_s = count_samples(H,gen_specs) + if tag in [STOP_TAG, PERSIS_STOP]: + continue - if n_s >= gen_specs['initial_sample_size']: - update_history_dist(H, n, gen_specs, c_flag) + n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - starting_inds = decide_where_to_start_localopt(H, n_s, rk_const, mu, nu) + # If manager returned function values for an existing local opt run, + # contact child process to get another x_new and send back to manager + import ipdb; ipdb.set_trace() + # Decide if there are any new points to start a localopt run + starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) + + if len(starting_inds): + O = np.empty(0, dtype=gen_specs['out']) for ind in starting_inds: + # Start localopt child processes + # Mark point as having started a run new_run_num = total_runs total_runs += 1 - H['started_run'][ind] = 1 + local_H['started_run'][ind] = 1 run_order[new_run_num] = [ind] # Initialize child process for new localopt run - x_new = start_localopt(H, gen_specs, c_flag) + x_new = start_localopt(local_H, gen_specs, c_flag) - # Send x_new to manager for evaluation - O = np.zeros(1, dtype=gen_specs['out']) - add_to_O(O, x_new, H, gen_specs, c_flag, persis_info, local_flag=1) - tag, Work, calc_in = send_mgr_worker_msg(comm, O) + # Add x_new to O (which will be sent to the manager for evaluation) + add_to_O(O, x_new, local_H, gen_specs, c_flag, persis_info, local_flag=1) - # Receive from manager (if possible), contact child process to get - # x_new, and send back to manager - get_mgr_worker_msg(comm, status) + send_mgr_worker_msg(comm, O) - else: - O = np.zeros(gen_specs['initial_sample_size'], dtype=gen_specs['out']) - sampled_points = persis_info['rand_stream'].uniform(0, 1, (len(O), n)) - on_cube = True - add_to_O(O, sampled_points, H, gen_specs, c_flag, persis_info, on_cube=on_cube) + return O, persis_info, tag - tag, Work, calc_in = send_mgr_worker_msg(comm, O) +def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): - return O, persis_info, tag + for name in calc_in.dtype.names: + local_H[name][Work['libE_info']['H_rows']] = calc_in[name] + + local_H['returned'][Work['libE_info']['H_rows']] = True + n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) + update_history_dist(local_H, n, gen_specs, c_flag) -def add_to_O(O, pts, H, gen_specs, c_flag, persis_info, local_flag=0, - sorted_run_inds=[], run=[], on_cube=True): + return n_s + + +def add_to_local_H(local_H, pts, gen_specs, c_flag, local_flag=0, sorted_run_inds=[], run=[], on_cube=True): """ Adds points to O, the numpy structured array to be sent back to the manager """ assert not local_flag or len(pts) == 1, "Can't > 1 local points" - original_len_O = len(O) + len_local_H = len(local_H) - len_H = len(H) ub = gen_specs['ub'] lb = gen_specs['lb'] if c_flag: m = gen_specs['components'] - assert len_H % m == 0, "Number of points in len_H not congruent to 0 mod 'components'" - pt_ids = np.sort(np.tile(np.arange((len_H+original_len_O)/m, (len_H+original_len_O)/m+len(pts)), (1, m))) + assert len_local_H % m == 0, "Number of points in local_H not congruent to 0 mod 'components'" + pt_ids = np.sort(np.tile(np.arange((len_local_H)/m, (len_local_H)/m+len(pts)), (1, m))) pts = np.tile(pts, (m, 1)) num_pts = len(pts) - O.resize(len(O)+num_pts, refcheck=False) # Adds num_pts rows of zeros to O + local_H.resize(len(local_H)+num_pts, refcheck=False) # Adds num_pts rows of zeros to O if on_cube: - O['x_on_cube'][-num_pts:] = pts - O['x'][-num_pts:] = pts*(ub-lb)+lb + local_H['x_on_cube'][-num_pts:] = pts + local_H['x'][-num_pts:] = pts*(ub-lb)+lb else: - O['x_on_cube'][-num_pts:] = (pts-lb)/(ub-lb) - O['x'][-num_pts:] = pts + local_H['x_on_cube'][-num_pts:] = (pts-lb)/(ub-lb) + local_H['x'][-num_pts:] = pts - O['sim_id'][-num_pts:] = np.arange(len_H+original_len_O, len_H+original_len_O+num_pts) - O['local_pt'][-num_pts:] = local_flag + local_H['sim_id'][-num_pts:] = np.arange(len_local_H, len_local_H+num_pts) + local_H['local_pt'][-num_pts:] = local_flag - O['dist_to_unit_bounds'][-num_pts:] = np.inf - O['dist_to_better_l'][-num_pts:] = np.inf - O['dist_to_better_s'][-num_pts:] = np.inf - O['ind_of_better_l'][-num_pts:] = -1 - O['ind_of_better_s'][-num_pts:] = -1 + local_H['dist_to_unit_bounds'][-num_pts:] = np.inf + local_H['dist_to_better_l'][-num_pts:] = np.inf + local_H['dist_to_better_s'][-num_pts:] = np.inf + local_H['ind_of_better_l'][-num_pts:] = -1 + local_H['ind_of_better_s'][-num_pts:] = -1 if c_flag: - O['obj_component'][-num_pts:] = np.tile(range(0, m), (1, num_pts//m)) - O['pt_id'][-num_pts:] = pt_ids + local_H['obj_component'][-num_pts:] = np.tile(range(0, m), (1, num_pts//m)) + local_H['pt_id'][-num_pts:] = pt_ids if local_flag: - O['num_active_runs'][-num_pts] += 1 - # O['priority'][-num_pts:] = 1 - # O['priority'][-num_pts:] = np.random.uniform(0,1,num_pts) - if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: - O['priority'][-num_pts:] = -min(H['f'][persis_info['run_order'][run]]) # Give highest priority to run with lowest function value - else: - O['priority'][-num_pts:] = persis_info['rand_stream'].uniform(0, 1, num_pts) - persis_info['run_order'][run].append(O[-num_pts]['sim_id']) + local_H['num_active_runs'][-num_pts] += 1 else: - if c_flag: - # p_tmp = np.sort(np.tile(np.random.uniform(0,1,num_pts/m),(m,1))) # If you want all "duplicate points" to have the same priority (meaning libEnsemble gives them all at once) - # p_tmp = np.random.uniform(0,1,num_pts) - p_tmp = persis_info['rand_stream'].uniform(0, 1, num_pts) - else: - # p_tmp = np.random.uniform(0,1,num_pts) - # persis_info['rand_stream'].uniform(lb,ub,(1,n)) - if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: - p_tmp = -np.inf*np.ones(num_pts) - else: - p_tmp = persis_info['rand_stream'].uniform(0, 1, num_pts) - O['priority'][-num_pts:] = p_tmp - # O['priority'][-num_pts:] = 1 - - return persis_info + local_H['priority'][-num_pts:] = 1 def update_history_dist(H, n, gen_specs, c_flag): @@ -646,7 +639,7 @@ def pounders_obj_func(tao, X, F, Run_H): return x_opt, exit_code -def decide_where_to_start_localopt(H, n_s, rk_const, mu=0, nu=0): +def decide_where_to_start_localopt(H, n, n_s, rk_const, mu=0, nu=0): """ Finds points in the history that satisfy the conditions (S1-S5 and L1-L8) in Table 1 of the `APOSMM paper `_ @@ -712,7 +705,7 @@ def decide_where_to_start_localopt(H, n_s, rk_const, mu=0, nu=0): mu, # have all components at least mu away from bounds (L4) )) # (L5) is always true when nu = 0 - assert gamma_quantile == 1, "This is not supported yet. What is the best way to decide this when there are NaNs present in H['f']?" + # assert gamma_quantile == 1, "This is not supported yet. What is the best way to decide this when there are NaNs present in H['f']?" # if gamma_quantile < 1: # cut_off_value = np.sort(H['f'][~H['local_pt']])[np.floor(gamma_quantile*(sum(~H['local_pt'])-1)).astype(int)] # else: @@ -861,8 +854,35 @@ def initialize_APOSMM(H, gen_specs, libE_info): comm = libE_info['comm'] - - return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm + local_H_fields = [('f',float), + ('grad',float,n), + ('x', float, n), + ('x_on_cube', float, n), + ('priority', float), + ('local_pt', bool), + ('known_to_aposmm', bool), + ('dist_to_unit_bounds', float), + ('dist_to_better_l', float), + ('dist_to_better_s', float), + ('ind_of_better_l', int), + ('ind_of_better_s', int), + ('started_run', bool), + ('num_active_runs', int), + ('local_min', bool), + ('sim_id', int), + ('paused', bool), + ('returned', bool), + ('pt_id', int), # Identify the same point evaluated by different sim_f's or components + ] + + local_H = np.empty(0,dtype=local_H_fields); + + return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm, local_H + +def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H): + sampled_points = persis_info['rand_stream'].uniform(0, 1, (gen_specs['initial_sample_size'], n)) + add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) + send_mgr_worker_msg(comm, local_H[['x','sim_id']]) def display_exception(e): diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 829b48fbf..ddae2ce7f 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -36,7 +36,6 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f', 'grad', 'returned'], 'out': gen_out, - 'num_active_gens': 1, 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index e6b7f433a..307dcadfa 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -34,10 +34,14 @@ 'in': ['x'], 'out': [('f', float), ('grad', float, n)]} +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int)] gen_specs = {'gen_f': gen_f, 'in': [], - 'gen_batch_size': 20, - 'out': [('x', float, (n,))], + 'out': gen_out, + 'batch_mode': True, + 'initial_sample_size': 5, + 'localopt_method': 'LD_MMA', + 'xtol_rel': 1e-3, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} From 5627ce3302b2427c644ad53ce59888f25b443e79 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 5 Jul 2019 13:39:06 -0500 Subject: [PATCH 007/644] minor changes to persis aposmm --- libensemble/gen_funcs/persistent_aposmm.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index d0f6592ef..2b11398b0 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -194,18 +194,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if tag in [STOP_TAG, PERSIS_STOP]: - continue + break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - # If manager returned function values for an existing local opt run, - # contact child process to get another x_new and send back to manager - import ipdb; ipdb.set_trace() - # Decide if there are any new points to start a localopt run starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) - if len(starting_inds): + if len(starting_inds): O = np.empty(0, dtype=gen_specs['out']) for ind in starting_inds: # Start localopt child processes @@ -217,6 +213,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): run_order[new_run_num] = [ind] # Initialize child process for new localopt run + #FIXME: Instead of rading the first value, poll for values from + # child processes later. x_new = start_localopt(local_H, gen_specs, c_flag) # Add x_new to O (which will be sent to the manager for evaluation) @@ -224,8 +222,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_mgr_worker_msg(comm, O) + raise NotImplementedError("Persistent APOSMM is not fully supported, yet.") + return O, persis_info, tag + def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): for name in calc_in.dtype.names: From e5fce96a0f945734975a31375b4e5c8d913baa33 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 10 Jul 2019 14:54:42 -0500 Subject: [PATCH 008/644] chalks out the main plan for the implementation --- libensemble/gen_funcs/persistent_aposmm.py | 107 +++++++++++++++--- .../test_6-hump_camel_persistent_aposmm.py | 2 +- 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 2b11398b0..72e7d22d7 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -28,6 +28,9 @@ from libensemble.gen_funcs.support import get_mgr_worker_msg +from multiprocessing import Event, Process, Queue + + class APOSMMException(Exception): "Raised for any exception in APOSMM" @@ -184,57 +187,133 @@ def aposmm(H, persis_info, gen_specs, libE_info): """ + # {{{ multiprocessing initialization + + child_can_read_evts = [] + processes = [] + parent_can_read_from_queue = Event() + comm_queue = Queue() + + # }}} + n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + if gen_specs['initial_sample_size'] != 1: + raise RuntimeError("sample size > not supported for now.") + send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H) + # Separate the logic of filling local_H and sending it. + # Reason: To a new developer this doesn't look like local_H will get + # edited. + + print('local_H =', local_H) + tag = None + O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) + # KK: tag: what tag is it?(need helpful naming) + # KK: 'tag' is present in 'Work' as well.(repitition accounts for + # more confusion and increases scope for human error) + # Aah ok, looking further tag seems one of the enums in libEnsemble. + # Anyways just the name 'tag' is not desciptive enough, something more + # descriptive would be better. + + # persis_info contains something called H_rows + + # print('Tag =', tag) + # print('Work =', Work) + print('calc_in =', calc_in) + if tag in [STOP_TAG, PERSIS_STOP]: + # FIXME: We need to kill all the child processes here. + raise NotImplementedError("Killing of child processes in not" + " implmeneted by the devs.") break + # Look at the message here. Is it a message for a point from which + # local optimization has to be started or does it have to be supplied + # to a currently running local optimization run. + # local_H contains a ton of information. + n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - # Decide if there are any new points to start a localopt run - starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) - if len(starting_inds): - O = np.empty(0, dtype=gen_specs['out']) + if current_eval_received_for_a_run(tag, Work, calc_in): + + child_idx = get_child_idx_for_the_received_evaluation(calc_in) + + comm_queue.push(f_x) + + parent_can_read_from_queue.unset() + + child_can_read_evts[child_idx].set() + parent_can_read_from_queue.wait() + + x_new = comm_queue.get() + + add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + + #FIXME: Be very careful about the sim_id. + # Once we have set the sim_id correctly our job + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + else: + n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) + + # Decide if there are any new points to start a localopt run + starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) + for ind in starting_inds: # Start localopt child processes # Mark point as having started a run - new_run_num = total_runs + # Why does it help to mark the total number of runs? total_runs += 1 local_H['started_run'][ind] = 1 - run_order[new_run_num] = [ind] - # Initialize child process for new localopt run - #FIXME: Instead of rading the first value, poll for values from - # child processes later. - x_new = start_localopt(local_H, gen_specs, c_flag) + child_can_read_evts.append(Event()) + + # FIXME: f_x must be read from local_H. + + p = Process(run_local_opt, args=(comm_queue, x_start, + child_can_read, parent_can_read)) + processes.append(p) + p.start() + + comm_queue.push(f_x) + parent_can_read_from_queue.unset() - # Add x_new to O (which will be sent to the manager for evaluation) - add_to_O(O, x_new, local_H, gen_specs, c_flag, persis_info, local_flag=1) + child_can_read_evts[-1].set() + parent_can_read_from_queue.wait() - send_mgr_worker_msg(comm, O) + x_new = comm_queue.get() + + add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + + #FIXME: Be very careful about the sim_id. + # That is all what we have. + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) raise NotImplementedError("Persistent APOSMM is not fully supported, yet.") + for p in processes: + p.join() + return O, persis_info, tag def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): - for name in calc_in.dtype.names: + for name in calc_in.dtype.names: local_H[name][Work['libE_info']['H_rows']] = calc_in[name] local_H['returned'][Work['libE_info']['H_rows']] = True n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) + # dist -> distance update_history_dist(local_H, n, gen_specs, c_flag) return n_s diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 307dcadfa..6dea74657 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -39,7 +39,7 @@ 'in': [], 'out': gen_out, 'batch_mode': True, - 'initial_sample_size': 5, + 'initial_sample_size': 1, 'localopt_method': 'LD_MMA', 'xtol_rel': 1e-3, 'lb': np.array([-3, -2]), From 34fcf063eeffa5547caab59bc8987ada3e8e711c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 10 Jul 2019 15:45:58 -0500 Subject: [PATCH 009/644] Answering one question. --- libensemble/gen_funcs/persistent_aposmm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 72e7d22d7..8252bbd4c 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -270,7 +270,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Start localopt child processes # Mark point as having started a run - # Why does it help to mark the total number of runs? + # Why does it help to mark the total number of runs? JL: it's + # useful to assign a unique ID to each run that is started. + # Such information is one way (but not the only way) to give + # funciton values to the correct child process. total_runs += 1 local_H['started_run'][ind] = 1 From a61d9de4c65a48910da102687fe1e3880ab1aa22 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 11 Jul 2019 13:55:53 -0500 Subject: [PATCH 010/644] Giving back sim_id to persistent gens --- libensemble/alloc_funcs/start_only_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 2d2d93fdc..a126f796d 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -25,7 +25,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): last_time_gen_gave_batch = np.max(H['gen_time'][gen_inds]) inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']], + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) task_avail = ~H['given'] From 517acb6b2da6fdd3b00b7bda4e2db831632acef3 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 11 Jul 2019 13:56:52 -0500 Subject: [PATCH 011/644] WIP: some more changes for persis aposmm --- libensemble/gen_funcs/persistent_aposmm.py | 40 +++++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 72e7d22d7..8c51d775a 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -207,8 +207,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Reason: To a new developer this doesn't look like local_H will get # edited. - print('local_H =', local_H) - tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: @@ -224,9 +222,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): # persis_info contains something called H_rows + # calc_in contains the tuple (x, f, grad_f). + # So one can theoretically read the local_H and do some operations to + # figure out the origin of this 'x'. + # print('Tag =', tag) # print('Work =', Work) - print('calc_in =', calc_in) + # print('calc_in =', calc_in) if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: We need to kill all the child processes here. @@ -241,6 +243,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) + print(local_H) + 1/0 + if current_eval_received_for_a_run(tag, Work, calc_in): @@ -278,12 +283,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): # FIXME: f_x must be read from local_H. - p = Process(run_local_opt, args=(comm_queue, x_start, + p = Process(run_local_opt, args=(local_opt_type, comm_queue, x_start, child_can_read, parent_can_read)) processes.append(p) p.start() - comm_queue.push(f_x) + comm_queue.push((f_x, grad_f_x)) parent_can_read_from_queue.unset() child_can_read_evts[-1].set() @@ -305,6 +310,30 @@ def aposmm(H, persis_info, gen_specs, libE_info): return O, persis_info, tag +def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): + comm_queue.put(x) + parent_can_read.set() + child_can_read.wait() + result = comm_queue.get() + child_can_read.unset() + if gen_specs['localopt_method'] in ['LD_MMA']: + grad[:] = result[1] + + return result[0] + + +def run_local_opt(gen_specs, comm_queue, x0, child_can_read, + parent_can_read): + + opt = nlopt.opt(nlopt.LN_BOBYQA, 2) + opt.set_min_objective(lambda x, grad: callback_function(x, grad, + comm_queue, child_can_read, parent_can_read, gen_specs)) + + opt.set_lower_bounds(-np.array([3, 2])) + opt.set_upper_bounds(np.array([3, 2])) + opt.set_ftol_rel(1e-4) + + def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): for name in calc_in.dtype.names: @@ -962,6 +991,7 @@ def initialize_APOSMM(H, gen_specs, libE_info): def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H): sampled_points = persis_info['rand_stream'].uniform(0, 1, (gen_specs['initial_sample_size'], n)) add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) + print('Sent =', local_H[['x','sim_id']]) send_mgr_worker_msg(comm, local_H[['x','sim_id']]) From 618007a604603bc5a12e07fb5968d5c952cc47ac Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 11 Jul 2019 15:20:07 -0500 Subject: [PATCH 012/644] reads sim_id into gen_func --- libensemble/gen_funcs/persistent_aposmm.py | 29 +++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 77b5a7bd6..51facf8dd 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -198,6 +198,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + sim_id_to_child_indices = {} + if gen_specs['initial_sample_size'] != 1: raise RuntimeError("sample size > not supported for now.") @@ -212,6 +214,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): while 1: # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) + x_recv, f_x_recv, grad_f_recv, sim_id_recv = calc_in[0] + print(calc_in) + 1/0 # KK: tag: what tag is it?(need helpful naming) # KK: 'tag' is present in 'Work' as well.(repitition accounts for @@ -246,25 +251,19 @@ def aposmm(H, persis_info, gen_specs, libE_info): print(local_H) 1/0 + if sim_id_to_child_indices.get(sim_id): + for child_idx in sim_id_to_child_indices[sim_id_recv]: + comm_queue.push((f_x_recv, grad_f_recv)) - if current_eval_received_for_a_run(tag, Work, calc_in): - - child_idx = get_child_idx_for_the_received_evaluation(calc_in) - - comm_queue.push(f_x) - - parent_can_read_from_queue.unset() - - child_can_read_evts[child_idx].set() - parent_can_read_from_queue.wait() + parent_can_read_from_queue.unset() - x_new = comm_queue.get() + child_can_read_evts[child_idx].set() + parent_can_read_from_queue.wait() - add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + x_new = comm_queue.get() - #FIXME: Be very careful about the sim_id. - # Once we have set the sim_id correctly our job - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) From d4a2932a666f7aa46443b48cbe2fac70154a3060 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 12 Jul 2019 15:10:17 -0500 Subject: [PATCH 013/644] saving s'more progress --- libensemble/gen_funcs/persistent_aposmm.py | 37 ++++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 51facf8dd..be24f7da0 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -203,7 +203,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): if gen_specs['initial_sample_size'] != 1: raise RuntimeError("sample size > not supported for now.") - send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H) + send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) # Separate the logic of filling local_H and sending it. # Reason: To a new developer this doesn't look like local_H will get @@ -214,7 +215,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): while 1: # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) - x_recv, f_x_recv, grad_f_recv, sim_id_recv = calc_in[0] + x_recv, f_x_recv, grad_f_x_recv, sim_id_recv = calc_in[0] print(calc_in) 1/0 @@ -251,9 +252,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): print(local_H) 1/0 - if sim_id_to_child_indices.get(sim_id): + if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: - comm_queue.push((f_x_recv, grad_f_recv)) + comm_queue.push((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.unset() @@ -263,7 +264,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = comm_queue.get() add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + # TODO:[KK]: Would it be true that the local_H[-1] would be our + # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) + else: + sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) else: n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -285,12 +292,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): # FIXME: f_x must be read from local_H. - p = Process(run_local_opt, args=(local_opt_type, comm_queue, x_start, - child_can_read, parent_can_read)) + p = Process(run_local_opt, args=(gen_specs, comm_queue, x_recv, + child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) p.start() - comm_queue.push((f_x, grad_f_x)) + comm_queue.push((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.unset() child_can_read_evts[-1].set() @@ -300,6 +307,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes), ) + else: + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes), ) + #FIXME: Be very careful about the sim_id. # That is all what we have. send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) @@ -990,11 +1002,16 @@ def initialize_APOSMM(H, gen_specs, libE_info): return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm, local_H -def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H): +def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices): sampled_points = persis_info['rand_stream'].uniform(0, 1, (gen_specs['initial_sample_size'], n)) add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) - print('Sent =', local_H[['x','sim_id']]) - send_mgr_worker_msg(comm, local_H[['x','sim_id']]) + for sim_id in local_H['sim_id'][-gen_specs['initial_sample_size']:]: + assert sim_id not in sim_id_to_child_indices + sim_id_to_child_indices[sim_id] = None + + print('Sent =', local_H[['x', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[['x', 'sim_id']]) def display_exception(e): From e8da8e3b563fdbcd4e21039b518d82e9d564777c Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 12 Jul 2019 15:31:14 -0500 Subject: [PATCH 014/644] some cleanup --- libensemble/gen_funcs/persistent_aposmm.py | 37 ---------------------- 1 file changed, 37 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index be24f7da0..463e995ac 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -216,25 +216,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) x_recv, f_x_recv, grad_f_x_recv, sim_id_recv = calc_in[0] - print(calc_in) - 1/0 - - # KK: tag: what tag is it?(need helpful naming) - # KK: 'tag' is present in 'Work' as well.(repitition accounts for - # more confusion and increases scope for human error) - # Aah ok, looking further tag seems one of the enums in libEnsemble. - # Anyways just the name 'tag' is not desciptive enough, something more - # descriptive would be better. - - # persis_info contains something called H_rows - - # calc_in contains the tuple (x, f, grad_f). - # So one can theoretically read the local_H and do some operations to - # figure out the origin of this 'x'. - - # print('Tag =', tag) - # print('Work =', Work) - # print('calc_in =', calc_in) if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: We need to kill all the child processes here. @@ -242,16 +223,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): " implmeneted by the devs.") break - # Look at the message here. Is it a message for a point from which - # local optimization has to be started or does it have to be supplied - # to a currently running local optimization run. - # local_H contains a ton of information. - n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - print(local_H) - 1/0 - if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: comm_queue.push((f_x_recv, grad_f_x_recv)) @@ -274,17 +247,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - # Decide if there are any new points to start a localopt run starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) for ind in starting_inds: - # Start localopt child processes - - # Mark point as having started a run - # Why does it help to mark the total number of runs? JL: it's - # useful to assign a unique ID to each run that is started. - # Such information is one way (but not the only way) to give - # funciton values to the correct child process. total_runs += 1 local_H['started_run'][ind] = 1 @@ -312,8 +277,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes), ) - #FIXME: Be very careful about the sim_id. - # That is all what we have. send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) raise NotImplementedError("Persistent APOSMM is not fully supported, yet.") From 4e2cea592f5e9198fde27e8046741e8ef3599d8a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 15 Jul 2019 13:23:13 -0500 Subject: [PATCH 015/644] logic fix for the starting the process --- libensemble/gen_funcs/persistent_aposmm.py | 44 ++++++++++++++-------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 463e995ac..096413104 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -206,21 +206,22 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) - # Separate the logic of filling local_H and sending it. - # Reason: To a new developer this doesn't look like local_H will get - # edited. - tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) - x_recv, f_x_recv, grad_f_x_recv, sim_id_recv = calc_in[0] + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in[0] if tag in [STOP_TAG, PERSIS_STOP]: - # FIXME: We need to kill all the child processes here. - raise NotImplementedError("Killing of child processes in not" - " implmeneted by the devs.") + # FIXME: If there is any indication that all child processes are + # done with their work, we can do this a bit more cleanly i.e. + # first join and check if they are still alive, only then kill. + + # sending SIGKILL to the processes. + # TODO: should we use SIGTERM instead? + for p in processes: + p.kill() break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -237,7 +238,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = comm_queue.get() add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) - # TODO:[KK]: Would it be true that the local_H[-1] would be our + # FIXME:[KK]: Would it be true that the local_H[-1] would be our # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) if local_H[-1]['sim_id'] in sim_id_to_child_indices: @@ -254,15 +255,18 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 child_can_read_evts.append(Event()) + child_can_read_evts[-1].unset() - # FIXME: f_x must be read from local_H. - - p = Process(run_local_opt, args=(gen_specs, comm_queue, x_recv, + parent_can_read_from_queue.unset() + p = Process(run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x'], child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) p.start() - comm_queue.push((f_x_recv, grad_f_x_recv)) + parent_can_read_from_queue.wait() + assert comm_queue.get() == local_H[ind]['x'] + + comm_queue.push(local_H[ind][['f', 'grad']]) parent_can_read_from_queue.unset() child_can_read_evts[-1].set() @@ -272,17 +276,25 @@ def aposmm(H, persis_info, gen_specs, libE_info): add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + # FIXME: again makes the assumption that the the newly added + # point comes in local_H[-1] + if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes), ) + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes), ) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + comm_queue.close() + comm_queue.join_thread() + raise NotImplementedError("Persistent APOSMM is not fully supported, yet.") for p in processes: - p.join() + if p.is_alive(): + raise RuntimeError("Atleast one child process is still active, even after" + " killing all the children.") return O, persis_info, tag From a97c96411bd6b88b5dd6fd41a5a6601c82ab26bf Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 15 Jul 2019 13:56:44 -0500 Subject: [PATCH 016/644] fixes multiprocessing syntax errors --- libensemble/gen_funcs/persistent_aposmm.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 096413104..f15792f59 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -211,7 +211,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): while 1: # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in[0] + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: If there is any indication that all child processes are @@ -230,7 +230,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): for child_idx in sim_id_to_child_indices[sim_id_recv]: comm_queue.push((f_x_recv, grad_f_x_recv)) - parent_can_read_from_queue.unset() + parent_can_read_from_queue.clear() child_can_read_evts[child_idx].set() parent_can_read_from_queue.wait() @@ -255,11 +255,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 child_can_read_evts.append(Event()) - child_can_read_evts[-1].unset() - parent_can_read_from_queue.unset() - p = Process(run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x'], - child_can_read_evts[-1], parent_can_read_from_queue)) + parent_can_read_from_queue.clear() + p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x'], + child_can_read_evts[-1], + parent_can_read_from_queue)) processes.append(p) p.start() @@ -267,7 +267,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): assert comm_queue.get() == local_H[ind]['x'] comm_queue.push(local_H[ind][['f', 'grad']]) - parent_can_read_from_queue.unset() + parent_can_read_from_queue.clear() child_can_read_evts[-1].set() parent_can_read_from_queue.wait() @@ -289,8 +289,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue.close() comm_queue.join_thread() - raise NotImplementedError("Persistent APOSMM is not fully supported, yet.") - for p in processes: if p.is_alive(): raise RuntimeError("Atleast one child process is still active, even after" @@ -304,7 +302,7 @@ def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_ parent_can_read.set() child_can_read.wait() result = comm_queue.get() - child_can_read.unset() + child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: grad[:] = result[1] @@ -985,7 +983,6 @@ def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, assert sim_id not in sim_id_to_child_indices sim_id_to_child_indices[sim_id] = None - print('Sent =', local_H[['x', 'sim_id']]) send_mgr_worker_msg(comm, local_H[['x', 'sim_id']]) From acfce629e30787e3ec8c7f2d5e04b0fb879ec894 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 15 Jul 2019 16:44:50 -0500 Subject: [PATCH 017/644] takes nlopt from develop --- libensemble/gen_funcs/persistent_aposmm.py | 57 ++++++++++++++++++---- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index f15792f59..c06f521d8 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -209,6 +209,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: + print('Here now!') # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in @@ -228,7 +229,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: - comm_queue.push((f_x_recv, grad_f_x_recv)) + comm_queue.put((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.clear() @@ -256,21 +257,25 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_can_read_evts.append(Event()) + print('Initing process') + parent_can_read_from_queue.clear() - p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x'], + p = Process(target=run_local_nlopt, args=(gen_specs, comm_queue, local_H[ind]['x'], child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) p.start() + print('[Master]: Started a process') parent_can_read_from_queue.wait() - assert comm_queue.get() == local_H[ind]['x'] + assert np.allclose(comm_queue.get(), local_H[ind]['x']) - comm_queue.push(local_H[ind][['f', 'grad']]) + comm_queue.put(local_H[ind][['f', 'grad']]) parent_can_read_from_queue.clear() child_can_read_evts[-1].set() parent_can_read_from_queue.wait() + print('[Master]: Getting a new x_new.') x_new = comm_queue.get() @@ -298,27 +303,61 @@ def aposmm(H, persis_info, gen_specs, libE_info): def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): + print('[Child]: Hello, here I am ') comm_queue.put(x) parent_can_read.set() child_can_read.wait() result = comm_queue.get() child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: + print('Result =', result[1]) + print('Grad.shape =', grad.shape) + 1/0 grad[:] = result[1] return result[0] -def run_local_opt(gen_specs, comm_queue, x0, child_can_read, +def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, parent_can_read): + print('[Child]: Whats up') + + n = len(gen_specs['ub']) + + opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) + + lb = gen_specs['lb'] + ub = gen_specs['ub'] + opt.set_lower_bounds(lb) + opt.set_upper_bounds(ub) + + # Care must be taken here because a too-large initial step causes nlopt to move the starting point! + dist_to_bound = min(min(ub-x0), min(x0-lb)) + assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" + + if 'dist_to_bound_multiple' in gen_specs: + opt.set_initial_step(dist_to_bound*gen_specs['dist_to_bound_multiple']) + else: + opt.set_initial_step(dist_to_bound) + + # FIXME: Setting max evaluations = 100 + opt.set_maxeval(100) - opt = nlopt.opt(nlopt.LN_BOBYQA, 2) opt.set_min_objective(lambda x, grad: callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs)) - opt.set_lower_bounds(-np.array([3, 2])) - opt.set_upper_bounds(np.array([3, 2])) - opt.set_ftol_rel(1e-4) + if 'xtol_rel' in gen_specs: + opt.set_xtol_rel(gen_specs['xtol_rel']) + if 'ftol_rel' in gen_specs: + opt.set_ftol_rel(gen_specs['ftol_rel']) + if 'xtol_abs' in gen_specs: + opt.set_xtol_abs(gen_specs['xtol_abs']) + if 'ftol_abs' in gen_specs: + opt.set_ftol_abs(gen_specs['ftol_abs']) + + x_opt = opt.optimize(x0) + + #FIXME: Do we need to do of the final 'x_opt'? def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): From 75e1b29f18f5ba95be57fd2f3089455d19179977 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 16 Jul 2019 04:45:13 -0500 Subject: [PATCH 018/644] Noting a possible trip-up in processing calc_in --- libensemble/gen_funcs/persistent_aposmm.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c06f521d8..843fa0349 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -213,6 +213,16 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + # Kaushik, I have concerns about the above approach for processing calc_in + # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. + # 2. Note that grad might not be available for some objectives. + # Perhaps it's better to use something like + # x_recv = calc_in['x'] + # f_x_recv = calc_in['f'] + # sim_id_recv = calc_in['sim_id'] + # if 'grad' in calc_in.dtype.names: + # grad_f_x_recv = calc_in['grad'] + if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: If there is any indication that all child processes are From e07f7bcc67405c84ea54cb6672930f0b2c412787 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 16 Jul 2019 14:04:23 -0500 Subject: [PATCH 019/644] adds debug statements, maintains consistency of x_on_cube --- libensemble/gen_funcs/persistent_aposmm.py | 45 ++++++++++++---------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c06f521d8..4ed0afd54 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -209,12 +209,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: - print('Here now!') # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + if calc_in: + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in if tag in [STOP_TAG, PERSIS_STOP]: + print('[Parent]: Received a stop tag, killing all children.') # FIXME: If there is any indication that all child processes are # done with their work, we can do this a bit more cleanly i.e. # first join and check if they are still alive, only then kill. @@ -229,6 +230,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: + print('[Parent]: Received the eval for {}, giving it to the' + ' child_idx: {}.'.format(x_recv, child_idx)) comm_queue.put((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.clear() @@ -237,8 +240,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue.wait() x_new = comm_queue.get() + print('[Parent]: The child returned {}.'.format(x_new)) + + add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) + assert np.allclose(x_new, local_H[-1]['x_on_cube']) - add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) # FIXME:[KK]: Would it be true that the local_H[-1] would be our # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) @@ -260,15 +266,18 @@ def aposmm(H, persis_info, gen_specs, libE_info): print('Initing process') parent_can_read_from_queue.clear() - p = Process(target=run_local_nlopt, args=(gen_specs, comm_queue, local_H[ind]['x'], + p = Process(target=run_local_nlopt, args=(gen_specs, + comm_queue, local_H[ind]['x_on_cube'], child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) p.start() - print('[Master]: Started a process') + print('[Parent]: Started a child process') parent_can_read_from_queue.wait() - assert np.allclose(comm_queue.get(), local_H[ind]['x']) + print('[Parent]: Done waiting.') + + assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) comm_queue.put(local_H[ind][['f', 'grad']]) parent_can_read_from_queue.clear() @@ -279,7 +288,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = comm_queue.get() - add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) # FIXME: again makes the assumption that the the newly added # point comes in local_H[-1] @@ -294,25 +303,24 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue.close() comm_queue.join_thread() + print('Done with everything') + for p in processes: if p.is_alive(): - raise RuntimeError("Atleast one child process is still active, even after" + raise RuntimeError("[Parent]: Atleast one child process is still active, even after" " killing all the children.") return O, persis_info, tag def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): - print('[Child]: Hello, here I am ') comm_queue.put(x) + print('[Child]: Parent should no longer wait.') parent_can_read.set() child_can_read.wait() result = comm_queue.get() child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: - print('Result =', result[1]) - print('Grad.shape =', grad.shape) - 1/0 grad[:] = result[1] return result[0] @@ -320,14 +328,13 @@ def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, parent_can_read): - print('[Child]: Whats up') - + print('[Child]: Started local opt at {}.'.format(x0)) n = len(gen_specs['ub']) opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) - lb = gen_specs['lb'] - ub = gen_specs['ub'] + lb = np.zeros(n) + ub = np.ones(n) opt.set_lower_bounds(lb) opt.set_upper_bounds(ub) @@ -355,10 +362,10 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, if 'ftol_abs' in gen_specs: opt.set_ftol_abs(gen_specs['ftol_abs']) + #FIXME: Do we need to do something of the final 'x_opt'? + print('[Child]: Started my optimization') x_opt = opt.optimize(x0) - #FIXME: Do we need to do of the final 'x_opt'? - def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): @@ -378,7 +385,6 @@ def add_to_local_H(local_H, pts, gen_specs, c_flag, local_flag=0, sorted_run_ind """ Adds points to O, the numpy structured array to be sent back to the manager """ - assert not local_flag or len(pts) == 1, "Can't > 1 local points" len_local_H = len(local_H) @@ -986,7 +992,6 @@ def initialize_APOSMM(H, gen_specs, libE_info): total_runs = 0 - comm = libE_info['comm'] local_H_fields = [('f',float), From 5a7494ac39b568c196b2db54bf93c362dc217bf2 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 17 Jul 2019 09:54:46 -0500 Subject: [PATCH 020/644] smore progress --- libensemble/gen_funcs/persistent_aposmm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index f0623c142..792a70b0d 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -213,7 +213,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in - # Kaushik, I have concerns about the above approach for processing calc_in + # JL: Kaushik, I have concerns about the above approach for processing calc_in # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. # 2. Note that grad might not be available for some objectives. # Perhaps it's better to use something like @@ -223,8 +223,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # if 'grad' in calc_in.dtype.names: # grad_f_x_recv = calc_in['grad'] ->>>>>>> 75e1b29f18f5ba95be57fd2f3089455d19179977 - if tag in [STOP_TAG, PERSIS_STOP]: print('[Parent]: Received a stop tag, killing all children.') # FIXME: If there is any indication that all child processes are From fb4397408f296758c6926d03965f119eece0aa1e Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 17 Jul 2019 13:51:46 -0500 Subject: [PATCH 021/644] idenitifies when the child local opt converges --- libensemble/gen_funcs/persistent_aposmm.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 792a70b0d..f6922d299 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -201,7 +201,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices = {} if gen_specs['initial_sample_size'] != 1: - raise RuntimeError("sample size > not supported for now.") + raise RuntimeError("sample size > 1 not supported for now.") + + # We just sampled a single point. send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) @@ -214,8 +216,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): if calc_in: (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in # JL: Kaushik, I have concerns about the above approach for processing calc_in - # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. - # 2. Note that grad might not be available for some objectives. + # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. + # 2. Note that grad might not be available for some objectives. # Perhaps it's better to use something like # x_recv = calc_in['x'] # f_x_recv = calc_in['f'] @@ -246,9 +248,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue.clear() child_can_read_evts[child_idx].set() + print('[Parent]: I am waiting') parent_can_read_from_queue.wait() + print('[Parent]: Wohoo I am free again') x_new = comm_queue.get() + if x_new == 'Converged': + raise NotImplementedError("We don't know what to do yet.") print('[Parent]: The child returned {}.'.format(x_new)) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) @@ -283,6 +289,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): p.start() print('[Parent]: Started a child process') + print('[Parent]: I am waiting') parent_can_read_from_queue.wait() print('[Parent]: Done waiting.') @@ -326,7 +333,9 @@ def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_ comm_queue.put(x) print('[Child]: Parent should no longer wait.') parent_can_read.set() + print('[Child]: I have started waiting') child_can_read.wait() + print('[Child]: Wohooo.. I am free folks') result = comm_queue.get() child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: @@ -374,6 +383,9 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, #FIXME: Do we need to do something of the final 'x_opt'? print('[Child]: Started my optimization') x_opt = opt.optimize(x0) + print('[Child]: I have converged.') + comm_queue.put('Converged') + parent_can_read.set() def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): @@ -1062,3 +1074,5 @@ def display_exception(e): # persis_info = persis_info.item() # import ipdb; ipdb.set_trace() # aposmm_logic(H,persis_info,gen_specs,{}) + +# vim:fdm=marker From fa68d4a9dec197a78112972396432ba1b97c3d36 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 18 Jul 2019 10:25:30 -0500 Subject: [PATCH 022/644] readds the support for >1 initial_sample_size --- libensemble/gen_funcs/persistent_aposmm.py | 38 ++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index f6922d299..7043f2ef8 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -200,9 +200,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices = {} - if gen_specs['initial_sample_size'] != 1: - raise RuntimeError("sample size > 1 not supported for now.") - # We just sampled a single point. send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, @@ -253,20 +250,26 @@ def aposmm(H, persis_info, gen_specs, libE_info): print('[Parent]: Wohoo I am free again') x_new = comm_queue.get() - if x_new == 'Converged': - raise NotImplementedError("We don't know what to do yet.") print('[Parent]: The child returned {}.'.format(x_new)) - - add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(x_new, local_H[-1]['x_on_cube']) - - # FIXME:[KK]: Would it be true that the local_H[-1] would be our - # new point? - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) + if isinstance(x_new, tuple): + # FIXME: [KK]: This is ugly and every bit of it happens in + # my mind, this should be cleaned up duing the cleanup + # phase. + x_opt = x_new[1] + # Need to figure out what are run_ids. + update_history_optimal(x_opt, H, 1/0) + raise NotImplementedError("We don't know what to do yet.") else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) + add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) + assert np.allclose(x_new, local_H[-1]['x_on_cube']) + + # FIXME:[KK]: Would it be true that the local_H[-1] would be our + # new point? + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) + else: + sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) else: n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -384,7 +387,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, print('[Child]: Started my optimization') x_opt = opt.optimize(x0) print('[Child]: I have converged.') - comm_queue.put('Converged') + comm_queue.put(('Converged', x_opt)) parent_can_read.set() @@ -1048,7 +1051,8 @@ def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, assert sim_id not in sim_id_to_child_indices sim_id_to_child_indices[sim_id] = None - send_mgr_worker_msg(comm, local_H[['x', 'sim_id']]) + for i in range(len(local_H)): + send_mgr_worker_msg(comm, local_H[i][['x', 'sim_id']]) def display_exception(e): From 5f87d104a875f0d869bad07158aea2eca2acdafa Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 18 Jul 2019 16:54:42 -0500 Subject: [PATCH 023/644] introduces child_id_to_run_id --- libensemble/gen_funcs/persistent_aposmm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 7043f2ef8..469283213 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -199,6 +199,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) sim_id_to_child_indices = {} + child_id_to_run_id = {} # We just sampled a single point. @@ -266,6 +267,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # FIXME:[KK]: Would it be true that the local_H[-1] would be our # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + persis_info['run_order'][child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: @@ -311,6 +313,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): # FIXME: again makes the assumption that the the newly added # point comes in local_H[-1] + persis_info['run_order'][total_runs] = [local_H[-1]['sim_id']] + child_id_to_run_id[len(processes)-1] = total_runs + # Need to figure out the child number from the run number. if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) From b0192f60c64bcfbb6b405baa1e13a5ea5c69d138 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 19 Jul 2019 16:09:01 -0500 Subject: [PATCH 024/644] brings back in update_history_optimal and uses a local run_order --- libensemble/gen_funcs/persistent_aposmm.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 469283213..4c2e5dbd9 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -186,7 +186,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - # {{{ multiprocessing initialization child_can_read_evts = [] @@ -200,6 +199,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices = {} child_id_to_run_id = {} + run_order = {} # We just sampled a single point. @@ -258,16 +258,17 @@ def aposmm(H, persis_info, gen_specs, libE_info): # phase. x_opt = x_new[1] # Need to figure out what are run_ids. - update_history_optimal(x_opt, H, 1/0) - raise NotImplementedError("We don't know what to do yet.") + add_to_local_H(local_H, (x_opt, ), gen_specs, c_flag, local_flag=1, on_cube=True) + update_history_optimal(x_opt, local_H, + run_order[child_id_to_run_id[child_idx]]) else: add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(x_new, local_H[-1]['x_on_cube']) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) # FIXME:[KK]: Would it be true that the local_H[-1] would be our # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - persis_info['run_order'][child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) + run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: @@ -278,7 +279,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) for ind in starting_inds: - total_runs += 1 local_H['started_run'][ind] = 1 child_can_read_evts.append(Event()) @@ -310,11 +310,17 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = comm_queue.get() add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) + + print('Index:', ind) + print('Sim_ids:', local_H[:]['sim_id']) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) # FIXME: again makes the assumption that the the newly added # point comes in local_H[-1] - persis_info['run_order'][total_runs] = [local_H[-1]['sim_id']] + run_order[total_runs] = [local_H[-1]['sim_id']] child_id_to_run_id[len(processes)-1] = total_runs + total_runs += 1 # Need to figure out the child number from the run number. if local_H[-1]['sim_id'] in sim_id_to_child_indices: From 119b4f6baaafab80d39ed25a00d5c30846e32c9f Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 22 Jul 2019 11:48:42 -0500 Subject: [PATCH 025/644] adds some fancy debug statements --- libensemble/gen_funcs/persistent_aposmm.py | 34 +++++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 4c2e5dbd9..5e2d1c979 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -209,10 +209,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: + # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) + if calc_in: (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + print(23*"-", "Received f({})".format(x_recv), 24*"-", + flush=True) # JL: Kaushik, I have concerns about the above approach for processing calc_in # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. # 2. Note that grad might not be available for some objectives. @@ -224,7 +228,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): # grad_f_x_recv = calc_in['grad'] if tag in [STOP_TAG, PERSIS_STOP]: - print('[Parent]: Received a stop tag, killing all children.') + + print('[Parent]: Received a stop tag, killing all children.', flush=True) # FIXME: If there is any indication that all child processes are # done with their work, we can do this a bit more cleanly i.e. # first join and check if they are still alive, only then kill. @@ -240,18 +245,18 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: print('[Parent]: Received the eval for {}, giving it to the' - ' child_idx: {}.'.format(x_recv, child_idx)) + ' child_idx: {}.'.format(x_recv, child_idx), flush=True) comm_queue.put((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.clear() child_can_read_evts[child_idx].set() - print('[Parent]: I am waiting') + print('[Parent]: I am waiting for child idx {}'.format(child_idx), flush=True) parent_can_read_from_queue.wait() - print('[Parent]: Wohoo I am free again') + print('[Parent]: Wohoo I am free again', flush=True) x_new = comm_queue.get() - print('[Parent]: The child returned {}.'.format(x_new)) + print('[Parent]: The child returned {}.'.format(x_new), flush=True) if isinstance(x_new, tuple): # FIXME: [KK]: This is ugly and every bit of it happens in # my mind, this should be cleaned up duing the cleanup @@ -261,6 +266,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): add_to_local_H(local_H, (x_opt, ), gen_specs, c_flag, local_flag=1, on_cube=True) update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) + print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) + processes[child_idx].join() + print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) else: add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) @@ -273,7 +281,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) + else: + print('[Parent]: Did not find any local opt run to associate with.', + flush=True) n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) @@ -283,7 +294,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_can_read_evts.append(Event()) - print('Initing process') + print('[Parent]: Initing process', flush=True) parent_can_read_from_queue.clear() p = Process(target=run_local_nlopt, args=(gen_specs, @@ -292,11 +303,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue)) processes.append(p) p.start() - print('[Parent]: Started a child process') + print('[Parent]: Started a child process', flush=True) - print('[Parent]: I am waiting') + print('[Parent]: I am waiting', flush=True) parent_can_read_from_queue.wait() - print('[Parent]: Done waiting.') + print('[Parent]: Done waiting.', flush=True) assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) @@ -305,7 +316,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_can_read_evts[-1].set() parent_can_read_from_queue.wait() - print('[Master]: Getting a new x_new.') + print('[Master]: Getting a new x_new.', flush=True) x_new = comm_queue.get() @@ -329,6 +340,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + print('[Parent]: Heading to next iteration', flush=True) comm_queue.close() comm_queue.join_thread() @@ -1063,7 +1075,7 @@ def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices[sim_id] = None for i in range(len(local_H)): - send_mgr_worker_msg(comm, local_H[i][['x', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[i:i+1][['x', 'sim_id']]) def display_exception(e): From 1409424a6c9bfcf3d605af1d8c6d777e28841e1d Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 23 Jul 2019 11:06:16 -0500 Subject: [PATCH 026/644] some more progress by ignoring same eval received from manager --- libensemble/gen_funcs/persistent_aposmm.py | 64 ++++++++++++++++------ 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 5e2d1c979..0035be5e8 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -197,14 +197,17 @@ def aposmm(H, persis_info, gen_specs, libE_info): n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + received_values = set() + sim_id_to_child_indices = {} child_id_to_run_id = {} run_order = {} # We just sampled a single point. - send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + for _ in range(gen_specs['initial_sample_size']): + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) tag = None O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 @@ -217,6 +220,15 @@ def aposmm(H, persis_info, gen_specs, libE_info): (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in print(23*"-", "Received f({})".format(x_recv), 24*"-", flush=True) + if tuple(x_recv) in received_values: + #FIXME:: This should never be the case. + print("DANGER!! Re-received a value ignoring it for now, but" + " need to fix it in manager, I guess?", flush=True) + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) + continue + + received_values.add(tuple(x_recv)) # JL: Kaushik, I have concerns about the above approach for processing calc_in # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. # 2. Note that grad might not be available for some objectives. @@ -244,8 +256,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: - print('[Parent]: Received the eval for {}, giving it to the' - ' child_idx: {}.'.format(x_recv, child_idx), flush=True) + print('[Parent]:Giving f = {} it to the' + ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) comm_queue.put((f_x_recv, grad_f_x_recv)) parent_can_read_from_queue.clear() @@ -269,22 +281,26 @@ def aposmm(H, persis_info, gen_specs, libE_info): print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) processes[child_idx].join() print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) + + # TODO: send a sample point on converging.. + # This one still sends the number of points equal to the + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) else: add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - # FIXME:[KK]: Would it be true that the local_H[-1] would be our - # new point? send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) - else: print('[Parent]: Did not find any local opt run to associate with.', flush=True) + if n_s < gen_specs['initial_sample_size']: + continue n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) @@ -323,12 +339,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - print('Index:', ind) - print('Sim_ids:', local_H[:]['sim_id']) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - # FIXME: again makes the assumption that the the newly added - # point comes in local_H[-1] run_order[total_runs] = [local_H[-1]['sim_id']] child_id_to_run_id[len(processes)-1] = total_runs total_runs += 1 @@ -340,6 +352,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + if len(starting_inds) == 0: + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) print('[Parent]: Heading to next iteration', flush=True) comm_queue.close() @@ -352,16 +367,19 @@ def aposmm(H, persis_info, gen_specs, libE_info): raise RuntimeError("[Parent]: Atleast one child process is still active, even after" " killing all the children.") + #TODO: Figure out what do we need to do from here. + 1/0 + return O, persis_info, tag def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) - print('[Child]: Parent should no longer wait.') + print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() - print('[Child]: I have started waiting') + print('[Child]: I have started waiting', flush=True) child_can_read.wait() - print('[Child]: Wohooo.. I am free folks') + print('[Child]: Wohooo.. I am free folks', flush=True) result = comm_queue.get() child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: @@ -372,7 +390,7 @@ def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, parent_can_read): - print('[Child]: Started local opt at {}.'.format(x0)) + print('[Child]: Started local opt at {}.'.format(x0), flush=True) n = len(gen_specs['ub']) opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) @@ -407,9 +425,9 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, opt.set_ftol_abs(gen_specs['ftol_abs']) #FIXME: Do we need to do something of the final 'x_opt'? - print('[Child]: Started my optimization') + print('[Child]: Started my optimization', flush=True) x_opt = opt.optimize(x0) - print('[Child]: I have converged.') + print('[Child]: I have converged.', flush=True) comm_queue.put(('Converged', x_opt)) parent_can_read.set() @@ -1066,8 +1084,20 @@ def initialize_APOSMM(H, gen_specs, libE_info): return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm, local_H + +def send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices): + sampled_points = persis_info['rand_stream'].uniform(0, 1, (1, n)) + add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) + assert local_H['sim_id'][-1] not in sim_id_to_child_indices + sim_id_to_child_indices[local_H['sim_id'][-1]] = None + + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + + def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): + 1/0 sampled_points = persis_info['rand_stream'].uniform(0, 1, (gen_specs['initial_sample_size'], n)) add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) for sim_id in local_H['sim_id'][-gen_specs['initial_sample_size']:]: From ae4dbe512c5e7af9ff83c450c9930c442f6f79d0 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 23 Jul 2019 13:55:09 -0500 Subject: [PATCH 027/644] updates current test params --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 6dea74657..4a1ccaaf9 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -39,7 +39,7 @@ 'in': [], 'out': gen_out, 'batch_mode': True, - 'initial_sample_size': 1, + 'initial_sample_size': 50, 'localopt_method': 'LD_MMA', 'xtol_rel': 1e-3, 'lb': np.array([-3, -2]), @@ -49,7 +49,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 40} +exit_criteria = {'sim_max': 400} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From a1968f89eceb8adc133976d3ba37acd86e2e6792 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 23 Jul 2019 13:57:10 -0500 Subject: [PATCH 028/644] debug statements --- libensemble/alloc_funcs/support.py | 3 +++ libensemble/gen_funcs/persistent_aposmm.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libensemble/alloc_funcs/support.py b/libensemble/alloc_funcs/support.py index 927003ffe..56ebe43aa 100644 --- a/libensemble/alloc_funcs/support.py +++ b/libensemble/alloc_funcs/support.py @@ -25,6 +25,7 @@ def count_persis_gens(W): def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add sim work record to work array." + print('[Allocator]: Calling sim_work for {}.'.format(H_rows), flush=True) libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, @@ -34,6 +35,8 @@ def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): def gen_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add gen work record to work array." + print('[Allocator]: Calling gen_work for {}.'.format(H_rows), flush=True) + libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 0035be5e8..c41ec4f6b 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -222,8 +222,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): flush=True) if tuple(x_recv) in received_values: #FIXME:: This should never be the case. - print("DANGER!! Re-received a value ignoring it for now, but" - " need to fix it in manager, I guess?", flush=True) + print("DANGER!! Re-received a value(SimID = {}) ignoring it for now, but" + " need to fix it in manager, I guess?".format(sim_id_recv), flush=True) send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) continue @@ -360,7 +360,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue.close() comm_queue.join_thread() - print('Done with everything') + print('Done with everything', flush=True) for p in processes: if p.is_alive(): @@ -369,6 +369,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): #TODO: Figure out what do we need to do from here. 1/0 + print('This line should not appear', flush=True) return O, persis_info, tag From fac46436e658922d0ecd75b07ba68bd74e215c60 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 23 Jul 2019 15:20:17 -0500 Subject: [PATCH 029/644] Broken code, but coming up to talk --- libensemble/alloc_funcs/start_only_persistent.py | 4 +++- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index a126f796d..c1a35e228 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -21,13 +21,15 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): gen_inds = (H['gen_worker'] == i) - if np.all(H['returned'][gen_inds]): + if np.any(np.logical_and(H['returned'][gen_inds],~H['given_back'][gen_inds])): last_time_gen_gave_batch = np.max(H['gen_time'][gen_inds]) inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) + H['given_back'][inds_of_last_batch_from_gen] = True + task_avail = ~H['given'] for i in avail_worker_ids(W, persistent=False): if np.any(task_avail): diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 4a1ccaaf9..9c3b7313b 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -45,7 +45,7 @@ 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} -alloc_specs = {'alloc_f': alloc_f, 'out': []} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back',bool)]} persis_info = per_worker_stream({}, nworkers + 1) From bddfdc675a89979450b97c3cb4e46d1b81a3b099 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 23 Jul 2019 16:12:01 -0500 Subject: [PATCH 030/644] Helping with the debugging for Kaushik --- libensemble/gen_funcs/support.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index e5084828d..e083c937e 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,6 +11,7 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ + print('About to send: ', O['sim_id']) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From f39155ee82213107805cca6a514112208218c02f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 24 Jul 2019 07:50:40 -0500 Subject: [PATCH 031/644] Helping with the debugging for Kaushik --- libensemble/gen_funcs/support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index e083c937e..094914344 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,7 +11,7 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ - print('About to send: ', O['sim_id']) + print('About to send: ', O['sim_id'],flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From 10f6f726868820c398ac13210f24a7eb3d1d2397 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 24 Jul 2019 08:53:05 -0500 Subject: [PATCH 032/644] Adding a libE check that generated sim_ids are in order --- libensemble/history.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libensemble/history.py b/libensemble/history.py index ce852d1f1..6dc4c1100 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -150,7 +150,11 @@ def update_history_x_in(self, gen_worker, O): update_inds = np.arange(self.index, self.index+num_new) self.H['sim_id'][self.index:self.index+num_new] = range(self.index, self.index+num_new) else: - # gen method is building sim_id. + # gen method is building sim_id or adjusting values in existing sim_id rows. + + # Ensure there aren't any gaps in the generated sim_id values: + assert np.all(np.isin(np.arange(self.index,np.max(O['sim_id'])+1),O['sim_id'])), "The generator function has produced sim_id that are not in order." + num_new = len(np.setdiff1d(O['sim_id'], self.H['sim_id'])) if num_new > rows_remaining: From 9e8d5103bb61a8eca02973b234b835a275375cdc Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 24 Jul 2019 09:17:17 -0500 Subject: [PATCH 033/644] some more debug statements about send/receive --- libensemble/gen_funcs/support.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index e5084828d..8e432e7b3 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -1,4 +1,5 @@ from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, UNSET_TAG, EVAL_GEN_TAG +import sys def sendrecv_mgr_worker_msg(comm, O, status=None): @@ -11,6 +12,8 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ + assert len(O) == 1 + print(25*"-", "Sending {}".format(O[0][0]), 25*"-", flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, @@ -22,9 +25,14 @@ def send_mgr_worker_msg(comm, O): def get_mgr_worker_msg(comm, status=None): """Get message to worker from manager. """ + print('[Parent]: Expecting a message from Manager...', flush=True) tag, Work = comm.recv() + print('[Parent]: Received a tag {}'.format(tag), flush=True) + sys.stdout.flush() if tag in [STOP_TAG, PERSIS_STOP]: comm.push_back(tag, Work) return tag, None, None + print('[Parent]: Expecting a message from Manager...', flush=True) _, calc_in = comm.recv() + print('[Parent]: Received some message.', flush=True) return tag, Work, calc_in From 8cd262cd01250333e9da69e9a4d569389af47d26 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Wed, 24 Jul 2019 11:19:13 -0500 Subject: [PATCH 034/644] fixes the bug of giving non-consecutive sim_ids --- libensemble/gen_funcs/persistent_aposmm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c41ec4f6b..6eb4dcbc3 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -275,15 +275,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): # phase. x_opt = x_new[1] # Need to figure out what are run_ids. - add_to_local_H(local_H, (x_opt, ), gen_specs, c_flag, local_flag=1, on_cube=True) update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) processes[child_idx].join() print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) - # TODO: send a sample point on converging.. - # This one still sends the number of points equal to the send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) else: From 26ea1b54d475dc8b5c67413337b7cd5ab47e7178 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 24 Jul 2019 13:31:57 -0500 Subject: [PATCH 035/644] Using a much older numpy function to accomplish the same thing --- libensemble/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/history.py b/libensemble/history.py index 6dc4c1100..22f3907e9 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -153,7 +153,7 @@ def update_history_x_in(self, gen_worker, O): # gen method is building sim_id or adjusting values in existing sim_id rows. # Ensure there aren't any gaps in the generated sim_id values: - assert np.all(np.isin(np.arange(self.index,np.max(O['sim_id'])+1),O['sim_id'])), "The generator function has produced sim_id that are not in order." + assert np.all(np.in1d(np.arange(self.index, np.max(O['sim_id'])+1), O['sim_id'])), "The generator function has produced sim_id that are not in order." num_new = len(np.setdiff1d(O['sim_id'], self.H['sim_id'])) From fda88612efc3dbb70fb155ce29e51951dce3b23b Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 25 Jul 2019 14:54:21 -0500 Subject: [PATCH 036/644] adds back the max_active_runs --- libensemble/gen_funcs/persistent_aposmm.py | 130 +++++++++++++++------ 1 file changed, 93 insertions(+), 37 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 6eb4dcbc3..a3384de91 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -210,6 +210,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices) tag = None + max_active_runs = gen_specs.get('max_active_runs', np.inf) + waiting_starting_inds = [] O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 while 1: @@ -281,7 +283,57 @@ def aposmm(H, persis_info, gen_specs, libE_info): processes[child_idx].join() print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, + if waiting_starting_inds: + print('[Parent]: There was a waiting starting index' + ' and started a run from there.', flush=True) + ind = waiting_starting_inds[0] + waiting_starting_inds = waiting_starting_inds[1:] + + # {{{ logic duplication(need to get rid of it) + + p = Process(target=run_local_nlopt, args=(gen_specs, + comm_queue, local_H[ind]['x_on_cube'], + child_can_read_evts[-1], + parent_can_read_from_queue)) + processes.append(p) + p.start() + print('[Parent]: Started a child process', flush=True) + + print('[Parent]: I am waiting', flush=True) + parent_can_read_from_queue.wait() + print('[Parent]: Done waiting.', flush=True) + + assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) + + comm_queue.put(local_H[ind][['f', 'grad']]) + parent_can_read_from_queue.clear() + + child_can_read_evts[-1].set() + parent_can_read_from_queue.wait() + print('[Master]: Getting a new x_new.', flush=True) + + x_new = comm_queue.get() + + add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) + + assert np.allclose(local_H[-1]['x_on_cube'], x_new) + + run_order[total_runs] = [local_H[-1]['sim_id']] + child_id_to_run_id[len(processes)-1] = total_runs + total_runs += 1 + # Need to figure out the child number from the run number. + + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) + else: + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) + + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + + # }}} + else: + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) else: add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) @@ -310,46 +362,52 @@ def aposmm(H, persis_info, gen_specs, libE_info): print('[Parent]: Initing process', flush=True) parent_can_read_from_queue.clear() - p = Process(target=run_local_nlopt, args=(gen_specs, - comm_queue, local_H[ind]['x_on_cube'], - child_can_read_evts[-1], - parent_can_read_from_queue)) - processes.append(p) - p.start() - print('[Parent]: Started a child process', flush=True) - - print('[Parent]: I am waiting', flush=True) - parent_can_read_from_queue.wait() - print('[Parent]: Done waiting.', flush=True) - assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) + if len([p for p in processes if p.is_alive()]) < max_active_runs: + p = Process(target=run_local_nlopt, args=(gen_specs, + comm_queue, local_H[ind]['x_on_cube'], + child_can_read_evts[-1], + parent_can_read_from_queue)) + processes.append(p) + p.start() + print('[Parent]: Started a child process', flush=True) - comm_queue.put(local_H[ind][['f', 'grad']]) - parent_can_read_from_queue.clear() + print('[Parent]: I am waiting', flush=True) + parent_can_read_from_queue.wait() + print('[Parent]: Done waiting.', flush=True) - child_can_read_evts[-1].set() - parent_can_read_from_queue.wait() - print('[Master]: Getting a new x_new.', flush=True) + assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - x_new = comm_queue.get() + comm_queue.put(local_H[ind][['f', 'grad']]) + parent_can_read_from_queue.clear() - add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) + child_can_read_evts[-1].set() + parent_can_read_from_queue.wait() + print('[Master]: Getting a new x_new.', flush=True) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) + x_new = comm_queue.get() - run_order[total_runs] = [local_H[-1]['sim_id']] - child_id_to_run_id[len(processes)-1] = total_runs - total_runs += 1 - # Need to figure out the child number from the run number. + add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) - if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) - else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) + assert np.allclose(local_H[-1]['x_on_cube'], x_new) - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - if len(starting_inds) == 0: + run_order[total_runs] = [local_H[-1]['sim_id']] + child_id_to_run_id[len(processes)-1] = total_runs + total_runs += 1 + # Need to figure out the child number from the run number. + + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) + else: + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) + + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + else: + print('[Parent]: Not exceeding max active runs, putting it' + ' in our stash', flush=True) + waiting_starting_inds.append(ind) + if len(starting_inds) == 0 and len(waiting_starting_inds) == 0: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) print('[Parent]: Heading to next iteration', flush=True) @@ -365,10 +423,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): " killing all the children.") #TODO: Figure out what do we need to do from here. - 1/0 - print('This line should not appear', flush=True) - return O, persis_info, tag + return local_H, persis_info, tag def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): @@ -1075,10 +1131,10 @@ def initialize_APOSMM(H, gen_specs, libE_info): ('sim_id', int), ('paused', bool), ('returned', bool), - ('pt_id', int), # Identify the same point evaluated by different sim_f's or components + ('pt_id', int), # Identify the same point evaluated by different sim_f's or components ] - local_H = np.empty(0,dtype=local_H_fields); + local_H = np.empty(0, dtype=local_H_fields) return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm, local_H From dad7f5274ed2bcb4111d721379d485b2469a3250 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Thu, 25 Jul 2019 14:55:03 -0500 Subject: [PATCH 037/644] enforces max active runs through gen_specs --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 9c3b7313b..cafb39620 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -42,6 +42,7 @@ 'initial_sample_size': 50, 'localopt_method': 'LD_MMA', 'xtol_rel': 1e-3, + 'max_active_runs': 1, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} From 52d8aa3eccc3ca5d31d43fc0ccca7350d8126c17 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Sun, 28 Jul 2019 19:07:35 -0500 Subject: [PATCH 038/644] preliminary work on porting the local optimization techniques to TAO/SciPy --- libensemble/gen_funcs/persistent_aposmm.py | 167 ++++++++++++++++++++- 1 file changed, 162 insertions(+), 5 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index a3384de91..32494a9a5 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -203,7 +203,18 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_id_to_run_id = {} run_order = {} - # We just sampled a single point. + if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', + 'LN_NELDERMEAD', 'LD_MMA']: + run_local_opt = run_local_nlopt + pass + elif gen_specs['localopt_method'] in ['pounders']: + run_local_opt = run_local_tao + elif gen_specs['localopt_method'] in ['scipy_COBYLA']: + run_local_opt = run_local_scipy + pass + else: + raise NotImplementedError("Unknown local optimization method " + "'{}'.".format(gen_specs['localopt_method'])) for _ in range(gen_specs['initial_sample_size']): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, @@ -364,7 +375,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue.clear() if len([p for p in processes if p.is_alive()]) < max_active_runs: - p = Process(target=run_local_nlopt, args=(gen_specs, + p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x_on_cube'], child_can_read_evts[-1], parent_can_read_from_queue)) @@ -412,6 +423,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): sim_id_to_child_indices) print('[Parent]: Heading to next iteration', flush=True) + print('[Parent]: The optimal points are:\n', + local_H[np.where(local_H['local_min'])]['x'], flush=True) + comm_queue.close() comm_queue.join_thread() @@ -427,7 +441,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): return local_H, persis_info, tag -def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): +# {{{ NLOPT for local opt.. + +def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() @@ -442,7 +458,7 @@ def callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_ return result[0] -def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, +def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): print('[Child]: Started local opt at {}.'.format(x0), flush=True) n = len(gen_specs['ub']) @@ -466,7 +482,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, # FIXME: Setting max evaluations = 100 opt.set_maxeval(100) - opt.set_min_objective(lambda x, grad: callback_function(x, grad, + opt.set_min_objective(lambda x, grad: nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs)) if 'xtol_rel' in gen_specs: @@ -485,6 +501,147 @@ def run_local_nlopt(gen_specs, comm_queue, x0, child_can_read, comm_queue.put(('Converged', x_opt)) parent_can_read.set() +# }}} + + +# {{{ SciPy optimization + +def scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_specs): + comm_queue.put(x) + print('[Child]: Parent should no longer wait.', flush=True) + parent_can_read.set() + print('[Child]: I have started waiting', flush=True) + child_can_read.wait() + print('[Child]: Wohooo.. I am free folks', flush=True) + result, = comm_queue.get() + child_can_read.clear() + return result + + +def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, + parent_can_read): + + # Construct the bounds in the form of constraints + cons = [] + for factor in range(len(x0)): + lo = {'type': 'ineq', + 'fun': lambda x, lb=gen_specs['lb'][factor], i=factor: x[i]-lb} + up = {'type': 'ineq', + 'fun': lambda x, ub=gen_specs['ub'][factor], i=factor: ub-x[i]} + cons.append(lo) + cons.append(up) + + method = gen_specs['localopt_method'][6:] + print('[Child]: Started my optimization', flush=True) + res = scipy_optimize.minimize(lambda x: scipy_callback_function(x, + comm_queue, child_can_read, parent_can_read, gen_specs), x0, + method=method, options={'maxiter': 100, 'tol': gen_specs['tol']}) + + if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum + exit_code = 0 + else: + if method == 'COBYLA': + assert res['status'] == 1, "Unknown status for COBYLA" + exit_code = 1 + + x_opt = res['x'] + + # FIXME: Need to do something with the exit codes. + + print('[Child]: I have converged.', flush=True) + comm_queue.put(('Converged', x_opt)) + parent_can_read.set() + +# }}} + + +# {{{ TAO routines for local opt + + +def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): + comm_queue.put(x) + print('[Child]: Parent should no longer wait.', flush=True) + parent_can_read.set() + print('[Child]: I have started waiting', flush=True) + child_can_read.wait() + print('[Child]: Wohooo.. I am free folks', flush=True) + result = comm_queue.get() + child_can_read.clear() + f.array[:] = result[0] + return f + + +def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, + parent_can_read): + assert isinstance(x0, np.array) + + tao_comm = MPI.COMM_SELF + n, = x0.shape + m, = f0.shape + + # Create starting point, bounds, and tao object + x = PETSc.Vec().create(tao_comm) + x.setSizes(n) + x.setFromOptions() + x.array = x0 + lb = x.duplicate() + ub = x.duplicate() + lb.array = 0*np.ones(n) + ub.array = 1*np.ones(n) + tao = PETSc.TAO().create(tao_comm) + tao.setType(gen_specs['localopt_method']) + + f = PETSc.Vec().create(tao_comm) + f.setSizes(m) + f.setFromOptions() + + delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + + PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) + # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') + + tao.setResidual(lambda tao, x, f: tao_callback_function(tao, x, f, + comm_queue, child_can_read, parent_can_read, gen_specs), f) + + # Set everything for tao before solving + # FIXME: Hard-coding 100 as the max funcs as couldn't find any other + # sensible value. + PETSc.Options().setValue('-tao_max_funcs', '100') + tao.setFromOptions() + tao.setVariableBounds((lb, ub)) + # tao.setObjectiveTolerances(fatol=gen_specs['fatol'], frtol=gen_specs['frtol']) + # tao.setGradientTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + tao.setInitial(x) + + print('[Child]: Started my optimization', flush=True) + tao.solve(x) + + x_opt = tao.getSolution().getArray() + exit_code = tao.getConvergedReason() + # print(exit_code) + # print(tao.view()) + # print(x_opt) + + # if gen_specs['localopt_method'] == 'pounders': + f.destroy() + # elif gen_specs['localopt_method'] == 'blmvm': + # g.destroy() + + lb.destroy() + ub.destroy() + x.destroy() + tao.destroy() + + #FIXME: Do we need to do something of the final 'x_opt'? + print('[Child]: I have converged.', flush=True) + comm_queue.put(('Converged', x_opt)) + parent_can_read.set() + + # FIXME: What do we do about the exit_code? + +# }}} + def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): From c23852ea3ea0d619363e90d756ea101f4fc3e114 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 09:58:41 -0500 Subject: [PATCH 039/644] generalizes the passing of f and grad_f --- libensemble/gen_funcs/persistent_aposmm.py | 46 ++++++++++++++-------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 32494a9a5..5d87f5bf2 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -203,19 +203,35 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_id_to_run_id = {} run_order = {} + # {{{ setting the local optimization method + if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: run_local_opt = run_local_nlopt - pass elif gen_specs['localopt_method'] in ['pounders']: run_local_opt = run_local_tao elif gen_specs['localopt_method'] in ['scipy_COBYLA']: - run_local_opt = run_local_scipy - pass + run_local_opt = run_local_scipy_opt + else: + raise NotImplementedError("Unknown local optimization method " + "'{}'.".format(gen_specs['localopt_method'])) + + # }}} + + # {{{ setting the data needed by the local optimization method + + if gen_specs['localopt_method'] in ['LD_MMA']: + fields_to_pass = ['f', 'grad'] + elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', + 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: + fields_to_pass = ['f'] else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) + # }}} + + for _ in range(gen_specs['initial_sample_size']): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) @@ -230,7 +246,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + if fields_to_pass == ['f', 'grad']: + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + data_to_give_processes = (f_x_recv, grad_f_x_recv) + else: + assert(fields_to_pass == ['f']) + (x_recv, f_x_recv, sim_id_recv), = calc_in + data_to_give_processes = (f_x_recv, ) + print(23*"-", "Received f({})".format(x_recv), 24*"-", flush=True) if tuple(x_recv) in received_values: @@ -242,15 +265,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): continue received_values.add(tuple(x_recv)) - # JL: Kaushik, I have concerns about the above approach for processing calc_in - # 1. I'm not sure that the contents of calc_in will always be in the order, x, f, grad, sim_id. - # 2. Note that grad might not be available for some objectives. - # Perhaps it's better to use something like - # x_recv = calc_in['x'] - # f_x_recv = calc_in['f'] - # sim_id_recv = calc_in['sim_id'] - # if 'grad' in calc_in.dtype.names: - # grad_f_x_recv = calc_in['grad'] if tag in [STOP_TAG, PERSIS_STOP]: @@ -271,7 +285,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): for child_idx in sim_id_to_child_indices[sim_id_recv]: print('[Parent]:Giving f = {} it to the' ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) - comm_queue.put((f_x_recv, grad_f_x_recv)) + comm_queue.put(data_to_give_processes) parent_can_read_from_queue.clear() @@ -316,7 +330,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - comm_queue.put(local_H[ind][['f', 'grad']]) + comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() child_can_read_evts[-1].set() @@ -389,7 +403,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - comm_queue.put(local_H[ind][['f', 'grad']]) + comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() child_can_read_evts[-1].set() From 94c0ef2580efceb3c78a02075ab419ce3b3e43fb Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 10:10:20 -0500 Subject: [PATCH 040/644] makes var names more intuitive --- libensemble/gen_funcs/persistent_aposmm.py | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 5d87f5bf2..3858670a3 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -316,7 +316,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # {{{ logic duplication(need to get rid of it) - p = Process(target=run_local_nlopt, args=(gen_specs, + p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x_on_cube'], child_can_read_evts[-1], parent_can_read_from_queue)) @@ -464,12 +464,17 @@ def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read print('[Child]: I have started waiting', flush=True) child_can_read.wait() print('[Child]: Wohooo.. I am free folks', flush=True) - result = comm_queue.get() - child_can_read.clear() if gen_specs['localopt_method'] in ['LD_MMA']: - grad[:] = result[1] + f_recv, grad_recv = comm_queue.get() + grad[:] = grad_recv + else: + assert gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', + 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] + f_recv, = comm_queue.get() - return result[0] + child_can_read.clear() + + return f_recv def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, @@ -527,9 +532,9 @@ def scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_ print('[Child]: I have started waiting', flush=True) child_can_read.wait() print('[Child]: Wohooo.. I am free folks', flush=True) - result, = comm_queue.get() + f_x_recv, = comm_queue.get() child_can_read.clear() - return result + return f_x_recv def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, @@ -571,7 +576,6 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, # {{{ TAO routines for local opt - def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) print('[Child]: Parent should no longer wait.', flush=True) @@ -579,9 +583,9 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read print('[Child]: I have started waiting', flush=True) child_can_read.wait() print('[Child]: Wohooo.. I am free folks', flush=True) - result = comm_queue.get() + f_recv, = comm_queue.get() child_can_read.clear() - f.array[:] = result[0] + f.array[:] = f_recv return f From 64b8a3dd9153de6d7536bdb0738bd3a91faaa4ea Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 10:42:31 -0500 Subject: [PATCH 041/644] need to fix parent hanging --- libensemble/gen_funcs/persistent_aposmm.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 3858670a3..73d7fe68a 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -251,7 +251,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): data_to_give_processes = (f_x_recv, grad_f_x_recv) else: assert(fields_to_pass == ['f']) - (x_recv, f_x_recv, sim_id_recv), = calc_in + if len(calc_in[0]) == 3: + (x_recv, f_x_recv, sim_id_recv), = calc_in + else: + # FIXME: *BIG BIG!*, even if we are passing grad_f even + # though we are not using it. + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, ) print(23*"-", "Received f({})".format(x_recv), 24*"-", @@ -318,6 +323,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x_on_cube'], + local_H[ind]['f'], child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) @@ -391,6 +397,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if len([p for p in processes if p.is_alive()]) < max_active_runs: p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x_on_cube'], + local_H[ind]['f'], child_can_read_evts[-1], parent_can_read_from_queue)) processes.append(p) @@ -591,11 +598,14 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): - assert isinstance(x0, np.array) + assert isinstance(x0, np.ndarray) tao_comm = MPI.COMM_SELF n, = x0.shape - m, = f0.shape + if f0.shape == (): + m = 1 + else: + m, = f0.shape # Create starting point, bounds, and tao object x = PETSc.Vec().create(tao_comm) From ef5c1bc01365ae2b4803076eb294ab77e1c7a2ea Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 11:14:32 -0500 Subject: [PATCH 042/644] pounders is running --- libensemble/gen_funcs/persistent_aposmm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 73d7fe68a..d54c0c5e2 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -408,7 +408,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue.wait() print('[Parent]: Done waiting.', flush=True) + print('[Parent]: Starting with assertion checking', flush=True) assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) + print('[Parent]: Assertion holds.', flush=True) comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() @@ -584,7 +586,8 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, # {{{ TAO routines for local opt def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): - comm_queue.put(x) + comm_queue.put(x.array) + print('[Child]: I just put x_on_cube =', x.array, flush=True) print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() print('[Child]: I have started waiting', flush=True) From c3cf881d562a243ac6952d96af76f387844f7696 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 11:45:22 -0500 Subject: [PATCH 043/644] removes the checking of receiving duplicate value as it is being checked at the allocation stage --- libensemble/gen_funcs/persistent_aposmm.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index d54c0c5e2..01dcfcaeb 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -197,8 +197,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) - received_values = set() - sim_id_to_child_indices = {} child_id_to_run_id = {} run_order = {} @@ -261,15 +259,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): print(23*"-", "Received f({})".format(x_recv), 24*"-", flush=True) - if tuple(x_recv) in received_values: - #FIXME:: This should never be the case. - print("DANGER!! Re-received a value(SimID = {}) ignoring it for now, but" - " need to fix it in manager, I guess?".format(sim_id_recv), flush=True) - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) - continue - - received_values.add(tuple(x_recv)) if tag in [STOP_TAG, PERSIS_STOP]: @@ -336,6 +325,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) + print('[Parent]: Assertion holds.', flush=True) + + comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() From a6daa563eea6a9f6350e35b8a1aed55e0b485355 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 17:19:46 -0500 Subject: [PATCH 044/644] adds the test file for pounders --- .../test_6-hump_camel_persistent_aposmm.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index cafb39620..03dc303ed 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -34,19 +34,24 @@ 'in': ['x'], 'out': [('f', float), ('grad', float, n)]} -gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int)] +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), + ('local_min', bool)] gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, 'batch_mode': True, 'initial_sample_size': 50, - 'localopt_method': 'LD_MMA', + 'localopt_method': 'pounders', 'xtol_rel': 1e-3, - 'max_active_runs': 1, + 'local_min': True, + 'dist_to_bound_multiple': 0.5, + 'grtol': 1e-4, + 'gatol': 1e-4, + 'tol': 1e-5, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back',bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} persis_info = per_worker_stream({}, nworkers + 1) @@ -57,4 +62,5 @@ alloc_specs, libE_specs) if is_master: - save_libE_output(H, persis_info, __file__, nworkers) + print('[Manager]:', H[np.where(H['local_min'])]['x']) + # save_libE_output(H, persis_info, __file__, nworkers) From b0332e24ca42d8f8f0ba8f7be53a39f496fa035e Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 21:58:10 -0500 Subject: [PATCH 045/644] handles a major bug in max_active runs --- libensemble/gen_funcs/persistent_aposmm.py | 23 ++++++++++++------- .../test_6-hump_camel_persistent_aposmm.py | 12 +++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 01dcfcaeb..2e3fb1c7d 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -229,7 +229,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # }}} - for _ in range(gen_specs['initial_sample_size']): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) @@ -257,7 +256,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, ) - print(23*"-", "Received f({})".format(x_recv), 24*"-", + print(23*"-", "Received f({})(SimID: {})".format(x_recv, + sim_id_recv), 24*"-", flush=True) if tag in [STOP_TAG, PERSIS_STOP]: @@ -308,6 +308,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): ind = waiting_starting_inds[0] waiting_starting_inds = waiting_starting_inds[1:] + child_can_read_evts.append(Event()) + + print('[Parent]: Initing process', flush=True) + + parent_can_read_from_queue.clear() + # {{{ logic duplication(need to get rid of it) p = Process(target=run_local_opt, args=(gen_specs, @@ -327,7 +333,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): print('[Parent]: Assertion holds.', flush=True) - comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() @@ -378,20 +383,22 @@ def aposmm(H, persis_info, gen_specs, libE_info): starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) for ind in starting_inds: + local_H['started_run'][ind] = 1 - child_can_read_evts.append(Event()) + if len([p for p in processes if p.is_alive()]) < max_active_runs: - print('[Parent]: Initing process', flush=True) + child_can_read_evts.append(Event()) - parent_can_read_from_queue.clear() + print('[Parent]: Initing process', flush=True) + + parent_can_read_from_queue.clear() - if len([p for p in processes if p.is_alive()]) < max_active_runs: p = Process(target=run_local_opt, args=(gen_specs, comm_queue, local_H[ind]['x_on_cube'], local_H[ind]['f'], child_can_read_evts[-1], - parent_can_read_from_queue)) + parent_can_read_from_queue, len(processes))) processes.append(p) p.start() print('[Parent]: Started a child process', flush=True) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 03dc303ed..f42f5c9f8 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -19,10 +19,12 @@ # Import libEnsemble items for this test from libensemble.libE import libE +from math import gamma, pi, sqrt from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() @@ -40,11 +42,15 @@ 'in': [], 'out': gen_out, 'batch_mode': True, - 'initial_sample_size': 50, - 'localopt_method': 'pounders', + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'LD_MMA', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'xtol_rel': 1e-3, + 'num_active_gens': 1, 'local_min': True, 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, 'grtol': 1e-4, 'gatol': 1e-4, 'tol': 1e-5, @@ -55,7 +61,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 400} +exit_criteria = {'sim_max': 1000} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From d378f8d8485fc3eb43a5184a9751a7fff5aec0f6 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 29 Jul 2019 22:07:22 -0500 Subject: [PATCH 046/644] cleans up extra arg to the local opt --- libensemble/gen_funcs/persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 2e3fb1c7d..b3f115595 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -398,7 +398,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue, local_H[ind]['x_on_cube'], local_H[ind]['f'], child_can_read_evts[-1], - parent_can_read_from_queue, len(processes))) + parent_can_read_from_queue)) processes.append(p) p.start() print('[Parent]: Started a child process', flush=True) From 091ee73fdad7c53b739d6b9909dd8e2e84257241 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 30 Jul 2019 09:03:11 -0500 Subject: [PATCH 047/644] comments debug statements --- libensemble/gen_funcs/persistent_aposmm.py | 101 +++++++++--------- .../test_6-hump_camel_persistent_aposmm.py | 5 + 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index b3f115595..7262688ee 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -186,6 +186,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ + np.random.seed(10) + + # {{{ multiprocessing initialization child_can_read_evts = [] @@ -256,13 +259,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, ) - print(23*"-", "Received f({})(SimID: {})".format(x_recv, - sim_id_recv), 24*"-", - flush=True) + # print(23*"-", "Received f({})(SimID: {})".format(x_recv, + # sim_id_recv), 24*"-", + # flush=True) if tag in [STOP_TAG, PERSIS_STOP]: - print('[Parent]: Received a stop tag, killing all children.', flush=True) + # print('[Parent]: Received a stop tag, killing all children.', flush=True) # FIXME: If there is any indication that all child processes are # done with their work, we can do this a bit more cleanly i.e. # first join and check if they are still alive, only then kill. @@ -277,19 +280,19 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: - print('[Parent]:Giving f = {} it to the' - ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) + # print('[Parent]:Giving f = {} it to the' + # ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) comm_queue.put(data_to_give_processes) parent_can_read_from_queue.clear() child_can_read_evts[child_idx].set() - print('[Parent]: I am waiting for child idx {}'.format(child_idx), flush=True) + # print('[Parent]: I am waiting for child idx {}'.format(child_idx), flush=True) parent_can_read_from_queue.wait() - print('[Parent]: Wohoo I am free again', flush=True) + # print('[Parent]: Wohoo I am free again', flush=True) x_new = comm_queue.get() - print('[Parent]: The child returned {}.'.format(x_new), flush=True) + # print('[Parent]: The child returned {}.'.format(x_new), flush=True) if isinstance(x_new, tuple): # FIXME: [KK]: This is ugly and every bit of it happens in # my mind, this should be cleaned up duing the cleanup @@ -298,19 +301,19 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Need to figure out what are run_ids. update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) - print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) + # print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) processes[child_idx].join() - print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) + # print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) if waiting_starting_inds: - print('[Parent]: There was a waiting starting index' - ' and started a run from there.', flush=True) + # print('[Parent]: There was a waiting starting index' + # ' and started a run from there.', flush=True) ind = waiting_starting_inds[0] waiting_starting_inds = waiting_starting_inds[1:] child_can_read_evts.append(Event()) - print('[Parent]: Initing process', flush=True) + # print('[Parent]: Initing process', flush=True) parent_can_read_from_queue.clear() @@ -323,22 +326,22 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue)) processes.append(p) p.start() - print('[Parent]: Started a child process', flush=True) + # print('[Parent]: Started a child process', flush=True) - print('[Parent]: I am waiting', flush=True) + # print('[Parent]: I am waiting', flush=True) parent_can_read_from_queue.wait() - print('[Parent]: Done waiting.', flush=True) + # print('[Parent]: Done waiting.', flush=True) assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - print('[Parent]: Assertion holds.', flush=True) + # print('[Parent]: Assertion holds.', flush=True) comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() child_can_read_evts[-1].set() parent_can_read_from_queue.wait() - print('[Master]: Getting a new x_new.', flush=True) + # print('[Master]: Getting a new x_new.', flush=True) x_new = comm_queue.get() @@ -374,8 +377,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) else: - print('[Parent]: Did not find any local opt run to associate with.', - flush=True) + # print('[Parent]: Did not find any local opt run to associate with.', + # flush=True) if n_s < gen_specs['initial_sample_size']: continue n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -390,7 +393,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_can_read_evts.append(Event()) - print('[Parent]: Initing process', flush=True) + # print('[Parent]: Initing process', flush=True) parent_can_read_from_queue.clear() @@ -401,22 +404,22 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue)) processes.append(p) p.start() - print('[Parent]: Started a child process', flush=True) + # print('[Parent]: Started a child process', flush=True) - print('[Parent]: I am waiting', flush=True) + # print('[Parent]: I am waiting', flush=True) parent_can_read_from_queue.wait() - print('[Parent]: Done waiting.', flush=True) + # print('[Parent]: Done waiting.', flush=True) - print('[Parent]: Starting with assertion checking', flush=True) + # print('[Parent]: Starting with assertion checking', flush=True) assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - print('[Parent]: Assertion holds.', flush=True) + # print('[Parent]: Assertion holds.', flush=True) comm_queue.put(local_H[ind][[*fields_to_pass]]) parent_can_read_from_queue.clear() child_can_read_evts[-1].set() parent_can_read_from_queue.wait() - print('[Master]: Getting a new x_new.', flush=True) + # print('[Master]: Getting a new x_new.', flush=True) x_new = comm_queue.get() @@ -437,13 +440,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: - print('[Parent]: Not exceeding max active runs, putting it' - ' in our stash', flush=True) + # print('[Parent]: Not exceeding max active runs, putting it' + # ' in our stash', flush=True) waiting_starting_inds.append(ind) if len(starting_inds) == 0 and len(waiting_starting_inds) == 0: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) - print('[Parent]: Heading to next iteration', flush=True) + # print('[Parent]: Heading to next iteration', flush=True) print('[Parent]: The optimal points are:\n', local_H[np.where(local_H['local_min'])]['x'], flush=True) @@ -451,7 +454,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue.close() comm_queue.join_thread() - print('Done with everything', flush=True) + # print('Done with everything', flush=True) for p in processes: if p.is_alive(): @@ -467,11 +470,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) - print('[Child]: Parent should no longer wait.', flush=True) + # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() - print('[Child]: I have started waiting', flush=True) + # print('[Child]: I have started waiting', flush=True) child_can_read.wait() - print('[Child]: Wohooo.. I am free folks', flush=True) + # print('[Child]: Wohooo.. I am free folks', flush=True) if gen_specs['localopt_method'] in ['LD_MMA']: f_recv, grad_recv = comm_queue.get() grad[:] = grad_recv @@ -487,7 +490,7 @@ def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): - print('[Child]: Started local opt at {}.'.format(x0), flush=True) + # print('[Child]: Started local opt at {}.'.format(x0), flush=True) n = len(gen_specs['ub']) opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) @@ -522,9 +525,9 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, opt.set_ftol_abs(gen_specs['ftol_abs']) #FIXME: Do we need to do something of the final 'x_opt'? - print('[Child]: Started my optimization', flush=True) + # print('[Child]: Started my optimization', flush=True) x_opt = opt.optimize(x0) - print('[Child]: I have converged.', flush=True) + # print('[Child]: I have converged.', flush=True) comm_queue.put(('Converged', x_opt)) parent_can_read.set() @@ -535,11 +538,11 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, def scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) - print('[Child]: Parent should no longer wait.', flush=True) + # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() - print('[Child]: I have started waiting', flush=True) + # print('[Child]: I have started waiting', flush=True) child_can_read.wait() - print('[Child]: Wohooo.. I am free folks', flush=True) + # print('[Child]: Wohooo.. I am free folks', flush=True) f_x_recv, = comm_queue.get() child_can_read.clear() return f_x_recv @@ -559,7 +562,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, cons.append(up) method = gen_specs['localopt_method'][6:] - print('[Child]: Started my optimization', flush=True) + # print('[Child]: Started my optimization', flush=True) res = scipy_optimize.minimize(lambda x: scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_specs), x0, method=method, options={'maxiter': 100, 'tol': gen_specs['tol']}) @@ -575,7 +578,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, # FIXME: Need to do something with the exit codes. - print('[Child]: I have converged.', flush=True) + # print('[Child]: I have converged.', flush=True) comm_queue.put(('Converged', x_opt)) parent_can_read.set() @@ -586,12 +589,12 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x.array) - print('[Child]: I just put x_on_cube =', x.array, flush=True) - print('[Child]: Parent should no longer wait.', flush=True) + # print('[Child]: I just put x_on_cube =', x.array, flush=True) + # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() - print('[Child]: I have started waiting', flush=True) + # print('[Child]: I have started waiting', flush=True) child_can_read.wait() - print('[Child]: Wohooo.. I am free folks', flush=True) + # print('[Child]: Wohooo.. I am free folks', flush=True) f_recv, = comm_queue.get() child_can_read.clear() f.array[:] = f_recv @@ -644,7 +647,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) tao.setInitial(x) - print('[Child]: Started my optimization', flush=True) + # print('[Child]: Started my optimization', flush=True) tao.solve(x) x_opt = tao.getSolution().getArray() @@ -664,7 +667,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, tao.destroy() #FIXME: Do we need to do something of the final 'x_opt'? - print('[Child]: I have converged.', flush=True) + # print('[Child]: I have converged.', flush=True) comm_queue.put(('Converged', x_opt)) parent_can_read.set() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index f42f5c9f8..606080994 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -25,9 +25,13 @@ from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima +from time import time nworkers, is_master, libE_specs, _ = parse_args() +if is_master: + start_time = time() + if nworkers < 2: sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") @@ -69,4 +73,5 @@ if is_master: print('[Manager]:', H[np.where(H['local_min'])]['x']) + print('[Manager]: Time taken =', time() - start_time, flush=True) # save_libE_output(H, persis_info, __file__, nworkers) From 30e6f487f4c97b509ed7f06170bd8987fcfcc31a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 30 Jul 2019 09:03:59 -0500 Subject: [PATCH 048/644] some more commenting statements --- libensemble/alloc_funcs/support.py | 4 ++-- libensemble/gen_funcs/support.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/alloc_funcs/support.py b/libensemble/alloc_funcs/support.py index 56ebe43aa..49f7907d4 100644 --- a/libensemble/alloc_funcs/support.py +++ b/libensemble/alloc_funcs/support.py @@ -25,7 +25,7 @@ def count_persis_gens(W): def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add sim work record to work array." - print('[Allocator]: Calling sim_work for {}.'.format(H_rows), flush=True) + # print('[Allocator]: Calling sim_work for {}.'.format(H_rows), flush=True) libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, @@ -35,7 +35,7 @@ def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): def gen_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add gen work record to work array." - print('[Allocator]: Calling gen_work for {}.'.format(H_rows), flush=True) + # print('[Allocator]: Calling gen_work for {}.'.format(H_rows), flush=True) libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index faa04199f..696ccf5d7 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -13,8 +13,8 @@ def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ assert len(O) == 1 - print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", - flush=True) + # print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", + # flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, @@ -26,14 +26,14 @@ def send_mgr_worker_msg(comm, O): def get_mgr_worker_msg(comm, status=None): """Get message to worker from manager. """ - print('[Parent]: Expecting a message from Manager...', flush=True) + # print('[Parent]: Expecting a message from Manager...', flush=True) tag, Work = comm.recv() - print('[Parent]: Received a tag {}'.format(tag), flush=True) + # print('[Parent]: Received a tag {}'.format(tag), flush=True) sys.stdout.flush() if tag in [STOP_TAG, PERSIS_STOP]: comm.push_back(tag, Work) return tag, None, None - print('[Parent]: Expecting a message from Manager...', flush=True) + # print('[Parent]: Expecting a message from Manager...', flush=True) _, calc_in = comm.recv() - print('[Parent]: Received some message.', flush=True) + # print('[Parent]: Received some message.', flush=True) return tag, Work, calc_in From 90c65e33983905fdef6f9f1f286a705e7b4a1796 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 11:03:54 -0500 Subject: [PATCH 049/644] some cleanup --- libensemble/gen_funcs/persistent_aposmm.py | 277 ++++++++++----------- 1 file changed, 125 insertions(+), 152 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 7262688ee..6e471f827 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -15,7 +15,6 @@ from scipy import optimize as scipy_optimize from mpi4py import MPI -from petsc4py import PETSc from numpy.lib.recfunctions import merge_arrays @@ -35,6 +34,14 @@ class APOSMMException(Exception): "Raised for any exception in APOSMM" +class ConvergedMsg(object): + """ + Message communicated when a local optimization is converged. + """ + def __init__(self, x): + self.x = x + + def aposmm(H, persis_info, gen_specs, libE_info): """ APOSMM coordinates multiple local optimization runs, starting from points @@ -186,13 +193,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - np.random.seed(10) - - # {{{ multiprocessing initialization - child_can_read_evts = [] - processes = [] + local_opters = [] parent_can_read_from_queue = Event() comm_queue = Queue() @@ -239,10 +242,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag = None max_active_runs = gen_specs.get('max_active_runs', np.inf) waiting_starting_inds = [] - O = np.empty(0, dtype=gen_specs['out']) # noqa: E741 + O = np.empty(0, dtype=gen_specs['out']) while 1: - - # Receive from manager tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: @@ -259,21 +260,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, ) - # print(23*"-", "Received f({})(SimID: {})".format(x_recv, - # sim_id_recv), 24*"-", - # flush=True) - if tag in [STOP_TAG, PERSIS_STOP]: - - # print('[Parent]: Received a stop tag, killing all children.', flush=True) - # FIXME: If there is any indication that all child processes are - # done with their work, we can do this a bit more cleanly i.e. - # first join and check if they are still alive, only then kill. - - # sending SIGKILL to the processes. - # TODO: should we use SIGTERM instead? - for p in processes: - p.kill() + # FIXME: This has to be a clean exit. + for p in local_opters: + p.terminate() break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -282,87 +272,35 @@ def aposmm(H, persis_info, gen_specs, libE_info): for child_idx in sim_id_to_child_indices[sim_id_recv]: # print('[Parent]:Giving f = {} it to the' # ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) - comm_queue.put(data_to_give_processes) - - parent_can_read_from_queue.clear() - - child_can_read_evts[child_idx].set() - # print('[Parent]: I am waiting for child idx {}'.format(child_idx), flush=True) - parent_can_read_from_queue.wait() - # print('[Parent]: Wohoo I am free again', flush=True) - - x_new = comm_queue.get() + x_new = local_opters.iterate(*data_to_give_processes) # print('[Parent]: The child returned {}.'.format(x_new), flush=True) - if isinstance(x_new, tuple): - # FIXME: [KK]: This is ugly and every bit of it happens in - # my mind, this should be cleaned up duing the cleanup - # phase. - x_opt = x_new[1] - # Need to figure out what are run_ids. + if isinstance(x_new, ConvergedMsg): + x_opt = x_new.x update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) - # print("[Parent]: Child {} has converged, miss you :'(".format(child_idx), flush=True) - processes[child_idx].join() - # print("[Parent]: Child {} has been joined.".format(child_idx), flush=True) if waiting_starting_inds: - # print('[Parent]: There was a waiting starting index' - # ' and started a run from there.', flush=True) ind = waiting_starting_inds[0] waiting_starting_inds = waiting_starting_inds[1:] - - child_can_read_evts.append(Event()) - - # print('[Parent]: Initing process', flush=True) - - parent_can_read_from_queue.clear() - - # {{{ logic duplication(need to get rid of it) - - p = Process(target=run_local_opt, args=(gen_specs, - comm_queue, local_H[ind]['x_on_cube'], - local_H[ind]['f'], - child_can_read_evts[-1], - parent_can_read_from_queue)) - processes.append(p) - p.start() - # print('[Parent]: Started a child process', flush=True) - - # print('[Parent]: I am waiting', flush=True) - parent_can_read_from_queue.wait() - # print('[Parent]: Done waiting.', flush=True) - - assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - - # print('[Parent]: Assertion holds.', flush=True) - - comm_queue.put(local_H[ind][[*fields_to_pass]]) - parent_can_read_from_queue.clear() - - child_can_read_evts[-1].set() - parent_can_read_from_queue.wait() - # print('[Master]: Getting a new x_new.', flush=True) - - x_new = comm_queue.get() - + local_opter = LocalOptInterfacer(gen_specs, + local_H[ind]['x_on_cube'], + parent_can_read_from_queue, comm_queue, + local_H[ind]['f']) + local_opters.append(local_opter) + x_new = local_opter.iterate(local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) - run_order[total_runs] = [local_H[-1]['sim_id']] - child_id_to_run_id[len(processes)-1] = total_runs + child_id_to_run_id[len(local_opters)-1] = total_runs total_runs += 1 - # Need to figure out the child number from the run number. if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - - # }}} else: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) @@ -370,15 +308,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: - # print('[Parent]: Did not find any local opt run to associate with.', - # flush=True) if n_s < gen_specs['initial_sample_size']: continue n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -389,64 +325,33 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 - if len([p for p in processes if p.is_alive()]) < max_active_runs: - - child_can_read_evts.append(Event()) - - # print('[Parent]: Initing process', flush=True) - - parent_can_read_from_queue.clear() - - p = Process(target=run_local_opt, args=(gen_specs, - comm_queue, local_H[ind]['x_on_cube'], - local_H[ind]['f'], - child_can_read_evts[-1], - parent_can_read_from_queue)) - processes.append(p) - p.start() - # print('[Parent]: Started a child process', flush=True) - - # print('[Parent]: I am waiting', flush=True) - parent_can_read_from_queue.wait() - # print('[Parent]: Done waiting.', flush=True) - - # print('[Parent]: Starting with assertion checking', flush=True) - assert np.allclose(comm_queue.get(), local_H[ind]['x_on_cube']) - # print('[Parent]: Assertion holds.', flush=True) - - comm_queue.put(local_H[ind][[*fields_to_pass]]) - parent_can_read_from_queue.clear() - - child_can_read_evts[-1].set() - parent_can_read_from_queue.wait() - # print('[Master]: Getting a new x_new.', flush=True) - - x_new = comm_queue.get() + if len([p for p in local_opters if p.is_running]) < max_active_runs: + local_opter = LocalOptInterfacer(gen_specs, + local_H[ind]['x_on_cube'], + parent_can_read_from_queue, comm_queue, + local_H[ind]['f']) + local_opters.append(local_opter) + x_new = local_opter.iterate(local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) - run_order[total_runs] = [local_H[-1]['sim_id']] - child_id_to_run_id[len(processes)-1] = total_runs + child_id_to_run_id[len(local_opters)-1] = total_runs total_runs += 1 - # Need to figure out the child number from the run number. if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(processes)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(processes)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: - # print('[Parent]: Not exceeding max active runs, putting it' - # ' in our stash', flush=True) waiting_starting_inds.append(ind) + if len(starting_inds) == 0 and len(waiting_starting_inds) == 0: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) - # print('[Parent]: Heading to next iteration', flush=True) print('[Parent]: The optimal points are:\n', local_H[np.where(local_H['local_min'])]['x'], flush=True) @@ -454,19 +359,98 @@ def aposmm(H, persis_info, gen_specs, libE_info): comm_queue.close() comm_queue.join_thread() - # print('Done with everything', flush=True) - - for p in processes: - if p.is_alive(): + for local_opter in local_opters: + if local_opter.is_running: raise RuntimeError("[Parent]: Atleast one child process is still active, even after" " killing all the children.") - #TODO: Figure out what do we need to do from here. - return local_H, persis_info, tag -# {{{ NLOPT for local opt.. +class LocalOptInterfacer(object): + def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, + grad0=None): + """ + :param x0: A numpy array of the initial guess solution. This guess + should be scaled to a unit cube. + :param f0: A numpy array of the initial function value. + + + .. warning:: In order to have correct functioning of the local + optimization child processes. ~self.iterate~ should be called + immediately after creating the class. + + """ + self.parent_can_read = parent_can_read + + self.comm_queue = comm_queue + self.child_can_read = Event() + + # {{{ setting the local optimization method + + if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', + 'LN_NELDERMEAD', 'LD_MMA']: + run_local_opt = run_local_nlopt + elif gen_specs['localopt_method'] in ['pounders']: + run_local_opt = run_local_tao + elif gen_specs['localopt_method'] in ['scipy_COBYLA']: + run_local_opt = run_local_scipy_opt + else: + raise NotImplementedError("Unknown local optimization method " + "'{}'.".format(gen_specs['localopt_method'])) + + # }}} + + self.parent_can_read_from_queue.clear() + self.process = Process(target=run_local_opt, args=(gen_specs, + comm_queue, + x0, f0, + self.child_can_read, + parent_can_read)) + + self.process.start() + self.is_running = True + self.parent_can_read.wait() + assert np.allclose(comm_queue.get(), x0) + + def iterate(self, f, grad=None): + """ + Returns an instance of either :class:`numpy.ndarray` corresponding to the next + iterative guess or :class:`ConvergedMsg` when the solver is converged. + + :param f: A numpy array of the function evaluation. + :param grad: A numpy array of the function's gradient. + """ + if grad is not None: + self.comm_queue.put((f, grad)) + else: + self.comm_queue.put((f, )) + + self.parent_can_read.clear() + self.child_can_read.set() + self.parent_can_read.wait() + + x_new = self.comm_queue.get() + if isinstance(x_new, ConvergedMsg): + self.is_running = False + self.process.join() + + return x_new + + def clean_exit(self): + self.comm_queue.put((np.nan*np.ones_like(self.f0), )) + self.parent_can_read.clear() + self.child_can_read.set() + self.parent_can_read.wait() + + x_new = self.comm_queue.get() + assert isinstance(x_new, ConvergedMsg) + self.is_running = False + self.process.join() + + + +# {{{ NLOPT for local opt def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) @@ -603,6 +587,8 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): + + from petsc4py import PETSc assert isinstance(x0, np.ndarray) tao_comm = MPI.COMM_SELF @@ -1339,19 +1325,6 @@ def send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) -def send_initial_sample(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices): - 1/0 - sampled_points = persis_info['rand_stream'].uniform(0, 1, (gen_specs['initial_sample_size'], n)) - add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) - for sim_id in local_H['sim_id'][-gen_specs['initial_sample_size']:]: - assert sim_id not in sim_id_to_child_indices - sim_id_to_child_indices[sim_id] = None - - for i in range(len(local_H)): - send_mgr_worker_msg(comm, local_H[i:i+1][['x', 'sim_id']]) - - def display_exception(e): print(e.__doc__) print(e.args) From ff89687fd4e0bbe0f0bdcd30d522f3441d030e64 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 2 Aug 2019 11:24:58 -0500 Subject: [PATCH 050/644] Adding blmvm support for PETSc/TAO --- libensemble/gen_funcs/aposmm.py | 60 ++++++++++--------- .../test_6-hump_camel_aposmm_LD_MMA.py | 11 +++- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 5b22e0fa6..f48ce9070 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -506,7 +506,7 @@ def advance_local_run(H, gen_specs, c_flag, run, persis_info): exit_code = 0 display_exception(e) - elif gen_specs['localopt_method'] in ['pounders']: + elif gen_specs['localopt_method'] in ['pounders','blmvm']: if c_flag: Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, gen_specs['components'])]) @@ -515,7 +515,10 @@ def advance_local_run(H, gen_specs, c_flag, run, persis_info): Run_H_F['fvec'][i, :] = H['f_i'][a1] Run_H = merge_arrays([H[['x_on_cube']][sorted_run_inds], Run_H_F], flatten=True) else: - Run_H = H[['x_on_cube', 'fvec']][sorted_run_inds] + if gen_specs['localopt_method'] == 'pounders': + Run_H = H[['x_on_cube', 'fvec']][sorted_run_inds] + else: + Run_H = H[['x_on_cube', 'f', 'grad']][sorted_run_inds] try: x_opt, exit_code = set_up_and_run_tao(Run_H, gen_specs) @@ -653,16 +656,15 @@ def set_up_and_run_tao(Run_H, gen_specs): """ tao_comm = MPI.COMM_SELF n = len(gen_specs['ub']) - m = len(Run_H['fvec'][0]) def pounders_obj_func(tao, X, F, Run_H): - F.array = look_in_history(X.array, Run_H, vector_return=True) + F.array = look_in_history(X.array_r, Run_H, vector_return=True) return F - # def blmvm_obj_func(tao, X, G, Run_H): - # (f, grad) = look_in_history_fd_grad(X.array, Run_H) - # G.array = grad - # return f + def blmvm_obj_func(tao, X, G, Run_H): + (f, grad) = look_in_history(X.array_r, Run_H) + G.array = grad + return f # Create starting point, bounds, and tao object x = PETSc.Vec().create(tao_comm) @@ -676,24 +678,26 @@ def pounders_obj_func(tao, X, F, Run_H): tao = PETSc.TAO().create(tao_comm) tao.setType(gen_specs['localopt_method']) - # if gen_specs['localopt_method'] == 'pounders': - f = PETSc.Vec().create(tao_comm) - f.setSizes(m) - f.setFromOptions() + if gen_specs['localopt_method'] == 'pounders': + f = PETSc.Vec().create(tao_comm) + f.setSizes(len(Run_H['fvec'][0])) + f.setFromOptions() - delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) - PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) - # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') - if hasattr(tao, 'setResidual'): - tao.setResidual(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) - else: - tao.setSeparableObjective(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) - # elif gen_specs['localopt_method'] == 'blmvm': - # g = PETSc.Vec().create(tao_comm) - # g.setSizes(n) - # g.setFromOptions() - # tao.setObjectiveGradient(lambda tao, x, g: blmvm_obj_func(tao, x, g, Run_H)) + PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) + + # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') + if hasattr(tao, 'setResidual'): + tao.setResidual(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) + else: + tao.setSeparableObjective(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) + + elif gen_specs['localopt_method'] == 'blmvm': + g = PETSc.Vec().create(tao_comm) + g.setSizes(n) + g.setFromOptions() + tao.setObjectiveGradient(lambda tao, x, g: blmvm_obj_func(tao, x, g, Run_H)) # Set everything for tao before solving PETSc.Options().setValue('-tao_max_funcs', str(len(Run_H)+1)) @@ -712,10 +716,10 @@ def pounders_obj_func(tao, X, F, Run_H): # print(tao.view()) # print(x_opt) - # if gen_specs['localopt_method'] == 'pounders': - f.destroy() - # elif gen_specs['localopt_method'] == 'blmvm': - # g.destroy() + if gen_specs['localopt_method'] == 'pounders': + f.destroy() + if gen_specs['localopt_method'] == 'blmvm': + g.destroy() lb.destroy() ub.destroy() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 829b48fbf..4a88c1473 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -72,11 +72,18 @@ def libE_mpi_abort(): quit() # Perform the run -for run in range(2): +for run in range(3): if libE_specs['comms'] == 'tcp' and is_master: libE_specs['worker_cmd'].append(str(run)) if run == 1: + gen_specs['localopt_method'] = 'blmvm' + gen_specs['grtol'] = 1e-5 + gen_specs['gatol'] = 1e-5 + persis_info = deepcopy(persis_info_safe) + + if run == 2: + gen_specs['localopt_method'] = 'LD_MMA' # Change the bounds to put a local min at a corner point (to test that # APOSMM handles the same point being in multiple runs) ability to # give back a previously evaluated point) @@ -94,7 +101,7 @@ def libE_mpi_abort(): exit_criteria = {'sim_max': 200, 'elapsed_wallclock_time': 300} minima = np.array([[-2.9, -1.9]]) - persis_info = persis_info_safe + persis_info = deepcopy(persis_info_safe) H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) From 70f09d298f23f587f11e85f62b33f83af5eb921d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 2 Aug 2019 11:37:28 -0500 Subject: [PATCH 051/644] flake8 --- libensemble/gen_funcs/aposmm.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index f48ce9070..806bc1447 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -506,7 +506,7 @@ def advance_local_run(H, gen_specs, c_flag, run, persis_info): exit_code = 0 display_exception(e) - elif gen_specs['localopt_method'] in ['pounders','blmvm']: + elif gen_specs['localopt_method'] in ['pounders', 'blmvm']: if c_flag: Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, gen_specs['components'])]) @@ -686,7 +686,7 @@ def blmvm_obj_func(tao, X, G, Run_H): delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) - + # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') if hasattr(tao, 'setResidual'): tao.setResidual(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) diff --git a/setup.py b/setup.py index e8df45a9b..a5327c053 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def run_tests(self): 'mock' ], - extras_require = { + extras_require={ 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py']}, classifiers=[ From 5f574fa05c4b4e628a81012e2f22abd8ef22d426 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 13:01:54 -0500 Subject: [PATCH 052/644] minor fix to pass both f and grad_f to the local optimization --- libensemble/gen_funcs/persistent_aposmm.py | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 6e471f827..fa9c711c9 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -263,7 +263,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: This has to be a clean exit. for p in local_opters: - p.terminate() + p.destroy() break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -287,7 +287,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue, comm_queue, local_H[ind]['f']) local_opters.append(local_opter) - x_new = local_opter.iterate(local_H[ind][fields_to_pass]) + x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) @@ -331,7 +331,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): parent_can_read_from_queue, comm_queue, local_H[ind]['f']) local_opters.append(local_opter) - x_new = local_opter.iterate(local_H[ind][fields_to_pass]) + x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) @@ -401,7 +401,7 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, # }}} - self.parent_can_read_from_queue.clear() + self.parent_can_read.clear() self.process = Process(target=run_local_opt, args=(gen_specs, comm_queue, x0, f0, @@ -421,12 +421,12 @@ def iterate(self, f, grad=None): :param f: A numpy array of the function evaluation. :param grad: A numpy array of the function's gradient. """ - if grad is not None: - self.comm_queue.put((f, grad)) - else: + self.parent_can_read.clear() + if grad is None: self.comm_queue.put((f, )) + else: + self.comm_queue.put((f, grad)) - self.parent_can_read.clear() self.child_can_read.set() self.parent_can_read.wait() @@ -437,7 +437,10 @@ def iterate(self, f, grad=None): return x_new - def clean_exit(self): + def destroy(self): + # FIXME: This isn't a clean exit yet, need to handle it. + self.process.terminate() + return self.comm_queue.put((np.nan*np.ones_like(self.f0), )) self.parent_can_read.clear() self.child_can_read.set() @@ -512,7 +515,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, # print('[Child]: Started my optimization', flush=True) x_opt = opt.optimize(x0) # print('[Child]: I have converged.', flush=True) - comm_queue.put(('Converged', x_opt)) + comm_queue.put(ConvergedMsg(x_opt)) parent_can_read.set() # }}} @@ -654,7 +657,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, #FIXME: Do we need to do something of the final 'x_opt'? # print('[Child]: I have converged.', flush=True) - comm_queue.put(('Converged', x_opt)) + comm_queue.put(ConvergedMsg(x_opt)) parent_can_read.set() # FIXME: What do we do about the exit_code? From 09c6e6a4aa0dc31ad96adc04200c773758139ae7 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 13:07:28 -0500 Subject: [PATCH 053/644] some more progress on cleaning up the code --- libensemble/gen_funcs/persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index fa9c711c9..268c426dc 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -272,7 +272,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): for child_idx in sim_id_to_child_indices[sim_id_recv]: # print('[Parent]:Giving f = {} it to the' # ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) - x_new = local_opters.iterate(*data_to_give_processes) + x_new = local_opters[child_idx].iterate(*data_to_give_processes) # print('[Parent]: The child returned {}.'.format(x_new), flush=True) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x From 1e43a4e1f745552827885f2b9a50fd8798033b17 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 14:31:35 -0500 Subject: [PATCH 054/644] preliminary work on clean exit --- libensemble/gen_funcs/persistent_aposmm.py | 71 +++++++++++----------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 268c426dc..c8eab82e6 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -207,21 +207,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): child_id_to_run_id = {} run_order = {} - # {{{ setting the local optimization method - - if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'LD_MMA']: - run_local_opt = run_local_nlopt - elif gen_specs['localopt_method'] in ['pounders']: - run_local_opt = run_local_tao - elif gen_specs['localopt_method'] in ['scipy_COBYLA']: - run_local_opt = run_local_scipy_opt - else: - raise NotImplementedError("Unknown local optimization method " - "'{}'.".format(gen_specs['localopt_method'])) - - # }}} - # {{{ setting the data needed by the local optimization method if gen_specs['localopt_method'] in ['LD_MMA']: @@ -247,6 +232,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: + grad_f_x_recv = None if fields_to_pass == ['f', 'grad']: (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, grad_f_x_recv) @@ -262,18 +248,20 @@ def aposmm(H, persis_info, gen_specs, libE_info): if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: This has to be a clean exit. + + print('[Parent]: The optimal points are:\n', + local_H[np.where(local_H['local_min'])]['x'], flush=True) + for p in local_opters: - p.destroy() + if p.is_running: + p.destroy() break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: - # print('[Parent]:Giving f = {} it to the' - # ' child_idx: {}.'.format(f_x_recv, child_idx), flush=True) x_new = local_opters[child_idx].iterate(*data_to_give_processes) - # print('[Parent]: The child returned {}.'.format(x_new), flush=True) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x update_history_optimal(x_opt, local_H, @@ -282,10 +270,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): if waiting_starting_inds: ind = waiting_starting_inds[0] waiting_starting_inds = waiting_starting_inds[1:] + + # {{{ initing a local opt run + local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, - local_H[ind]['f']) + local_H[ind]['f'], grad_f_x_recv) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) @@ -300,6 +291,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + # }}} + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, @@ -326,10 +319,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 if len([p for p in local_opters if p.is_running]) < max_active_runs: + + # {{{ initing a local opt run + local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, - local_H[ind]['f']) + local_H[ind]['f'], grad_f_x_recv) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) @@ -345,6 +341,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + # }}} + send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: waiting_starting_inds.append(ind) @@ -353,9 +351,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) - print('[Parent]: The optimal points are:\n', - local_H[np.where(local_H['local_min'])]['x'], flush=True) - comm_queue.close() comm_queue.join_thread() @@ -386,6 +381,10 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, self.comm_queue = comm_queue self.child_can_read = Event() + self.x0 = x0 + self.f0 = f0 + self.grad0 = grad0 + # {{{ setting the local optimization method if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', @@ -438,30 +437,30 @@ def iterate(self, f, grad=None): return x_new def destroy(self): - # FIXME: This isn't a clean exit yet, need to handle it. - self.process.terminate() - return - self.comm_queue.put((np.nan*np.ones_like(self.f0), )) - self.parent_can_read.clear() - self.child_can_read.set() - self.parent_can_read.wait() + x_new = None + while not isinstance(x_new, ConvergedMsg): + self.parent_can_read.clear() + if self.grad0 is None: + self.comm_queue.put((-np.inf*np.ones_like(self.f0),)) + else: + self.comm_queue.put((-np.inf*np.ones_like(self.f0), + np.zeros_like(self.grad0))) - x_new = self.comm_queue.get() + self.child_can_read.set() + self.parent_can_read.wait() + + x_new = self.comm_queue.get() assert isinstance(x_new, ConvergedMsg) self.is_running = False self.process.join() - # {{{ NLOPT for local opt def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) - # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() - # print('[Child]: I have started waiting', flush=True) child_can_read.wait() - # print('[Child]: Wohooo.. I am free folks', flush=True) if gen_specs['localopt_method'] in ['LD_MMA']: f_recv, grad_recv = comm_queue.get() grad[:] = grad_recv From 359f5e29188024828e3c22bc86db8092a5bd1999 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 15:21:13 -0500 Subject: [PATCH 055/644] checks clean exit for scipy and nlopt --- libensemble/gen_funcs/persistent_aposmm.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c8eab82e6..7f4dfdb2b 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -232,7 +232,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: - grad_f_x_recv = None if fields_to_pass == ['f', 'grad']: (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, grad_f_x_recv) @@ -276,7 +275,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv) + local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) @@ -325,7 +324,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv) + local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) @@ -431,8 +430,8 @@ def iterate(self, f, grad=None): x_new = self.comm_queue.get() if isinstance(x_new, ConvergedMsg): - self.is_running = False self.process.join() + self.is_running = False return x_new @@ -441,9 +440,9 @@ def destroy(self): while not isinstance(x_new, ConvergedMsg): self.parent_can_read.clear() if self.grad0 is None: - self.comm_queue.put((-np.inf*np.ones_like(self.f0),)) + self.comm_queue.put((0*np.ones_like(self.f0),)) else: - self.comm_queue.put((-np.inf*np.ones_like(self.f0), + self.comm_queue.put((0*np.ones_like(self.f0), np.zeros_like(self.grad0))) self.child_can_read.set() @@ -451,8 +450,8 @@ def destroy(self): x_new = self.comm_queue.get() assert isinstance(x_new, ConvergedMsg) - self.is_running = False self.process.join() + self.is_running = False # {{{ NLOPT for local opt @@ -565,7 +564,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, # FIXME: Need to do something with the exit codes. # print('[Child]: I have converged.', flush=True) - comm_queue.put(('Converged', x_opt)) + comm_queue.put(ConvergedMsg(x_opt)) parent_can_read.set() # }}} From c3742d1571d8bb3afe56ba029ba4d1e0b118d5f9 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 15:28:43 -0500 Subject: [PATCH 056/644] imports petsc at the top --- libensemble/gen_funcs/persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 7f4dfdb2b..b1bb17772 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -13,6 +13,7 @@ import numpy as np from scipy.spatial.distance import cdist, pdist, squareform from scipy import optimize as scipy_optimize +from petsc4py import PETSc from mpi4py import MPI @@ -589,7 +590,6 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): - from petsc4py import PETSc assert isinstance(x0, np.ndarray) tao_comm = MPI.COMM_SELF From f1d86c5a3afcda993a43b19d1a53d40480425b64 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 15:58:29 -0500 Subject: [PATCH 057/644] reverts any changes in support.py --- libensemble/gen_funcs/support.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 696ccf5d7..e5084828d 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -1,5 +1,4 @@ from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, UNSET_TAG, EVAL_GEN_TAG -import sys def sendrecv_mgr_worker_msg(comm, O, status=None): @@ -12,9 +11,6 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ - assert len(O) == 1 - # print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", - # flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, @@ -26,14 +22,9 @@ def send_mgr_worker_msg(comm, O): def get_mgr_worker_msg(comm, status=None): """Get message to worker from manager. """ - # print('[Parent]: Expecting a message from Manager...', flush=True) tag, Work = comm.recv() - # print('[Parent]: Received a tag {}'.format(tag), flush=True) - sys.stdout.flush() if tag in [STOP_TAG, PERSIS_STOP]: comm.push_back(tag, Work) return tag, None, None - # print('[Parent]: Expecting a message from Manager...', flush=True) _, calc_in = comm.recv() - # print('[Parent]: Received some message.', flush=True) return tag, Work, calc_in From 87ed78e9fb3b4d3552775033c6fb1c2355269f36 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 16:11:44 -0500 Subject: [PATCH 058/644] reverts any changes in alloc_funcs/support.py --- libensemble/alloc_funcs/support.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libensemble/alloc_funcs/support.py b/libensemble/alloc_funcs/support.py index 49f7907d4..927003ffe 100644 --- a/libensemble/alloc_funcs/support.py +++ b/libensemble/alloc_funcs/support.py @@ -25,7 +25,6 @@ def count_persis_gens(W): def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add sim work record to work array." - # print('[Allocator]: Calling sim_work for {}.'.format(H_rows), flush=True) libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, @@ -35,8 +34,6 @@ def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): def gen_work(Work, i, H_fields, H_rows, persis_info, **libE_info): "Add gen work record to work array." - # print('[Allocator]: Calling gen_work for {}.'.format(H_rows), flush=True) - libE_info['H_rows'] = H_rows Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, From 8536e30b982a2db431f1c0cb1fc74353e1f55843 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 16:13:24 -0500 Subject: [PATCH 059/644] reverts any changes in other tests --- .../test_6-hump_camel_aposmm_LD_MMA.py | 72 ++++--------------- 1 file changed, 12 insertions(+), 60 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index ddae2ce7f..3da958d54 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -16,6 +16,7 @@ import numpy as np from math import gamma, pi, sqrt from copy import deepcopy +from mpi4py import MPI # Import libEnsemble items for this test from libensemble.libE import libE, libE_tcp_worker @@ -24,8 +25,13 @@ from libensemble.alloc_funcs.fast_alloc_to_aposmm import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_1 as persis_info, aposmm_gen_out as gen_out, six_hump_camel_minima as minima +from time import time nworkers, is_master, libE_specs, _ = parse_args() +np.random.seed(MPI.COMM_WORLD.Get_rank()) + +if is_master: + start_time = time() n = 2 sim_specs = {'sim_f': sim_f, @@ -36,6 +42,7 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f', 'grad', 'returned'], 'out': gen_out, + 'num_active_gens': 1, 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), @@ -55,65 +62,10 @@ exit_criteria = {'sim_max': 1000} # Set up appropriate abort mechanism depending on comms -libE_abort = quit -if libE_specs['comms'] == 'mpi': - from mpi4py import MPI - - def libE_mpi_abort(): - MPI.COMM_WORLD.Abort(1) - - libE_abort = libE_mpi_abort - -# Perform the run (TCP worker mode) -if libE_specs['comms'] == 'tcp' and not is_master: - run = int(sys.argv[-1]) - libE_tcp_worker(sim_specs, gen_specs[run], libE_specs) - quit() - -# Perform the run -for run in range(2): - if libE_specs['comms'] == 'tcp' and is_master: - libE_specs['worker_cmd'].append(str(run)) - - if run == 1: - # Change the bounds to put a local min at a corner point (to test that - # APOSMM handles the same point being in multiple runs) ability to - # give back a previously evaluated point) - gen_specs['ub'] = np.array([-2.9, -1.9]) - gen_specs['mu'] = 1e-4 - gen_specs['rk_const'] = 0.01*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi) - gen_specs['lhs_divisions'] = 2 - # APOSMM can be called when some run is incomplete - gen_specs.pop('batch_mode') - - gen_specs.pop('xtol_rel') - gen_specs['ftol_rel'] = 1e-2 - gen_specs['xtol_abs'] = 1e-3 - gen_specs['ftol_abs'] = 1e-8 - exit_criteria = {'sim_max': 200, 'elapsed_wallclock_time': 300} - minima = np.array([[-2.9, -1.9]]) - - persis_info = persis_info_safe - - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs) - - if is_master: - if flag != 0: - print("Exit was not on convergence (code {})".format(flag)) - sys.stdout.flush() - libE_abort() - tol = 1e-5 - for m in minima: - # The minima are known on this test problem. - # 1) We use their values to test APOSMM has identified all minima - # 2) We use their approximate values to ensure APOSMM evaluates a - # point in each minima's basin of attraction. - print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1))) - sys.stdout.flush() - if np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) > tol: - libE_abort() +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, + persis_info, alloc_specs, libE_specs) - print("\nlibEnsemble with APOSMM using a gradient-based localopt method has identified the " + str(np.shape(minima)[0]) + " minima within a tolerance " + str(tol)) - save_libE_output(H, persis_info, __file__, nworkers) +if is_master: + print('[Manager]:', H[np.where(H['local_min'])]['x']) + print('[Manager]: Time taken =', time()-start_time) From 271ba78992e59b523c35be5bd8572b4cd1a0ac6a Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 16:16:54 -0500 Subject: [PATCH 060/644] re-adds 2 runs in test_6humpcamel_aposmm_LD_MMA.py --- .../test_6-hump_camel_aposmm_LD_MMA.py | 71 ++++++++++++++++--- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 3da958d54..829b48fbf 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -16,7 +16,6 @@ import numpy as np from math import gamma, pi, sqrt from copy import deepcopy -from mpi4py import MPI # Import libEnsemble items for this test from libensemble.libE import libE, libE_tcp_worker @@ -25,13 +24,8 @@ from libensemble.alloc_funcs.fast_alloc_to_aposmm import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_1 as persis_info, aposmm_gen_out as gen_out, six_hump_camel_minima as minima -from time import time nworkers, is_master, libE_specs, _ = parse_args() -np.random.seed(MPI.COMM_WORLD.Get_rank()) - -if is_master: - start_time = time() n = 2 sim_specs = {'sim_f': sim_f, @@ -62,10 +56,65 @@ exit_criteria = {'sim_max': 1000} # Set up appropriate abort mechanism depending on comms +libE_abort = quit +if libE_specs['comms'] == 'mpi': + from mpi4py import MPI + + def libE_mpi_abort(): + MPI.COMM_WORLD.Abort(1) + + libE_abort = libE_mpi_abort + +# Perform the run (TCP worker mode) +if libE_specs['comms'] == 'tcp' and not is_master: + run = int(sys.argv[-1]) + libE_tcp_worker(sim_specs, gen_specs[run], libE_specs) + quit() + +# Perform the run +for run in range(2): + if libE_specs['comms'] == 'tcp' and is_master: + libE_specs['worker_cmd'].append(str(run)) + + if run == 1: + # Change the bounds to put a local min at a corner point (to test that + # APOSMM handles the same point being in multiple runs) ability to + # give back a previously evaluated point) + gen_specs['ub'] = np.array([-2.9, -1.9]) + gen_specs['mu'] = 1e-4 + gen_specs['rk_const'] = 0.01*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi) + gen_specs['lhs_divisions'] = 2 + # APOSMM can be called when some run is incomplete + gen_specs.pop('batch_mode') + + gen_specs.pop('xtol_rel') + gen_specs['ftol_rel'] = 1e-2 + gen_specs['xtol_abs'] = 1e-3 + gen_specs['ftol_abs'] = 1e-8 + exit_criteria = {'sim_max': 200, 'elapsed_wallclock_time': 300} + minima = np.array([[-2.9, -1.9]]) + + persis_info = persis_info_safe + + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, + persis_info, alloc_specs, libE_specs) + + if is_master: + if flag != 0: + print("Exit was not on convergence (code {})".format(flag)) + sys.stdout.flush() + libE_abort() -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs) + tol = 1e-5 + for m in minima: + # The minima are known on this test problem. + # 1) We use their values to test APOSMM has identified all minima + # 2) We use their approximate values to ensure APOSMM evaluates a + # point in each minima's basin of attraction. + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1))) + sys.stdout.flush() + if np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) > tol: + libE_abort() -if is_master: - print('[Manager]:', H[np.where(H['local_min'])]['x']) - print('[Manager]: Time taken =', time()-start_time) + print("\nlibEnsemble with APOSMM using a gradient-based localopt method has identified the " + str(np.shape(minima)[0]) + " minima within a tolerance " + str(tol)) + save_libE_output(H, persis_info, __file__, nworkers) From 83167c582e4741a574ec1751456459a19c37e245 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 2 Aug 2019 23:07:48 -0500 Subject: [PATCH 061/644] removes unused parts from earlier aposmm logic --- libensemble/gen_funcs/persistent_aposmm.py | 293 --------------------- 1 file changed, 293 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index b1bb17772..cce506729 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -829,253 +829,6 @@ def update_history_optimal(x_opt, H, run_inds): H['num_active_runs'][run_inds] -= 1 -def advance_local_run(H, gen_specs, c_flag, run, persis_info): - """ - Moves a local optimization method one iteration forward. We currently do - this by feeding all past evaluations from a run to the method and then - storing the first new point generated - """ - - while 1: - sorted_run_inds = persis_info['run_order'][run] - advance_local_run.x_new = np.ones((1, len(gen_specs['ub'])))*np.inf - advance_local_run.pt_in_run = 0 - - if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', - 'LN_COBYLA', 'LN_NELDERMEAD', - 'LD_MMA']: - - if gen_specs['localopt_method'] in ['LD_MMA']: - fields_to_pass = ['x_on_cube', 'f', 'grad'] - else: - fields_to_pass = ['x_on_cube', 'f'] - - try: - x_opt, exit_code = set_up_and_run_nlopt(H[fields_to_pass][sorted_run_inds], gen_specs) - except Exception as e: - x_opt = 0 - exit_code = 0 - display_exception(e) - - elif gen_specs['localopt_method'] in ['pounders']: - - if c_flag: - Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, gen_specs['components'])]) - for i, ind in enumerate(sorted_run_inds): - a1 = H['pt_id'] == H['pt_id'][ind] - Run_H_F['fvec'][i, :] = H['f_i'][a1] - Run_H = merge_arrays([H[['x_on_cube']][sorted_run_inds], Run_H_F], flatten=True) - else: - Run_H = H[['x_on_cube', 'fvec']][sorted_run_inds] - - try: - x_opt, exit_code = set_up_and_run_tao(Run_H, gen_specs) - except Exception as e: - x_opt = 0 - exit_code = 0 - display_exception(e) - - elif gen_specs['localopt_method'] == 'scipy_COBYLA': - - fields_to_pass = ['x_on_cube', 'f'] - - try: - x_opt, exit_code = set_up_and_run_scipy_minimize(H[fields_to_pass][sorted_run_inds], gen_specs) - except Exception as e: - x_opt = 0 - exit_code = 0 - display_exception(e) - - else: - raise APOSMMException("Unknown localopt method. Exiting") - - match_ind = np.equal(advance_local_run.x_new, H['x_on_cube']).all(1) - if ~match_ind.any(): - # Generated a new point - break - else: - # We need to add a previously evaluated point into this run - persis_info['run_order'][run].append(np.nonzero(match_ind)[0][0]) - - return x_opt, exit_code, persis_info, sorted_run_inds, advance_local_run.x_new - - -def set_up_and_run_scipy_minimize(Run_H, gen_specs): - """ Set up objective and runs scipy - - Declares the appropriate syntax for our special objective function to read - through Run_H, sets the parameters and starting points for the run. - """ - - def scipy_obj_fun(x, Run_H): - out = look_in_history(x, Run_H) - - return out - - obj = lambda x: scipy_obj_fun(x, Run_H) - x0 = Run_H['x_on_cube'][0] - - # Construct the bounds in the form of constraints - cons = [] - for factor in range(len(x0)): - lo = {'type': 'ineq', - 'fun': lambda x, lb=gen_specs['lb'][factor], i=factor: x[i]-lb} - up = {'type': 'ineq', - 'fun': lambda x, ub=gen_specs['ub'][factor], i=factor: ub-x[i]} - cons.append(lo) - cons.append(up) - - method = gen_specs['localopt_method'][6:] - res = scipy_optimize.minimize(obj, x0, method=method, options={'maxiter': len(Run_H['x_on_cube'])+1, 'tol': gen_specs['tol']}) - - if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum - exit_code = 0 - else: - if method == 'COBYLA': - assert res['status'] == 1, "Unknown status for COBYLA" - exit_code = 1 - - x_opt = res['x'] - return x_opt, exit_code - - -def set_up_and_run_nlopt(Run_H, gen_specs): - """ Set up objective and runs nlopt - - Declares the appropriate syntax for our special objective function to read - through Run_H, sets the parameters and starting points for the run. - """ - - assert 'xtol_rel' or 'xtol_abs' or 'ftol_rel' or 'ftol_abs' in gen_specs, "NLopt can cycle if xtol_rel, xtol_abs, ftol_rel, or ftol_abs are not set" - - def nlopt_obj_fun(x, grad, Run_H): - out = look_in_history(x, Run_H) - - if gen_specs['localopt_method'] in ['LD_MMA']: - grad[:] = out[1] - out = out[0] - - return out - - n = len(gen_specs['ub']) - - opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) - - lb = np.zeros(n) - ub = np.ones(n) - opt.set_lower_bounds(lb) - opt.set_upper_bounds(ub) - x0 = Run_H['x_on_cube'][0] - - # Care must be taken here because a too-large initial step causes nlopt to move the starting point! - dist_to_bound = min(min(ub-x0), min(x0-lb)) - assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" - - if 'dist_to_bound_multiple' in gen_specs: - opt.set_initial_step(dist_to_bound*gen_specs['dist_to_bound_multiple']) - else: - opt.set_initial_step(dist_to_bound) - - opt.set_maxeval(len(Run_H)+1) # evaluate one more point - opt.set_min_objective(lambda x, grad: nlopt_obj_fun(x, grad, Run_H)) - if 'xtol_rel' in gen_specs: - opt.set_xtol_rel(gen_specs['xtol_rel']) - if 'ftol_rel' in gen_specs: - opt.set_ftol_rel(gen_specs['ftol_rel']) - if 'xtol_abs' in gen_specs: - opt.set_xtol_abs(gen_specs['xtol_abs']) - if 'ftol_abs' in gen_specs: - opt.set_ftol_abs(gen_specs['ftol_abs']) - - x_opt = opt.optimize(x0) - exit_code = opt.last_optimize_result() - - if exit_code == 5: # NLOPT code for exhausting budget of evaluations, so not at a minimum - exit_code = 0 - - return x_opt, exit_code - - -def set_up_and_run_tao(Run_H, gen_specs): - """ Set up objective and runs PETSc on the comm_self communicator - - Declares the appropriate syntax for our special objective function to read - through Run_H, sets the parameters and starting points for the run. - """ - tao_comm = MPI.COMM_SELF - n = len(gen_specs['ub']) - m = len(Run_H['fvec'][0]) - - def pounders_obj_func(tao, X, F, Run_H): - F.array = look_in_history(X.array, Run_H, vector_return=True) - return F - - # def blmvm_obj_func(tao, X, G, Run_H): - # (f, grad) = look_in_history_fd_grad(X.array, Run_H) - # G.array = grad - # return f - - # Create starting point, bounds, and tao object - x = PETSc.Vec().create(tao_comm) - x.setSizes(n) - x.setFromOptions() - x.array = Run_H['x_on_cube'][0] - lb = x.duplicate() - ub = x.duplicate() - lb.array = 0*np.ones(n) - ub.array = 1*np.ones(n) - tao = PETSc.TAO().create(tao_comm) - tao.setType(gen_specs['localopt_method']) - - # if gen_specs['localopt_method'] == 'pounders': - f = PETSc.Vec().create(tao_comm) - f.setSizes(m) - f.setFromOptions() - - delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) - - PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) - # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') - if hasattr(tao, 'setResidual'): - tao.setResidual(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) - else: - tao.setSeparableObjective(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) - # elif gen_specs['localopt_method'] == 'blmvm': - # g = PETSc.Vec().create(tao_comm) - # g.setSizes(n) - # g.setFromOptions() - # tao.setObjectiveGradient(lambda tao, x, g: blmvm_obj_func(tao, x, g, Run_H)) - - # Set everything for tao before solving - PETSc.Options().setValue('-tao_max_funcs', str(len(Run_H)+1)) - tao.setFromOptions() - tao.setVariableBounds((lb, ub)) - # tao.setObjectiveTolerances(fatol=gen_specs['fatol'], frtol=gen_specs['frtol']) - # tao.setGradientTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) - tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) - tao.setInitial(x) - - tao.solve(x) - - x_opt = tao.getSolution().getArray() - exit_code = tao.getConvergedReason() - # print(exit_code) - # print(tao.view()) - # print(x_opt) - - # if gen_specs['localopt_method'] == 'pounders': - f.destroy() - # elif gen_specs['localopt_method'] == 'blmvm': - # g.destroy() - - lb.destroy() - ub.destroy() - x.destroy() - tao.destroy() - - return x_opt, exit_code - - def decide_where_to_start_localopt(H, n, n_s, rk_const, mu=0, nu=0): """ Finds points in the history that satisfy the conditions (S1-S5 and L1-L8) in @@ -1187,40 +940,6 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, mu=0, nu=0): return start_inds -def look_in_history(x, Run_H, vector_return=False): - """ See if Run['x_on_cube'][advance_local_run.pt_in_run] matches x, - returning f or fvec, or saves x to advance_local_run.x_new if every point in Run_H has been - checked. - """ - - if vector_return: - to_return = 'fvec' - else: - if 'grad' in Run_H.dtype.names: - to_return = ['f', 'grad'] - else: - to_return = 'f' - - if advance_local_run.pt_in_run < len(Run_H): - # Return the value in history to the localopt algorithm. - assert np.allclose(x, Run_H['x_on_cube'][advance_local_run.pt_in_run], rtol=1e-08, atol=1e-08), \ - "History point does not match Localopt point" - f_out = Run_H[to_return][advance_local_run.pt_in_run] - else: - if advance_local_run.pt_in_run == len(Run_H): - # The history of points is exhausted. Save the requested point x to - # x_new. x_new will be returned to the manager. - advance_local_run.x_new[:] = x - - # Just in case the local opt method requests more points after a new - # point has been identified. - f_out = Run_H[to_return][-1] - - advance_local_run.pt_in_run += 1 - - return f_out - - def calc_rk(n, n_s, rk_const, lhs_divisions=0): """ Calculate the critical distance r_k """ @@ -1235,18 +954,6 @@ def calc_rk(n, n_s, rk_const, lhs_divisions=0): return r_k -def count_samples(H,gen_specs): - if 'single_component_at_a_time' in gen_specs and gen_specs['single_component_at_a_time']: - # Get the pt_id for non-nan, returned points - pt_ids = H['pt_id'][np.logical_and(H['returned'], ~np.isnan(H['f_i']))] - _, counts = np.unique(pt_ids, return_counts=True) - n_s = np.sum(counts == gen_specs['components']) - - else: - # Number of returned sampled points (excluding nans) - n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['returned']))) - - return n_s def initialize_APOSMM(H, gen_specs, libE_info): """ From 1e95d93628d4c2a5a7fb01805c01f74e72cddef8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Sat, 3 Aug 2019 23:59:14 -0500 Subject: [PATCH 062/644] Trying to get blmvm --- libensemble/gen_funcs/persistent_aposmm.py | 51 ++++++++++++++++------ 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index b1bb17772..0c42d31d7 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -210,7 +210,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # {{{ setting the data needed by the local optimization method - if gen_specs['localopt_method'] in ['LD_MMA']: + if gen_specs['localopt_method'] in ['LD_MMA','blmvm']: fields_to_pass = ['f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: @@ -390,7 +390,7 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: run_local_opt = run_local_nlopt - elif gen_specs['localopt_method'] in ['pounders']: + elif gen_specs['localopt_method'] in ['pounders','blmvm']: run_local_opt = run_local_tao elif gen_specs['localopt_method'] in ['scipy_COBYLA']: run_local_opt = run_local_scipy_opt @@ -586,6 +586,24 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read f.array[:] = f_recv return f +def tao_callback_function_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs): + import ipdb; ipdb.set_trace() + comm_queue.put(x.array) + # print('[Child]: I just put x_on_cube =', x.array, flush=True) + # print('[Child]: Parent should no longer wait.', flush=True) + parent_can_read.set() + # print('[Child]: I have started waiting', flush=True) + child_can_read.wait() + # print('[Child]: Wohooo.. I am free folks', flush=True) + if gen_specs['localopt_method'] in ['blmvm']: + f_recv, grad_recv = comm_queue.get() + g.array[:] = grad_recv + else: + assert gen_specs['localopt_method'] in ['pounders'] + f_recv, = comm_queue.get() + child_can_read.clear() + f.array[:] = f_recv + return f def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): @@ -611,17 +629,22 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, tao = PETSc.TAO().create(tao_comm) tao.setType(gen_specs['localopt_method']) - f = PETSc.Vec().create(tao_comm) - f.setSizes(m) - f.setFromOptions() + if gen_specs['localopt_method'] == 'pounders': + f = PETSc.Vec().create(tao_comm) + f.setSizes(m) + f.setFromOptions() - delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + tao.setResidual(lambda tao, x, f: tao_callback_function(tao, x, f, + comm_queue, child_can_read, parent_can_read, gen_specs), f) - PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) - # PETSc.Options().setValue('-pounders_subsolver_tao_type','bqpip') + elif gen_specs['localopt_method'] == 'blmvm': + g = PETSc.Vec().create(tao_comm) + g.setSizes(n) + g.setFromOptions() + tao.setObjectiveGradient(lambda tao, x, f, g: tao_callback_function_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs)) - tao.setResidual(lambda tao, x, f: tao_callback_function(tao, x, f, - comm_queue, child_can_read, parent_can_read, gen_specs), f) + delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) # Set everything for tao before solving # FIXME: Hard-coding 100 as the max funcs as couldn't find any other @@ -643,10 +666,10 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, # print(tao.view()) # print(x_opt) - # if gen_specs['localopt_method'] == 'pounders': - f.destroy() - # elif gen_specs['localopt_method'] == 'blmvm': - # g.destroy() + if gen_specs['localopt_method'] == 'pounders': + f.destroy() + elif gen_specs['localopt_method'] == 'blmvm': + g.destroy() lb.destroy() ub.destroy() From 95b7e63472c8a7cd6dcb51f0c7612eb8dff31adc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 5 Aug 2019 07:10:50 -0500 Subject: [PATCH 063/644] Trying blmvm --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 606080994..8a95151b4 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -48,7 +48,7 @@ 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), - 'localopt_method': 'LD_MMA', + 'localopt_method': 'blmvm', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'xtol_rel': 1e-3, 'num_active_gens': 1, From 00bfb923a9bc4e24d696ee87da47dc61c56a4a7e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 6 Aug 2019 04:01:38 -0500 Subject: [PATCH 064/644] flake8 --- .../alloc_funcs/start_only_persistent.py | 2 +- libensemble/gen_funcs/persistent_aposmm.py | 142 ++++++++---------- .../test_6-hump_camel_persistent_aposmm.py | 5 +- 3 files changed, 70 insertions(+), 79 deletions(-) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index c1a35e228..4b3543b60 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -21,7 +21,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): gen_inds = (H['gen_worker'] == i) - if np.any(np.logical_and(H['returned'][gen_inds],~H['given_back'][gen_inds])): + if np.any(np.logical_and(H['returned'][gen_inds], ~H['given_back'][gen_inds])): last_time_gen_gave_batch = np.max(H['gen_time'][gen_inds]) inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] gen_work(Work, i, diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index a64fe3cb8..a079d5bca 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -4,21 +4,17 @@ described in detail in the paper `https://doi.org/10.1007/s12532-017-0131-4 `_ """ -__all__ = ['aposmm_logic', 'initialize_APOSMM', - 'decide_where_to_start_localopt', 'update_history_dist'] +__all__ = ['initialize_APOSMM', 'decide_where_to_start_localopt', 'update_history_dist'] import sys -import pickle import traceback import numpy as np -from scipy.spatial.distance import cdist, pdist, squareform -from scipy import optimize as scipy_optimize -from petsc4py import PETSc +from scipy.spatial.distance import cdist +from scipy import optimize as sp_opt +from petsc5py import PETSc from mpi4py import MPI -from numpy.lib.recfunctions import merge_arrays - from math import log, gamma, pi, sqrt import nlopt @@ -202,7 +198,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # }}} - n, n_s, c_flag, rk_const, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + n, n_s, c_flag, rk_const, ld, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) sim_id_to_child_indices = {} child_id_to_run_id = {} @@ -210,47 +206,46 @@ def aposmm(H, persis_info, gen_specs, libE_info): # {{{ setting the data needed by the local optimization method - if gen_specs['localopt_method'] in ['LD_MMA','blmvm']: + if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: + 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: fields_to_pass = ['f'] else: - raise NotImplementedError("Unknown local optimization method " - "'{}'.".format(gen_specs['localopt_method'])) + raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) # }}} for _ in range(gen_specs['initial_sample_size']): - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, + comm, local_H, sim_id_to_child_indices) tag = None max_active_runs = gen_specs.get('max_active_runs', np.inf) waiting_starting_inds = [] - O = np.empty(0, dtype=gen_specs['out']) + while 1: tag, Work, calc_in = get_mgr_worker_msg(comm) if calc_in: if fields_to_pass == ['f', 'grad']: - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, grad_f_x_recv) else: assert(fields_to_pass == ['f']) if len(calc_in[0]) == 3: - (x_recv, f_x_recv, sim_id_recv), = calc_in + (x_recv, f_x_recv, sim_id_recv), = calc_in else: # FIXME: *BIG BIG!*, even if we are passing grad_f even # though we are not using it. - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in + (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in data_to_give_processes = (f_x_recv, ) if tag in [STOP_TAG, PERSIS_STOP]: # FIXME: This has to be a clean exit. print('[Parent]: The optimal points are:\n', - local_H[np.where(local_H['local_min'])]['x'], flush=True) + local_H[np.where(local_H['local_min'])]['x'], flush=True) for p in local_opters: if p.is_running: @@ -264,8 +259,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opters[child_idx].iterate(*data_to_give_processes) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x - update_history_optimal(x_opt, local_H, - run_order[child_id_to_run_id[child_idx]]) + update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) if waiting_starting_inds: ind = waiting_starting_inds[0] @@ -274,9 +268,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): # {{{ initing a local opt run local_opter = LocalOptInterfacer(gen_specs, - local_H[ind]['x_on_cube'], - parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) + local_H[ind]['x_on_cube'], + parent_can_read_from_queue, comm_queue, + local_H[ind]['f'], grad_f_x_recv if 'grad' in + fields_to_pass else None) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) @@ -296,7 +291,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) else: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + sim_id_to_child_indices) else: add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) assert np.allclose(local_H[-1]['x_on_cube'], x_new) @@ -312,7 +307,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): continue n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, mu, nu) + starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, ld, mu, nu) for ind in starting_inds: @@ -323,9 +318,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): # {{{ initing a local opt run local_opter = LocalOptInterfacer(gen_specs, - local_H[ind]['x_on_cube'], - parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) + local_H[ind]['x_on_cube'], + parent_can_read_from_queue, comm_queue, + local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) local_opters.append(local_opter) x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) @@ -349,7 +344,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if len(starting_inds) == 0 and len(waiting_starting_inds) == 0: send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + sim_id_to_child_indices) comm_queue.close() comm_queue.join_thread() @@ -357,14 +352,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): for local_opter in local_opters: if local_opter.is_running: raise RuntimeError("[Parent]: Atleast one child process is still active, even after" - " killing all the children.") + " killing all the children.") return local_H, persis_info, tag class LocalOptInterfacer(object): - def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, - grad0=None): + def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, grad0=None): """ :param x0: A numpy array of the initial guess solution. This guess should be scaled to a unit cube. @@ -387,25 +381,22 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, # {{{ setting the local optimization method - if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'LD_MMA']: + if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: run_local_opt = run_local_nlopt - elif gen_specs['localopt_method'] in ['pounders','blmvm']: + elif gen_specs['localopt_method'] in ['pounders', 'blmvm']: run_local_opt = run_local_tao elif gen_specs['localopt_method'] in ['scipy_COBYLA']: run_local_opt = run_local_scipy_opt else: raise NotImplementedError("Unknown local optimization method " - "'{}'.".format(gen_specs['localopt_method'])) + "'{}'.".format(gen_specs['localopt_method'])) # }}} self.parent_can_read.clear() self.process = Process(target=run_local_opt, args=(gen_specs, - comm_queue, - x0, f0, - self.child_can_read, - parent_can_read)) + comm_queue, x0, f0, self.child_can_read, + parent_can_read)) self.process.start() self.is_running = True @@ -443,8 +434,7 @@ def destroy(self): if self.grad0 is None: self.comm_queue.put((0*np.ones_like(self.f0),)) else: - self.comm_queue.put((0*np.ones_like(self.f0), - np.zeros_like(self.grad0))) + self.comm_queue.put((0*np.ones_like(self.f0), np.zeros_like(self.grad0))) self.child_can_read.set() self.parent_can_read.wait() @@ -457,7 +447,7 @@ def destroy(self): # {{{ NLOPT for local opt -def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): +def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) parent_can_read.set() child_can_read.wait() @@ -466,7 +456,7 @@ def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read grad[:] = grad_recv else: assert gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', - 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] + 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] f_recv, = comm_queue.get() child_can_read.clear() @@ -474,8 +464,7 @@ def nlopt_callback_function(x, grad, comm_queue, child_can_read, parent_can_read return f_recv -def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, - parent_can_read): +def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): # print('[Child]: Started local opt at {}.'.format(x0), flush=True) n = len(gen_specs['ub']) @@ -498,8 +487,9 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, # FIXME: Setting max evaluations = 100 opt.set_maxeval(100) - opt.set_min_objective(lambda x, grad: nlopt_callback_function(x, grad, - comm_queue, child_can_read, parent_can_read, gen_specs)) + opt.set_min_objective(lambda x, grad: nlopt_callback_fun(x, grad, + comm_queue, child_can_read, parent_can_read, + gen_specs)) if 'xtol_rel' in gen_specs: opt.set_xtol_rel(gen_specs['xtol_rel']) @@ -510,7 +500,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, if 'ftol_abs' in gen_specs: opt.set_ftol_abs(gen_specs['ftol_abs']) - #FIXME: Do we need to do something of the final 'x_opt'? + # FIXME: Do we need to do something of the final 'x_opt'? # print('[Child]: Started my optimization', flush=True) x_opt = opt.optimize(x0) # print('[Child]: I have converged.', flush=True) @@ -522,7 +512,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, # {{{ SciPy optimization -def scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_specs): +def scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x) # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() @@ -534,8 +524,7 @@ def scipy_callback_function(x, comm_queue, child_can_read, parent_can_read, gen_ return f_x_recv -def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, - parent_can_read): +def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): # Construct the bounds in the form of constraints cons = [] @@ -549,9 +538,9 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, method = gen_specs['localopt_method'][6:] # print('[Child]: Started my optimization', flush=True) - res = scipy_optimize.minimize(lambda x: scipy_callback_function(x, - comm_queue, child_can_read, parent_can_read, gen_specs), x0, - method=method, options={'maxiter': 100, 'tol': gen_specs['tol']}) + res = sp_opt.minimize(lambda x: scipy_callback_fun(x, comm_queue, + child_can_read, parent_can_read, gen_specs), x0, + method=method, options={'maxiter': 100, 'tol': gen_specs['tol']}) if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum exit_code = 0 @@ -563,6 +552,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, x_opt = res['x'] # FIXME: Need to do something with the exit codes. + print(exit_code) # print('[Child]: I have converged.', flush=True) comm_queue.put(ConvergedMsg(x_opt)) @@ -573,7 +563,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, # {{{ TAO routines for local opt -def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): +def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x.array) # print('[Child]: I just put x_on_cube =', x.array, flush=True) # print('[Child]: Parent should no longer wait.', flush=True) @@ -581,13 +571,14 @@ def tao_callback_function(tao, x, f, comm_queue, child_can_read, parent_can_read # print('[Child]: I have started waiting', flush=True) child_can_read.wait() # print('[Child]: Wohooo.. I am free folks', flush=True) - f_recv, = comm_queue.get() + f_recv, = comm_queue.get() child_can_read.clear() f.array[:] = f_recv return f -def tao_callback_function_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs): - import ipdb; ipdb.set_trace() + +def tao_callback_fun_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs): + comm_queue.put(x.array) # print('[Child]: I just put x_on_cube =', x.array, flush=True) # print('[Child]: Parent should no longer wait.', flush=True) @@ -605,8 +596,8 @@ def tao_callback_function_grad(tao, x, f, g, comm_queue, child_can_read, parent_ f.array[:] = f_recv return f -def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, - parent_can_read): + +def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): assert isinstance(x0, np.ndarray) @@ -634,14 +625,13 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, f.setSizes(m) f.setFromOptions() - tao.setResidual(lambda tao, x, f: tao_callback_function(tao, x, f, - comm_queue, child_can_read, parent_can_read, gen_specs), f) + tao.setResidual(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) elif gen_specs['localopt_method'] == 'blmvm': g = PETSc.Vec().create(tao_comm) g.setSizes(n) g.setFromOptions() - tao.setObjectiveGradient(lambda tao, x, f, g: tao_callback_function_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs)) + tao.setObjectiveGradient(lambda tao, x, f, g: tao_callback_fun_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs)) delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) @@ -662,7 +652,9 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, x_opt = tao.getSolution().getArray() exit_code = tao.getConvergedReason() - # print(exit_code) + + # FIXME: Need to do something with the exit codes. + print(exit_code) # print(tao.view()) # print(x_opt) @@ -676,7 +668,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, x.destroy() tao.destroy() - #FIXME: Do we need to do something of the final 'x_opt'? + # FIXME: Do we need to do something of the final 'x_opt'? # print('[Child]: I have converged.', flush=True) comm_queue.put(ConvergedMsg(x_opt)) parent_can_read.set() @@ -819,7 +811,6 @@ def update_history_dist(H, n, gen_specs, c_flag): # H['worse_within_rk'][H['dist_to_all'] > r_k] = False - def update_history_optimal(x_opt, H, run_inds): """ Updated the history after any point has been declared a local minimum @@ -852,7 +843,7 @@ def update_history_optimal(x_opt, H, run_inds): H['num_active_runs'][run_inds] -= 1 -def decide_where_to_start_localopt(H, n, n_s, rk_const, mu=0, nu=0): +def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): """ Finds points in the history that satisfy the conditions (S1-S5 and L1-L8) in Table 1 of the `APOSMM paper `_ @@ -894,7 +885,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, mu=0, nu=0): ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ - r_k = calc_rk(n, n_s, rk_const) + r_k = calc_rk(n, n_s, rk_const, ld) if nu > 0: test_2_through_5 = np.logical_and.reduce(( @@ -1020,8 +1011,8 @@ def initialize_APOSMM(H, gen_specs, libE_info): comm = libE_info['comm'] - local_H_fields = [('f',float), - ('grad',float,n), + local_H_fields = [('f', float), + ('grad', float, n), ('x', float, n), ('x_on_cube', float, n), ('priority', float), @@ -1043,11 +1034,10 @@ def initialize_APOSMM(H, gen_specs, libE_info): local_H = np.empty(0, dtype=local_H_fields) - return n, n_s, c_flag, rk_c, mu, nu, total_runs, comm, local_H + return n, n_s, c_flag, rk_c, ld, mu, nu, total_runs, comm, local_H -def send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices): +def send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): sampled_points = persis_info['rand_stream'].uniform(0, 1, (1, n)) add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) assert local_H['sim_id'][-1] not in sim_id_to_child_indices diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 8a95151b4..f6fc3c0c5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -41,7 +41,8 @@ 'out': [('f', float), ('grad', float, n)]} gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), - ('local_min', bool)] + ('local_min', bool)] + gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, @@ -74,4 +75,4 @@ if is_master: print('[Manager]:', H[np.where(H['local_min'])]['x']) print('[Manager]: Time taken =', time() - start_time, flush=True) - # save_libE_output(H, persis_info, __file__, nworkers) + save_libE_output(H, persis_info, __file__, nworkers) From 44d8881da98e61cd2905a1ad20cc454e5266ccd7 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 6 Aug 2019 06:41:11 -0500 Subject: [PATCH 065/644] Fixing typo --- libensemble/gen_funcs/persistent_aposmm.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index a079d5bca..a5600c132 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -11,7 +11,7 @@ import numpy as np from scipy.spatial.distance import cdist from scipy import optimize as sp_opt -from petsc5py import PETSc +from petsc4py import PETSc from mpi4py import MPI diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index f6fc3c0c5..c6d0a1dc5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -49,7 +49,7 @@ 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), - 'localopt_method': 'blmvm', + 'localopt_method': 'LD_MMA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'xtol_rel': 1e-3, 'num_active_gens': 1, From 31aaa40d3bde953d7acd7a96db29512c7c80caad Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 7 Aug 2019 03:48:48 -0500 Subject: [PATCH 066/644] Updating travis file --- .travis.yml | 78 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9aa0d66a7..8dacfbeda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,57 +6,71 @@ python: - 3.5 - 3.6 - 3.7 - + os: linux env: global: - HYDRA_LAUNCHER=fork - - OMPI_MCA_rmaps_base_oversubscribe=yes + - OMPI_MCA_rmaps_base_oversubscribe=yes matrix: - MPI=mpich #- MPI=openmpi -#matrix: +matrix: + include: + - os: osx + osx_image: xcode11 + env: MPI=mpich PY=3 + language: generic + python: 3 + + +# matrix: # allow_failures: # - env: MPI=openmpi - -addons: - apt: - packages: - - gfortran - - libblas-dev - - liblapack-dev - -cache: + +# addons: +# apt: +# packages: +# - gfortran +# - libblas-dev +# - liblapack-dev + +cache: pip: true apt: true # Setup Miniconda before_install: - # Do this conditionally because it saves some downloading if the version is the same. - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh -O miniconda.sh; else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - conda config --set always_yes yes --set changeps1 no - - conda update -q -y conda + #- conda update -q -y conda - conda info -a # For debugging any issues with conda - - conda config --add channels conda-forge - - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION + - conda config --add channels conda-forge + - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv install: - - conda install gcc_linux-64 - - conda install $MPI - - conda install --no-update-deps scipy # includes numpy - - conda install --no-update-deps mpi4py - - conda install petsc4py petsc - - conda install --no-update-deps nlopt + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.13.sdk.tar.xz; + mkdir ../sdk; tar xf MacOSX10.13.sdk.tar.xz -C ../sdk; + COMPILERS=clang_osx-64; + MUMPS=mumps-mpi=5.1.2=haf446c3_1007; + else + COMPILERS=gcc_linux-64; + MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; + fi + - conda install $COMPILERS + #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI + - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 - pip install flake8 - pip install pytest @@ -65,12 +79,18 @@ install: - pip install mock - pip install coveralls # For confirmation of MPI library being used. - - python conda/find_mpi.py #locate compilers - - mpiexec --version #Show MPI library details + - python conda/find_mpi.py # locate compilers + - mpiexec --version # Show MPI library details - pip install -e . + before_script: - - flake8 libensemble + - flake8 libensemble + # Set conda compilers to use new SDK instead of Travis default. + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; + source setenv.sh; + fi # Run test (-z show output) script: @@ -78,7 +98,7 @@ script: # Coverage after_success: - - mv libensemble/tests/.cov* . + - mv libensemble/tests/.cov* . - coveralls after_failure: From 882064856e53bfffa595699c8315d2409edd289e Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 8 Aug 2019 11:34:11 -0500 Subject: [PATCH 067/644] Modifies .travis.yml to include postgresql service. Adds line for balsam-setup. Adds balsam-setup script to generate balsam db and initialize. Adds regression test that modifies balsam, then submits script as job to balsam --- .travis.yml | 4 + conda/balsam-setup.sh | 17 +++ ...script_test_jobcontroller_hworld_balsam.py | 103 ++++++++++++++++++ .../test_jobcontroller_hworld_balsam.py | 36 ++++++ 4 files changed, 160 insertions(+) create mode 100755 conda/balsam-setup.sh create mode 100644 libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py create mode 100644 libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py diff --git a/.travis.yml b/.travis.yml index 8dacfbeda..4487d1883 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,8 @@ matrix: language: generic python: 3 +services: + - postgresql # matrix: # allow_failures: @@ -72,6 +74,7 @@ install: #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 + - pip install balsam-flow - pip install flake8 - pip install pytest - pip install pytest-cov @@ -86,6 +89,7 @@ install: before_script: - flake8 libensemble + - ./conda/balsam-setup.sh # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh new file mode 100755 index 000000000..af614f35e --- /dev/null +++ b/conda/balsam-setup.sh @@ -0,0 +1,17 @@ +balsam init ~/test-balsam +source balsamactivate test-balsam + +export EXE=script_test_jobcontroller_hworld_balsam.py +export NUM_WORKERS=2 +export WORKFLOW_NAME=libe_test-balsam +export SCRIPT_ARGS=$NUM_WORKERS +export LIBE_WALLCLOCK=3 +export THIS_DIR=$PWD +export SCRIPT_BASENAME=${EXE%.*} + +balsam rm apps --all --force +balsam rm jobs --all --force + +balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" + +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes diff --git a/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py new file mode 100644 index 000000000..5bd61c26e --- /dev/null +++ b/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py @@ -0,0 +1,103 @@ +# This script is meant to be launched by Balsam + +import os +import numpy as np +import multiprocessing + +from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED +from libensemble.libE import libE +from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f +from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.tests.regression_tests.common import build_simfunc, parse_args, per_worker_stream + +from mpi4py import MPI + +nworkers = MPI.COMM_WORLD.Get_size() - 1 +is_master = MPI.COMM_WORLD.Get_rank() == 0 +libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} + +cores_per_job = 1 +logical_cores = multiprocessing.cpu_count() +cores_all_jobs = nworkers*cores_per_job + +if cores_all_jobs > logical_cores: + use_auto_resources = False + mess_resources = 'Oversubscribing - auto_resources set to False' +else: + use_auto_resources = True + mess_resources = 'Auto_resources set to True' + +if is_master: + print('\nCores req: {} Cores avail: {}\n {}\n'.format(cores_all_jobs, logical_cores, mess_resources)) + +sim_app = './my_simjob.x' +if not os.path.isfile(sim_app): + build_simfunc() + +from libensemble.balsam_controller import BalsamJobController +jobctrl = BalsamJobController(auto_resources=use_auto_resources) + +jobctrl.register_calc(full_path=sim_app, calc_type='sim') + +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('cstat', int)], + 'save_every_k': 400, + 'cores': cores_per_job} + +gen_specs = {'gen_f': gen_f, + 'in': ['sim_id'], + 'out': [('x', float, (2,))], + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'gen_batch_size': nworkers, + 'batch_mode': True, + 'num_active_gens': 1, + 'save_every_k': 20} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'elapsed_wallclock_time': 10} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + libE_specs=libE_specs) + +if is_master: + print('\nChecking expected job status against Workers ...\n') + + # Expected list: Last is zero as will not be entered into H array on + # manager kill - but should show in the summary file. + # Repeat expected lists nworkers times and compare with list of status's + # received from workers + calc_status_list_in = np.asarray([WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED, 0]) + calc_status_list = np.repeat(calc_status_list_in, nworkers) + + # For debug + print("Expecting: {}".format(calc_status_list)) + print("Received: {}\n".format(H['cstat'])) + + assert np.array_equal(H['cstat'], calc_status_list), "Error - unexpected calc status. Received: " + str(H['cstat']) + + # Check summary file: + print('Checking expected job status against job summary file ...\n') + + calc_desc_list_in = ['Completed', 'Worker killed job on Error', + 'Worker killed job on Timeout', 'Job Failed', + 'Manager killed on finish'] + + # Repeat N times for N workers and insert Completed at start for generator + calc_desc_list = ['Completed'] + calc_desc_list_in*nworkers + # script_name = os.path.splitext(os.path.basename(__file__))[0] + # short_name = script_name.split("test_", 1).pop() + # summary_file_name = short_name + '.libe_summary.txt' + # with open(summary_file_name,'r') as f: + # i=0 + # for line in f: + # if "Status:" in line: + # _, file_status = line.partition("Status:")[::2] + # print("Expected: {} Filestatus: {}".format(calc_desc_list[i], file_status.strip())) + # assert calc_desc_list[i] == file_status.strip(), "Status does not match file" + # i+=1 + + print("\n\n\nRun completed.") diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py new file mode 100644 index 000000000..0d2126b42 --- /dev/null +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py @@ -0,0 +1,36 @@ +import subprocess +import os +import balsam + +# Balsam is meant for HPC systems that commonly distribute jobs across many +# nodes. Due to the nature of testing Balsam on local or CI systems which usually +# only contain a single node, we need to change Balsam's default worker setup +# so multiple workers can be run on a single node (until this feature is [hopefully] added!). +# For our purposes, we append ten workers to Balsam's WorkerGroup + + +### MODIFY BALSAM WORKERGROUP ### + +home = os.getcwd() +balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' +os.chdir(balsam_worker_path) + +with open('worker.py', 'r') as f: + lines = f.readlines() + +if lines[-3] != " for idx in range(10):\n": + lines = lines[:-2] # Will re-add these lines + lines.extend([" for idx in range(10):\n", + " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", + " self.workers.append(w)\n"]) + +with open('testworker.py', 'w') as f: + for line in lines: + f.write(line) + +os.chdir(home) + +### EXECUTE BALSAM JOB ### + +runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' +subprocess.check_call(runstr.split(' ')) From 393ca75fce428c332d9cc84694398c4e2cfa85ea Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 8 Aug 2019 11:46:18 -0500 Subject: [PATCH 068/644] flake8 --- .../script_test_jobcontroller_hworld_balsam.py | 9 ++++++--- .../regression_tests/test_jobcontroller_hworld_balsam.py | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py index 5bd61c26e..56131e3ec 100644 --- a/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py +++ b/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py @@ -1,17 +1,21 @@ -# This script is meant to be launched by Balsam +# This script is meant to be submitted to Balsam for execution import os import numpy as np import multiprocessing +from libensemble.balsam_controller import BalsamJobController from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import build_simfunc, parse_args, per_worker_stream +from libensemble.tests.regression_tests.common import build_simfunc, per_worker_stream from mpi4py import MPI +# This script is meant to be launched by Balsam + + nworkers = MPI.COMM_WORLD.Get_size() - 1 is_master = MPI.COMM_WORLD.Get_rank() == 0 libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} @@ -34,7 +38,6 @@ if not os.path.isfile(sim_app): build_simfunc() -from libensemble.balsam_controller import BalsamJobController jobctrl = BalsamJobController(auto_resources=use_auto_resources) jobctrl.register_calc(full_path=sim_app, calc_type='sim') diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py index 0d2126b42..983178d21 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py @@ -9,7 +9,7 @@ # For our purposes, we append ten workers to Balsam's WorkerGroup -### MODIFY BALSAM WORKERGROUP ### +# MODIFY BALSAM WORKERGROUP home = os.getcwd() balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' @@ -19,7 +19,7 @@ lines = f.readlines() if lines[-3] != " for idx in range(10):\n": - lines = lines[:-2] # Will re-add these lines + lines = lines[:-2] # Will re-add these lines lines.extend([" for idx in range(10):\n", " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", " self.workers.append(w)\n"]) @@ -30,7 +30,7 @@ os.chdir(home) -### EXECUTE BALSAM JOB ### +# EXECUTE BALSAM JOB runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' subprocess.check_call(runstr.split(' ')) From 72cb078056812ae394fb76af9bf42d8a8a5d10e9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 8 Aug 2019 14:23:04 -0500 Subject: [PATCH 069/644] db configuration attempts --- conda/balsam-setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index af614f35e..92e367e3b 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -1,4 +1,6 @@ -balsam init ~/test-balsam +psql -c 'create database ../test-balsam;' -U postgres +balsam init ../test-balsam +sudo chown -R postgres:postgres /var/run/postgresql source balsamactivate test-balsam export EXE=script_test_jobcontroller_hworld_balsam.py From 3c016769d70a807a94b0845ef5202b6257f27ed7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 8 Aug 2019 14:51:31 -0500 Subject: [PATCH 070/644] changes init location, modifies permissions --- conda/balsam-setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index 92e367e3b..511f0bf19 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -1,6 +1,7 @@ -psql -c 'create database ../test-balsam;' -U postgres -balsam init ../test-balsam +export BALSAM_DB_PATH='~/test-balsam' sudo chown -R postgres:postgres /var/run/postgresql +balsam init ~/test-balsam +sudo chmod -R 700 ~/test-balsam/balsamdb source balsamactivate test-balsam export EXE=script_test_jobcontroller_hworld_balsam.py From 4786b002cc0af90fc4c4b33e7f808225211b9f82 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 8 Aug 2019 15:11:04 -0500 Subject: [PATCH 071/644] modifies local travis build script for balsam --- conda/run_travis_locally/build_mpich_libE.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index cb6e0de9e..757015092 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -5,8 +5,8 @@ # -x echo commands set -x # problem with this - loads of conda internal commands shown - overwhelming. -export PYTHON_VERSION=3.7 # default - override with -p -export LIBE_BRANCH="develop" # default - override with -b +export PYTHON_VERSION=3.6 # default - override with -p +export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH @@ -76,6 +76,7 @@ conda install nlopt petsc4py petsc mumps-mpi=5.1.2=h5bebb2f_1007 mpi4py scipy $M # conda install nlopt || return # pip install these as the conda installs downgrade pytest on python3.4 +pip install balsam-flow || return pip install pytest || return pip install pytest-cov || return pip install pytest-timeout || return @@ -87,6 +88,7 @@ git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || retu cd libensemble/ || return pip install -e . || return +conda/balsam-setup.sh libensemble/tests/run-tests.sh echo -e "\n\nScript completed...\n\n" From bfd523b4e90b8797df8ae04dbb37874ff47e7be8 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 15:20:56 -0500 Subject: [PATCH 072/644] Adds lots of permission and directory changes to setup scripts. Temporary ease-of-testing changes. Renames testing scripts. --- conda/balsam-setup.sh | 3 +- conda/run_travis_locally/build_mpich_libE.sh | 40 +++++++++++++++++-- ...hworld_balsam.py => script_test_balsam.py} | 0 ...roller_hworld_balsam.py => test_balsam.py} | 0 4 files changed, 39 insertions(+), 4 deletions(-) rename libensemble/tests/regression_tests/{script_test_jobcontroller_hworld_balsam.py => script_test_balsam.py} (100%) rename libensemble/tests/regression_tests/{test_jobcontroller_hworld_balsam.py => test_balsam.py} (100%) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index 511f0bf19..f2af6e8e3 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -1,10 +1,11 @@ export BALSAM_DB_PATH='~/test-balsam' sudo chown -R postgres:postgres /var/run/postgresql +sudo chmod a+w /var/run/postgresql balsam init ~/test-balsam sudo chmod -R 700 ~/test-balsam/balsamdb source balsamactivate test-balsam -export EXE=script_test_jobcontroller_hworld_balsam.py +export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam export SCRIPT_ARGS=$NUM_WORKERS diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 757015092..aba4357e4 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -41,7 +41,16 @@ done echo -e "\nBuilding libE on ${SYSTEM} with python $PYTHON_VERSION and branch ${LIBE_BRANCH}\n" -# sudo apt-get update +# mkdir pg; cd pg; +# wget http://sbp.enterprisedb.com/getfile.jsp?fileid=11966 +# tar xf getfile.jsp?fileid=11966 +# export PATH="$PATH:/home/travis/pg/pgsql/bin" +# cd ~ + +sudo pip install --upgrade pip +sudo /etc/init.d/postgresql stop 9.2 +sudo /etc/init.d/postgresql start 9.6 +export PATH=$PATH:/usr/lib/postgresql/9.6/bin # This works if not sourced but if sourced its no good. # set -e @@ -88,8 +97,33 @@ git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || retu cd libensemble/ || return pip install -e . || return -conda/balsam-setup.sh -libensemble/tests/run-tests.sh +export BALSAM_DB_PATH='~/test-balsam' +sudo chown -R postgres:postgres /var/run/postgresql +sudo chmod a+w /var/run/postgresql +balsam init ~/test-balsam +sudo chmod -R 700 ~/test-balsam/balsamdb +source balsamactivate test-balsam + +export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py +export NUM_WORKERS=2 +export WORKFLOW_NAME=libe_test-balsam +export SCRIPT_ARGS=$NUM_WORKERS +export LIBE_WALLCLOCK=3 +export THIS_DIR=$PWD +export SCRIPT_BASENAME=${EXE%.*} + +balsam rm apps --all --force +balsam rm jobs --all --force + +balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" + +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes + +MYHOME=$PWD +cd libensemble/tests/regression_tests +mpiexec -n 3 python test_balsam.py +cd $MYHOME +#libensemble/tests/run-tests.sh echo -e "\n\nScript completed...\n\n" set +ex diff --git a/libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py similarity index 100% rename from libensemble/tests/regression_tests/script_test_jobcontroller_hworld_balsam.py rename to libensemble/tests/regression_tests/script_test_balsam.py diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py b/libensemble/tests/regression_tests/test_balsam.py similarity index 100% rename from libensemble/tests/regression_tests/test_jobcontroller_hworld_balsam.py rename to libensemble/tests/regression_tests/test_balsam.py From 6a55cd6f1aaa9b240f85886b21ba621ec0808679 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 15:37:44 -0500 Subject: [PATCH 073/644] Removes extra args, adds mpi4py import, other cleaning --- conda/balsam-setup.sh | 6 ++++-- conda/run_travis_locally/build_mpich_libE.sh | 8 +++++--- libensemble/tests/regression_tests/script_test_balsam.py | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index f2af6e8e3..88bb5ef40 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -8,7 +8,7 @@ source balsamactivate test-balsam export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam -export SCRIPT_ARGS=$NUM_WORKERS +# export SCRIPT_ARGS=$NUM_WORKERS export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD export SCRIPT_BASENAME=${EXE%.*} @@ -18,4 +18,6 @@ balsam rm jobs --all --force balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" -balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes + +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index aba4357e4..88b561d44 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -107,7 +107,7 @@ source balsamactivate test-balsam export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam -export SCRIPT_ARGS=$NUM_WORKERS +#export SCRIPT_ARGS=$NUM_WORKERS export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD export SCRIPT_BASENAME=${EXE%.*} @@ -117,11 +117,13 @@ balsam rm jobs --all --force balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" -balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes + +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes MYHOME=$PWD cd libensemble/tests/regression_tests -mpiexec -n 3 python test_balsam.py +python test_balsam.py cd $MYHOME #libensemble/tests/run-tests.sh diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 56131e3ec..c88978d7a 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -11,14 +11,14 @@ from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import build_simfunc, per_worker_stream +import mpi4py from mpi4py import MPI # This script is meant to be launched by Balsam - +mpi4py.rc.recv_mprobe = False nworkers = MPI.COMM_WORLD.Get_size() - 1 -is_master = MPI.COMM_WORLD.Get_rank() == 0 -libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} +is_master = (MPI.COMM_WORLD.Get_rank() == 0) cores_per_job = 1 logical_cores = multiprocessing.cpu_count() From f05cf990da6a88f43be718b0a30193b407eb908d Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 16:08:24 -0500 Subject: [PATCH 074/644] truncates test script basename to prevent too many nested directories. --- conda/balsam-setup.sh | 3 ++- conda/run_travis_locally/build_mpich_libE.sh | 3 ++- .../tests/regression_tests/script_test_balsam.py | 12 +++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index 88bb5ef40..b826a3949 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -11,7 +11,8 @@ export WORKFLOW_NAME=libe_test-balsam # export SCRIPT_ARGS=$NUM_WORKERS export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD -export SCRIPT_BASENAME=${EXE%.*} +#export SCRIPT_BASENAME=${EXE%.*} +export SCRIPT_BASENAME=script_test_balsam balsam rm apps --all --force balsam rm jobs --all --force diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 88b561d44..84e72b6ed 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -110,7 +110,8 @@ export WORKFLOW_NAME=libe_test-balsam #export SCRIPT_ARGS=$NUM_WORKERS export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD -export SCRIPT_BASENAME=${EXE%.*} +#export SCRIPT_BASENAME=${EXE%.*} +export SCRIPT_BASENAME=script_test_balsam balsam rm apps --all --force balsam rm jobs --all --force diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index c88978d7a..acdbfc33f 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -9,11 +9,21 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import build_simfunc, per_worker_stream +from libensemble.tests.regression_tests.common import per_worker_stream import mpi4py from mpi4py import MPI +def build_simfunc(): + import subprocess + print(os.getcwd()) + + # Build simfunc + # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn + buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' + # subprocess.run(buildstring.split(),check=True) #Python3.5+ + subprocess.check_call(buildstring.split()) + # This script is meant to be launched by Balsam mpi4py.rc.recv_mprobe = False From 5afdb581c583fc8c80c9c885d3ce04c8062ef167 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 16:31:35 -0500 Subject: [PATCH 075/644] fixes build_simfunc path, attempt at prepopulating libE_specs --- libensemble/tests/regression_tests/script_test_balsam.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index acdbfc33f..6690d8b50 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -16,16 +16,16 @@ def build_simfunc(): import subprocess - print(os.getcwd()) - # Build simfunc # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn - buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' + buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) # This script is meant to be launched by Balsam +libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} + mpi4py.rc.recv_mprobe = False nworkers = MPI.COMM_WORLD.Get_size() - 1 is_master = (MPI.COMM_WORLD.Get_rank() == 0) @@ -49,7 +49,6 @@ def build_simfunc(): build_simfunc() jobctrl = BalsamJobController(auto_resources=use_auto_resources) - jobctrl.register_calc(full_path=sim_app, calc_type='sim') sim_specs = {'sim_f': sim_f, From 78cd783cfc1cda0870d80769d00738e6298a8c6b Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 16:57:06 -0500 Subject: [PATCH 076/644] Removes duplicate code, cleans up commented-out code --- conda/balsam-setup.sh | 6 +--- conda/run_travis_locally/build_mpich_libE.sh | 36 +++++--------------- 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index b826a3949..3532e7adc 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -6,12 +6,10 @@ sudo chmod -R 700 ~/test-balsam/balsamdb source balsamactivate test-balsam export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py -export NUM_WORKERS=2 +export NUM_WORKERS=1 export WORKFLOW_NAME=libe_test-balsam -# export SCRIPT_ARGS=$NUM_WORKERS export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD -#export SCRIPT_BASENAME=${EXE%.*} export SCRIPT_BASENAME=script_test_balsam balsam rm apps --all --force @@ -19,6 +17,4 @@ balsam rm jobs --all --force balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" -# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes - balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 84e72b6ed..48a35f3e9 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -11,6 +11,7 @@ export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH export HYDRA_LAUNCHER=fork +export TESTBALSAM=true # Allow user to optionally set python version and branch # E.g: ". ./build_mpich_libE.sh -p 3.4 -b feature/myfeature" @@ -97,35 +98,14 @@ git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || retu cd libensemble/ || return pip install -e . || return -export BALSAM_DB_PATH='~/test-balsam' -sudo chown -R postgres:postgres /var/run/postgresql -sudo chmod a+w /var/run/postgresql -balsam init ~/test-balsam -sudo chmod -R 700 ~/test-balsam/balsamdb -source balsamactivate test-balsam - -export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py -export NUM_WORKERS=2 -export WORKFLOW_NAME=libe_test-balsam -#export SCRIPT_ARGS=$NUM_WORKERS -export LIBE_WALLCLOCK=3 -export THIS_DIR=$PWD -#export SCRIPT_BASENAME=${EXE%.*} -export SCRIPT_BASENAME=script_test_balsam - -balsam rm apps --all --force -balsam rm jobs --all --force - -balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" - -# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes - -balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +if [[ "$TESTBALSAM" == true]]; then + source /conda/balsam-setup.sh + MYHOME=$PWD + cd libensemble/tests/regression_tests + python test_balsam.py + cd $MYHOME +fi -MYHOME=$PWD -cd libensemble/tests/regression_tests -python test_balsam.py -cd $MYHOME #libensemble/tests/run-tests.sh echo -e "\n\nScript completed...\n\n" From 58a83607cd14b14956da41bbeedd2667c8074ea6 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 17:10:23 -0500 Subject: [PATCH 077/644] attempt to fix if block --- conda/run_travis_locally/build_mpich_libE.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 48a35f3e9..d35ccf024 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -99,11 +99,11 @@ cd libensemble/ || return pip install -e . || return if [[ "$TESTBALSAM" == true]]; then - source /conda/balsam-setup.sh - MYHOME=$PWD - cd libensemble/tests/regression_tests - python test_balsam.py - cd $MYHOME + source /conda/balsam-setup.sh; + MYHOME=$PWD; + cd libensemble/tests/regression_tests; + python test_balsam.py; + cd $MYHOME; fi #libensemble/tests/run-tests.sh From 6e0e9b93895ed298f8576e0a070a344866cd8566 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 9 Aug 2019 17:21:50 -0500 Subject: [PATCH 078/644] added period --- conda/run_travis_locally/build_mpich_libE.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index d35ccf024..64b191344 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -98,8 +98,8 @@ git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || retu cd libensemble/ || return pip install -e . || return -if [[ "$TESTBALSAM" == true]]; then - source /conda/balsam-setup.sh; +if [[ "$TESTBALSAM" == true ]]; then + source ./conda/balsam-setup.sh; MYHOME=$PWD; cd libensemble/tests/regression_tests; python test_balsam.py; From 64ed27f03b034584bba74b8137e0a6e3794ffdfb Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Mon, 12 Aug 2019 08:49:11 -0500 Subject: [PATCH 079/644] corrects the setObjectiveGradient invocation --- libensemble/gen_funcs/persistent_aposmm.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index a5600c132..80c5fe1c8 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -577,7 +577,7 @@ def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen return f -def tao_callback_fun_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs): +def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, gen_specs): comm_queue.put(x.array) # print('[Child]: I just put x_on_cube =', x.array, flush=True) @@ -586,15 +586,10 @@ def tao_callback_fun_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_r # print('[Child]: I have started waiting', flush=True) child_can_read.wait() # print('[Child]: Wohooo.. I am free folks', flush=True) - if gen_specs['localopt_method'] in ['blmvm']: - f_recv, grad_recv = comm_queue.get() - g.array[:] = grad_recv - else: - assert gen_specs['localopt_method'] in ['pounders'] - f_recv, = comm_queue.get() + f_recv, grad_recv = comm_queue.get() + g.array[:] = grad_recv child_can_read.clear() - f.array[:] = f_recv - return f + return f_recv def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): @@ -631,7 +626,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read g = PETSc.Vec().create(tao_comm) g.setSizes(n) g.setFromOptions() - tao.setObjectiveGradient(lambda tao, x, f, g: tao_callback_fun_grad(tao, x, f, g, comm_queue, child_can_read, parent_can_read, gen_specs)) + tao.setObjectiveGradient(lambda tao, x, g: tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, gen_specs)) delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) From b7b4f0bac1c0cd9427cd92186c179314d75861ec Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 09:03:22 -0500 Subject: [PATCH 080/644] re-adds TESTSUITE fields for proper parsing by run-tests --- libensemble/tests/regression_tests/test_balsam.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libensemble/tests/regression_tests/test_balsam.py b/libensemble/tests/regression_tests/test_balsam.py index 983178d21..8c9dff107 100644 --- a/libensemble/tests/regression_tests/test_balsam.py +++ b/libensemble/tests/regression_tests/test_balsam.py @@ -8,6 +8,8 @@ # so multiple workers can be run on a single node (until this feature is [hopefully] added!). # For our purposes, we append ten workers to Balsam's WorkerGroup +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 2 # MODIFY BALSAM WORKERGROUP From 2197da5388a36718b9b30e9e8d27a4c2f743ff79 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 09:05:44 -0500 Subject: [PATCH 081/644] flake8 --- libensemble/tests/regression_tests/script_test_balsam.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 6690d8b50..93f05ecd2 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -14,6 +14,7 @@ import mpi4py from mpi4py import MPI + def build_simfunc(): import subprocess # Build simfunc @@ -22,6 +23,7 @@ def build_simfunc(): # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) + # This script is meant to be launched by Balsam libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} From 09c0e4ed93798049d7b19590deb53918e282e049 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 09:40:07 -0500 Subject: [PATCH 082/644] Fixes balsam worker file modification, some numerical changes --- .../tests/regression_tests/script_test_balsam.py | 2 +- libensemble/tests/regression_tests/test_balsam.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 93f05ecd2..82537dad8 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -71,7 +71,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 10} +exit_criteria = {'elapsed_wallclock_time': 15} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, diff --git a/libensemble/tests/regression_tests/test_balsam.py b/libensemble/tests/regression_tests/test_balsam.py index 8c9dff107..34ebb7e56 100644 --- a/libensemble/tests/regression_tests/test_balsam.py +++ b/libensemble/tests/regression_tests/test_balsam.py @@ -9,24 +9,26 @@ # For our purposes, we append ten workers to Balsam's WorkerGroup # TESTSUITE_COMMS: local -# TESTSUITE_NPROCS: 2 +# TESTSUITE_NPROCS: 3 # MODIFY BALSAM WORKERGROUP +workerfile = 'worker.py' + home = os.getcwd() balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' os.chdir(balsam_worker_path) -with open('worker.py', 'r') as f: +with open(workerfile, 'r') as f: lines = f.readlines() -if lines[-3] != " for idx in range(10):\n": +if lines[-3] != " for idx in range(3):\n": lines = lines[:-2] # Will re-add these lines - lines.extend([" for idx in range(10):\n", + lines.extend([" for idx in range(3):\n", " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", " self.workers.append(w)\n"]) -with open('testworker.py', 'w') as f: +with open(workerfile, 'w') as f: for line in lines: f.write(line) @@ -35,4 +37,4 @@ # EXECUTE BALSAM JOB runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' -subprocess.check_call(runstr.split(' ')) +subprocess.check_call(runstr.split()) From 3aa4cebff5d66306e2cce58f12924cce2f4364c1 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 10:17:40 -0500 Subject: [PATCH 083/644] balsam-specific sim func added --- .../sim_funcs/job_control_hworld_balsam.py | 121 ++++++++++++++++++ .../regression_tests/script_test_balsam.py | 3 +- 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 libensemble/sim_funcs/job_control_hworld_balsam.py diff --git a/libensemble/sim_funcs/job_control_hworld_balsam.py b/libensemble/sim_funcs/job_control_hworld_balsam.py new file mode 100644 index 000000000..638eb9ef8 --- /dev/null +++ b/libensemble/sim_funcs/job_control_hworld_balsam.py @@ -0,0 +1,121 @@ +from libensemble.balsam_controller import BalsamJobController +from libensemble.message_numbers import UNSET_TAG, WORKER_KILL_ON_ERR, MAN_SIGNAL_FINISH, WORKER_DONE, JOB_FAILED, WORKER_KILL_ON_TIMEOUT +import numpy as np + +__all__ = ['job_control_hworld'] + +# Alt send values through X +sim_count = 0 + + +def polling_loop(comm, jobctl, job, timeout_sec=3.0, delay=0.3): + import time + + calc_status = UNSET_TAG # Sim func determines status of libensemble calc - returned to worker + + while job.runtime < timeout_sec: + time.sleep(delay) + + # print('Probing manager at time: ', job.runtime) + jobctl.manager_poll(comm) + if jobctl.manager_signal == 'finish': + jobctl.kill(job) + calc_status = MAN_SIGNAL_FINISH # Worker will pick this up and close down + print('Job {} killed by manager on worker {}'.format(job.id, jobctl.workerID)) + break + + # print('Polling job at time', job.runtime) + job.poll() + if job.finished: + break + elif job.state == 'RUNNING': + print('Job {} still running on worker {} ....'.format(job.id, jobctl.workerID)) + + # Check output file for error + # print('Checking output file for error at time:', job.runtime) + if job.stdout_exists(): + if 'Error' in job.read_stdout(): + print("Found (deliberate) Error in ouput file - cancelling job {} on worker {}".format(job.id, jobctl.workerID)) + jobctl.kill(job) + calc_status = WORKER_KILL_ON_ERR + break + + # After exiting loop + if job.finished: + print('Job {} done on worker {}'.format(job.id, jobctl.workerID)) + # Fill in calc_status if not already + if calc_status == UNSET_TAG: + if job.state == 'FINISHED': # Means finished succesfully + calc_status = WORKER_DONE + elif job.state == 'FAILED': + calc_status = JOB_FAILED + # elif job.state == 'USER_KILLED': + # calc_status = WORKER_KILL + else: + # assert job.state == 'RUNNING', "job.state expected to be RUNNING. Returned: " + str(job.state) + print("Job {} timed out - killing on worker {}".format(job.id, jobctl.workerID)) + jobctl.kill(job) + if job.finished: + print('Job {} done on worker {}'.format(job.id, jobctl.workerID)) + calc_status = WORKER_KILL_ON_TIMEOUT + + return job, calc_status + + +def job_control_hworld(H, persis_info, sim_specs, libE_specs): + """ Test of launching and polling job and exiting on job finish""" + jobctl = BalsamJobController.controller + cores = sim_specs['cores'] + comm = libE_specs['comm'] + + args_for_sim = 'sleep 1' + # pref send this in X as a sim_in from calling script + global sim_count + sim_count += 1 + timeout = 3.0 + if sim_count == 1: + args_for_sim = 'sleep 1' # Should finish + elif sim_count == 2: + args_for_sim = 'sleep 1 Error' # Worker kill on error + elif sim_count == 3: + args_for_sim = 'sleep 3' # Worker kill on timeout + timeout = 1.0 + elif sim_count == 4: + args_for_sim = 'sleep 1 Fail' # Manager kill - if signal received else completes + elif sim_count == 5: + args_for_sim = 'sleep 18' # Manager kill - if signal received else completes + timeout = 20.0 + + job = jobctl.launch(calc_type='sim', num_procs=cores, app_args=args_for_sim, hyperthreads=True) + job, calc_status = polling_loop(comm, jobctl, job, timeout) + + # assert job.finished, "job.finished should be True. Returned " + str(job.finished) + # assert job.state == 'FINISHED', "job.state should be FINISHED. Returned " + str(job.state) + + # This is temp - return something - so doing six_hump_camel_func again... + batch = len(H['x']) + O = np.zeros(batch, dtype=sim_specs['out']) + for i, x in enumerate(H['x']): + O['f'][i] = six_hump_camel_func(x) + + # This is just for testing at calling script level - status of each job + O['cstat'] = calc_status + + # v = np.random.uniform(0, 10) + # print('About to sleep for :' + str(v)) + # time.sleep(v) + + return O, persis_info, calc_status + + +def six_hump_camel_func(x): + """ + Definition of the six-hump camel + """ + x1 = x[0] + x2 = x[1] + term1 = (4-2.1*x1**2+(x1**4)/3) * x1**2 + term2 = x1*x2 + term3 = (-4+4*x2**2) * x2**2 + + return term1 + term2 + term3 diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 82537dad8..0b13ac564 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -7,11 +7,12 @@ from libensemble.balsam_controller import BalsamJobController from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE -from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f +from libensemble.sim_funcs.job_control_hworld_balsam import job_control_hworld as sim_f from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import per_worker_stream import mpi4py +mpi4py.rc.recv_mprobe = False # Disable matching probes from mpi4py import MPI From 014c2e4bb6bdf02954e07266f0080e5230c37df8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 12 Aug 2019 10:34:34 -0500 Subject: [PATCH 084/644] Using array_r to PETSc read array (Otherwise I get errors) --- libensemble/gen_funcs/persistent_aposmm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 80c5fe1c8..0952f8298 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -564,7 +564,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_ca # {{{ TAO routines for local opt def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): - comm_queue.put(x.array) + comm_queue.put(x.array_r) # print('[Child]: I just put x_on_cube =', x.array, flush=True) # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() @@ -579,7 +579,7 @@ def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, gen_specs): - comm_queue.put(x.array) + comm_queue.put(x.array_r) # print('[Child]: I just put x_on_cube =', x.array, flush=True) # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() From 29e550e195e9a68bcf576eaa8958595682ee778b Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 11:26:54 -0500 Subject: [PATCH 085/644] experimental numerical adjustments --- conda/balsam-setup.sh | 2 +- libensemble/tests/regression_tests/script_test_balsam.py | 5 ++--- libensemble/tests/regression_tests/test_balsam.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/conda/balsam-setup.sh b/conda/balsam-setup.sh index 3532e7adc..aa658cb19 100755 --- a/conda/balsam-setup.sh +++ b/conda/balsam-setup.sh @@ -6,7 +6,7 @@ sudo chmod -R 700 ~/test-balsam/balsamdb source balsamactivate test-balsam export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py -export NUM_WORKERS=1 +export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 0b13ac564..1cfe851b0 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -29,9 +29,8 @@ def build_simfunc(): libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} -mpi4py.rc.recv_mprobe = False nworkers = MPI.COMM_WORLD.Get_size() - 1 -is_master = (MPI.COMM_WORLD.Get_rank() == 0) +is_master = MPI.COMM_WORLD.Get_rank() == 0 cores_per_job = 1 logical_cores = multiprocessing.cpu_count() @@ -72,7 +71,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 15} +exit_criteria = {'elapsed_wallclock_time': 10} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, diff --git a/libensemble/tests/regression_tests/test_balsam.py b/libensemble/tests/regression_tests/test_balsam.py index 34ebb7e56..016588960 100644 --- a/libensemble/tests/regression_tests/test_balsam.py +++ b/libensemble/tests/regression_tests/test_balsam.py @@ -22,9 +22,9 @@ with open(workerfile, 'r') as f: lines = f.readlines() -if lines[-3] != " for idx in range(3):\n": +if lines[-3] != " for idx in range(10):\n": lines = lines[:-2] # Will re-add these lines - lines.extend([" for idx in range(3):\n", + lines.extend([" for idx in range(10):\n", " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", " self.workers.append(w)\n"]) From 3e16c37b74a7e3e47f9760c987c796b92af5db48 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 13:55:22 -0500 Subject: [PATCH 086/644] try disabling autoresources, get job controller instance from JobController class instead --- libensemble/sim_funcs/job_control_hworld_balsam.py | 4 +++- .../tests/regression_tests/script_test_balsam.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libensemble/sim_funcs/job_control_hworld_balsam.py b/libensemble/sim_funcs/job_control_hworld_balsam.py index 638eb9ef8..92aed518b 100644 --- a/libensemble/sim_funcs/job_control_hworld_balsam.py +++ b/libensemble/sim_funcs/job_control_hworld_balsam.py @@ -1,4 +1,5 @@ from libensemble.balsam_controller import BalsamJobController +from libensemble.controller import JobController from libensemble.message_numbers import UNSET_TAG, WORKER_KILL_ON_ERR, MAN_SIGNAL_FINISH, WORKER_DONE, JOB_FAILED, WORKER_KILL_ON_TIMEOUT import numpy as np @@ -64,7 +65,8 @@ def polling_loop(comm, jobctl, job, timeout_sec=3.0, delay=0.3): def job_control_hworld(H, persis_info, sim_specs, libE_specs): """ Test of launching and polling job and exiting on job finish""" - jobctl = BalsamJobController.controller + #jobctl = BalsamJobController.controller + jobctl = JobController.controller cores = sim_specs['cores'] comm = libE_specs['comm'] diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 1cfe851b0..521e0fc7a 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -36,12 +36,12 @@ def build_simfunc(): logical_cores = multiprocessing.cpu_count() cores_all_jobs = nworkers*cores_per_job -if cores_all_jobs > logical_cores: - use_auto_resources = False - mess_resources = 'Oversubscribing - auto_resources set to False' -else: - use_auto_resources = True - mess_resources = 'Auto_resources set to True' +# if cores_all_jobs > logical_cores: +# use_auto_resources = False +# mess_resources = 'Oversubscribing - auto_resources set to False' +# else: +# use_auto_resources = True +# mess_resources = 'Auto_resources set to True' if is_master: print('\nCores req: {} Cores avail: {}\n {}\n'.format(cores_all_jobs, logical_cores, mess_resources)) @@ -50,7 +50,7 @@ def build_simfunc(): if not os.path.isfile(sim_app): build_simfunc() -jobctrl = BalsamJobController(auto_resources=use_auto_resources) +jobctrl = BalsamJobController(auto_resources=False) jobctrl.register_calc(full_path=sim_app, calc_type='sim') sim_specs = {'sim_f': sim_f, From 61cf005a514eacadee70c63954910a1e8e5390c8 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 14:47:42 -0500 Subject: [PATCH 087/644] remove reference to autoresources, reenable resource message --- .../tests/regression_tests/script_test_balsam.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 521e0fc7a..1b56ec4cb 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -36,12 +36,12 @@ def build_simfunc(): logical_cores = multiprocessing.cpu_count() cores_all_jobs = nworkers*cores_per_job -# if cores_all_jobs > logical_cores: -# use_auto_resources = False -# mess_resources = 'Oversubscribing - auto_resources set to False' -# else: -# use_auto_resources = True -# mess_resources = 'Auto_resources set to True' +if cores_all_jobs > logical_cores: + use_auto_resources = False + mess_resources = 'Oversubscribing - auto_resources set to False' +else: + use_auto_resources = True + mess_resources = 'Auto_resources set to True' if is_master: print('\nCores req: {} Cores avail: {}\n {}\n'.format(cores_all_jobs, logical_cores, mess_resources)) @@ -50,7 +50,7 @@ def build_simfunc(): if not os.path.isfile(sim_app): build_simfunc() -jobctrl = BalsamJobController(auto_resources=False) +jobctrl = BalsamJobController() jobctrl.register_calc(full_path=sim_app, calc_type='sim') sim_specs = {'sim_f': sim_f, From b837119be4ebe7251f555c597b43bc4e3fdc1f26 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 16:14:40 -0500 Subject: [PATCH 088/644] re-add autoresources = false flag --- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 1b56ec4cb..fbdd2d284 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -50,7 +50,7 @@ def build_simfunc(): if not os.path.isfile(sim_app): build_simfunc() -jobctrl = BalsamJobController() +jobctrl = BalsamJobController(use_auto_resources=False) jobctrl.register_calc(full_path=sim_app, calc_type='sim') sim_specs = {'sim_f': sim_f, From f4f8d9ca2295e6ad35dd06c34bd06e7c000e3717 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 16:33:01 -0500 Subject: [PATCH 089/644] Timing and settings adjustments --- libensemble/sim_funcs/job_control_hworld_balsam.py | 4 ++-- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/sim_funcs/job_control_hworld_balsam.py b/libensemble/sim_funcs/job_control_hworld_balsam.py index 92aed518b..5dd615d98 100644 --- a/libensemble/sim_funcs/job_control_hworld_balsam.py +++ b/libensemble/sim_funcs/job_control_hworld_balsam.py @@ -74,7 +74,7 @@ def job_control_hworld(H, persis_info, sim_specs, libE_specs): # pref send this in X as a sim_in from calling script global sim_count sim_count += 1 - timeout = 3.0 + timeout = 6.0 if sim_count == 1: args_for_sim = 'sleep 1' # Should finish elif sim_count == 2: @@ -88,7 +88,7 @@ def job_control_hworld(H, persis_info, sim_specs, libE_specs): args_for_sim = 'sleep 18' # Manager kill - if signal received else completes timeout = 20.0 - job = jobctl.launch(calc_type='sim', num_procs=cores, app_args=args_for_sim, hyperthreads=True) + job = jobctl.launch(calc_type='sim', num_procs=cores, app_args=args_for_sim, hyperthreads=True, wait_on_run=True) job, calc_status = polling_loop(comm, jobctl, job, timeout) # assert job.finished, "job.finished should be True. Returned " + str(job.finished) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index fbdd2d284..a4c7d0c1d 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -71,7 +71,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 10} +exit_criteria = {'elapsed_wallclock_time': 30} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 8fd59a11d8bd67f4eaf2be71e7d2f3919ce2e397 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 16:46:50 -0500 Subject: [PATCH 090/644] fixes autoresources, reeanbles running all tests with setup script --- conda/run_travis_locally/build_mpich_libE.sh | 6 +----- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 64b191344..47d50ff4e 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -100,13 +100,9 @@ pip install -e . || return if [[ "$TESTBALSAM" == true ]]; then source ./conda/balsam-setup.sh; - MYHOME=$PWD; - cd libensemble/tests/regression_tests; - python test_balsam.py; - cd $MYHOME; fi -#libensemble/tests/run-tests.sh +libensemble/tests/run-tests.sh echo -e "\n\nScript completed...\n\n" set +ex diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index a4c7d0c1d..39358d271 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -50,7 +50,7 @@ def build_simfunc(): if not os.path.isfile(sim_app): build_simfunc() -jobctrl = BalsamJobController(use_auto_resources=False) +jobctrl = BalsamJobController(auto_resources=False) jobctrl.register_calc(full_path=sim_app, calc_type='sim') sim_specs = {'sim_f': sim_f, From b7c7f35a3df07be9e2c7df11706249549205f47d Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 12 Aug 2019 16:53:08 -0500 Subject: [PATCH 091/644] flake8 --- libensemble/sim_funcs/job_control_hworld_balsam.py | 4 +--- libensemble/tests/regression_tests/script_test_balsam.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/libensemble/sim_funcs/job_control_hworld_balsam.py b/libensemble/sim_funcs/job_control_hworld_balsam.py index 5dd615d98..a8dd89e8c 100644 --- a/libensemble/sim_funcs/job_control_hworld_balsam.py +++ b/libensemble/sim_funcs/job_control_hworld_balsam.py @@ -1,5 +1,4 @@ from libensemble.balsam_controller import BalsamJobController -from libensemble.controller import JobController from libensemble.message_numbers import UNSET_TAG, WORKER_KILL_ON_ERR, MAN_SIGNAL_FINISH, WORKER_DONE, JOB_FAILED, WORKER_KILL_ON_TIMEOUT import numpy as np @@ -65,8 +64,7 @@ def polling_loop(comm, jobctl, job, timeout_sec=3.0, delay=0.3): def job_control_hworld(H, persis_info, sim_specs, libE_specs): """ Test of launching and polling job and exiting on job finish""" - #jobctl = BalsamJobController.controller - jobctl = JobController.controller + jobctl = BalsamJobController.controller cores = sim_specs['cores'] comm = libE_specs['comm'] diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 39358d271..05f413f3b 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -3,6 +3,8 @@ import os import numpy as np import multiprocessing +import mpi4py +from mpi4py import MPI from libensemble.balsam_controller import BalsamJobController from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED @@ -11,9 +13,7 @@ from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import per_worker_stream -import mpi4py mpi4py.rc.recv_mprobe = False # Disable matching probes -from mpi4py import MPI def build_simfunc(): From 84e8c76e56cc8e1944475305b3aba94bdf494e57 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 10:19:34 -0500 Subject: [PATCH 092/644] Most recent version of balsam obtained from github. py-version dependent balsam testing and configuring. --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4487d1883..2bef8cbc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,6 @@ install: #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 - - pip install balsam-flow - pip install flake8 - pip install pytest - pip install pytest-cov @@ -85,11 +84,16 @@ install: - python conda/find_mpi.py # locate compilers - mpiexec --version # Show MPI library details - pip install -e . + - if [[ $TRAVIS_PYTHON_VERSION >= 3.6 ]]; then + git clone https://github.com/balsam-alcf/balsam.git ../balsam; + cd ../balsam; pip install -e .; cd ../libensemble; + mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; + ./conda/balsam-setup.sh; + fi before_script: - flake8 libensemble - - ./conda/balsam-setup.sh # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; From 4511f638368f5ce1b2b426b45ac0ff9f0c94ff62 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 10:24:32 -0500 Subject: [PATCH 093/644] fixes py version comparision --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2bef8cbc7..de130fbbb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,7 +84,7 @@ install: - python conda/find_mpi.py # locate compilers - mpiexec --version # Show MPI library details - pip install -e . - - if [[ $TRAVIS_PYTHON_VERSION >= 3.6 ]]; then + - if [[ $TRAVIS_PYTHON_VERSION -ge 3.6 ]]; then git clone https://github.com/balsam-alcf/balsam.git ../balsam; cd ../balsam; pip install -e .; cd ../libensemble; mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; From 7d07f21a1680f102dfdc1578a013d42b6792a24b Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 11:08:14 -0500 Subject: [PATCH 094/644] better version comparision --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index de130fbbb..f6461ae7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,7 +84,7 @@ install: - python conda/find_mpi.py # locate compilers - mpiexec --version # Show MPI library details - pip install -e . - - if [[ $TRAVIS_PYTHON_VERSION -ge 3.6 ]]; then + - if [[ "{ $TRAVIS_PYTHON_VERSION | cut -c 3 } -ge 6" ]]; then git clone https://github.com/balsam-alcf/balsam.git ../balsam; cd ../balsam; pip install -e .; cd ../libensemble; mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; From 998dd22164e0abff7c577669131b4cb57b63c634 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 12:08:52 -0500 Subject: [PATCH 095/644] Moved python version comparision, balsam installation to external py script. --- .travis.yml | 13 +++++++------ .../{balsam-setup.sh => configure-balsam-test.sh} | 0 conda/install-balsam.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) rename conda/{balsam-setup.sh => configure-balsam-test.sh} (100%) create mode 100644 conda/install-balsam.py diff --git a/.travis.yml b/.travis.yml index f6461ae7a..de9db6390 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,12 +84,13 @@ install: - python conda/find_mpi.py # locate compilers - mpiexec --version # Show MPI library details - pip install -e . - - if [[ "{ $TRAVIS_PYTHON_VERSION | cut -c 3 } -ge 6" ]]; then - git clone https://github.com/balsam-alcf/balsam.git ../balsam; - cd ../balsam; pip install -e .; cd ../libensemble; - mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; - ./conda/balsam-setup.sh; - fi + - python conda/install-balsam.py + # - if [[ "{ $TRAVIS_PYTHON_VERSION | cut -c 3 } -ge 6" ]]; then + # git clone https://github.com/balsam-alcf/balsam.git ../balsam; + # cd ../balsam; pip install -e .; cd ../libensemble; + # mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; + # ./conda/balsam-setup.sh; + # fi before_script: diff --git a/conda/balsam-setup.sh b/conda/configure-balsam-test.sh similarity index 100% rename from conda/balsam-setup.sh rename to conda/configure-balsam-test.sh diff --git a/conda/install-balsam.py b/conda/install-balsam.py new file mode 100644 index 000000000..c7aaf4d44 --- /dev/null +++ b/conda/install-balsam.py @@ -0,0 +1,14 @@ +import sys +import os +import subprocess + +balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' + +if sys.version[2] >= 6: + subprocess.check_call(balsamclone.split()) + os.chdir('../balsam') + subprocess.check_call('pip install -e .'.split()) + os.chdir('../libensemble') + os.rename('./conda/test_balsam.py', + './libensemble/tests/regression_tests/test_balsam.py') + subprocess.check_call('./conda/configure-balsam-test.sh') From 68919a4bef99ed6e24e24f2d2de57fbb07385d3e Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 12:13:44 -0500 Subject: [PATCH 096/644] fixes version check --- conda/install-balsam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index c7aaf4d44..df8356541 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -4,11 +4,11 @@ balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' -if sys.version[2] >= 6: +if int(sys.version[2]) >= 6: subprocess.check_call(balsamclone.split()) os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) os.chdir('../libensemble') - os.rename('./conda/test_balsam.py', + os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') subprocess.check_call('./conda/configure-balsam-test.sh') From 7ec42ec5eb0f69070133cfd83cdf44505291e5d7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 13:03:30 -0500 Subject: [PATCH 097/644] moved test to correct starting directory --- {libensemble/tests/regression_tests => conda}/test_balsam.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {libensemble/tests/regression_tests => conda}/test_balsam.py (100%) diff --git a/libensemble/tests/regression_tests/test_balsam.py b/conda/test_balsam.py similarity index 100% rename from libensemble/tests/regression_tests/test_balsam.py rename to conda/test_balsam.py From 4cf3f471fbe78bea4266993cee0cd140504c650b Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 13:26:10 -0500 Subject: [PATCH 098/644] implements install-balsam script in build_mpich script. fixes check_call --- conda/install-balsam.py | 2 +- conda/run_travis_locally/build_mpich_libE.sh | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index df8356541..e4e64ba57 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,4 +11,4 @@ os.chdir('../libensemble') os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - subprocess.check_call('./conda/configure-balsam-test.sh') + subprocess.check_call(['./conda/configure-balsam-test.sh']) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 47d50ff4e..b5c4361dd 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -11,7 +11,6 @@ export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH export HYDRA_LAUNCHER=fork -export TESTBALSAM=true # Allow user to optionally set python version and branch # E.g: ". ./build_mpich_libE.sh -p 3.4 -b feature/myfeature" @@ -86,7 +85,6 @@ conda install nlopt petsc4py petsc mumps-mpi=5.1.2=h5bebb2f_1007 mpi4py scipy $M # conda install nlopt || return # pip install these as the conda installs downgrade pytest on python3.4 -pip install balsam-flow || return pip install pytest || return pip install pytest-cov || return pip install pytest-timeout || return @@ -97,10 +95,7 @@ pip install coveralls || return git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || return cd libensemble/ || return pip install -e . || return - -if [[ "$TESTBALSAM" == true ]]; then - source ./conda/balsam-setup.sh; -fi +python conda/install-balsam.py libensemble/tests/run-tests.sh From 7544c7f18decc580874f48b723857c0a179d9554 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 13:45:42 -0500 Subject: [PATCH 099/644] check_call() -> os.system() --- conda/install-balsam.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index e4e64ba57..8c0645b0f 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + import sys import os import subprocess @@ -11,4 +13,4 @@ os.chdir('../libensemble') os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - subprocess.check_call(['./conda/configure-balsam-test.sh']) + os.system('./conda/configure-balsam-test.sh') From 2aac867eaea207b486a9f9bb41cc8d711282f70d Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 13:55:27 -0500 Subject: [PATCH 100/644] bang bin bash --- conda/configure-balsam-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index aa658cb19..7b1d6b99f 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -1,3 +1,5 @@ +#!/bin/bash + export BALSAM_DB_PATH='~/test-balsam' sudo chown -R postgres:postgres /var/run/postgresql sudo chmod a+w /var/run/postgresql From 12fc6b92a8b6463e38cc94f849fb34e7dee0a0ba Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 14:21:22 -0500 Subject: [PATCH 101/644] trying to ensure that BALSAM_DB_PATH stays set --- .travis.yml | 2 ++ conda/install-balsam.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index de9db6390..a2e0bffe4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,6 +95,8 @@ install: before_script: - flake8 libensemble + - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh + - source setbalsampath.sh # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 8c0645b0f..46534168e 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -13,4 +13,4 @@ os.chdir('../libensemble') os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - os.system('./conda/configure-balsam-test.sh') + os.system('source ./conda/configure-balsam-test.sh') From b05286809f64bcac0d8b90ebd1f40625f881b3a1 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 14:26:28 -0500 Subject: [PATCH 102/644] remove redundant source (?) --- conda/install-balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 46534168e..8c0645b0f 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -13,4 +13,4 @@ os.chdir('../libensemble') os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - os.system('source ./conda/configure-balsam-test.sh') + os.system('./conda/configure-balsam-test.sh') From bf495c8e029356ab3aec0f822fc3cdbb685e1c99 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 13 Aug 2019 15:39:24 -0500 Subject: [PATCH 103/644] Updating persistent_uniform_sampling call --- libensemble/gen_funcs/persistent_aposmm.py | 3 +++ .../test_6-hump_camel_persistent_uniform_sampling.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 0952f8298..5746b5364 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -192,6 +192,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): """ # {{{ multiprocessing initialization + # from libensemble.util.forkpdb import ForkablePdb + # ForkablePdb().set_trace() + local_opters = [] parent_can_read_from_queue = Event() comm_queue = Queue() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py index c64bb672f..c0eda0790 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py @@ -41,11 +41,11 @@ 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} -alloc_specs = {'alloc_f': alloc_f, 'out': []} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300} +exit_criteria = {'gen_max': 40, 'elapsed_wallclock_time': 300} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 87c859ee83639e29c264b6ec316d85d7c04c7c5a Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 16:11:38 -0500 Subject: [PATCH 104/644] fix build simfunc path --- conda/run_travis_locally/build_mpich_libE.sh | 1 + libensemble/tests/regression_tests/script_test_balsam.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index b5c4361dd..2efe9243e 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -96,6 +96,7 @@ git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || retu cd libensemble/ || return pip install -e . || return python conda/install-balsam.py +export BALSAM_DB_PATH=~/test-balsam libensemble/tests/run-tests.sh diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 05f413f3b..46c98ed19 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -20,7 +20,8 @@ def build_simfunc(): import subprocess # Build simfunc # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn - buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' + # buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' + buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) From 85fce8c369c93fdcce1d79cf43e583a1dd2f8c34 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 13 Aug 2019 16:50:39 -0500 Subject: [PATCH 105/644] check if test_balsam already moved, more verbose tests, error checking --- conda/install-balsam.py | 5 +++-- conda/run_travis_locally/build_mpich_libE.sh | 2 +- libensemble/tests/regression_tests/script_test_balsam.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 8c0645b0f..40489521d 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,6 +11,7 @@ os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) os.chdir('../libensemble') - os.rename('./conda/test_balsam.py', - './libensemble/tests/regression_tests/test_balsam.py') + if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'): + os.rename('./conda/test_balsam.py', + './libensemble/tests/regression_tests/test_balsam.py') os.system('./conda/configure-balsam-test.sh') diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 2efe9243e..6a560a77f 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -98,7 +98,7 @@ pip install -e . || return python conda/install-balsam.py export BALSAM_DB_PATH=~/test-balsam -libensemble/tests/run-tests.sh +libensemble/tests/run-tests.sh -z echo -e "\n\nScript completed...\n\n" set +ex diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 46c98ed19..30fc8575b 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -18,6 +18,7 @@ def build_simfunc(): import subprocess + print('Balsam job launched in: {}'.format(os.getcwd())) # Build simfunc # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn # buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' From 468777cbebcca62ed704c2fe3cfc2b989245575d Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 09:26:54 -0500 Subject: [PATCH 106/644] fix buildstring path (again) --- libensemble/tests/regression_tests/script_test_balsam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 30fc8575b..9677908a0 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -21,8 +21,8 @@ def build_simfunc(): print('Balsam job launched in: {}'.format(os.getcwd())) # Build simfunc # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn - # buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' - buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' + buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' + # buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) From c076a0cce2c6ef135c906dc3387188bd79b50c78 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 10:12:12 -0500 Subject: [PATCH 107/644] parse_args to address errors when running through run-tests --- conda/test_balsam.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 016588960..2bb2061c9 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,7 +1,7 @@ import subprocess import os import balsam - +from libensemble.tests.regression_tests.common import parse_args # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which usually # only contain a single node, we need to change Balsam's default worker setup @@ -13,6 +13,8 @@ # MODIFY BALSAM WORKERGROUP +nworkers, is_master, libE_specs, _ = parse_args() # None of these will be used here + workerfile = 'worker.py' home = os.getcwd() From 73ba84e572b447be6c70b2f9c84e924ccc7224be Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 10:21:50 -0500 Subject: [PATCH 108/644] flake8 and comments --- conda/test_balsam.py | 5 ++++- libensemble/tests/regression_tests/script_test_balsam.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 2bb2061c9..83b2435fe 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -2,6 +2,7 @@ import os import balsam from libensemble.tests.regression_tests.common import parse_args + # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which usually # only contain a single node, we need to change Balsam's default worker setup @@ -13,7 +14,7 @@ # MODIFY BALSAM WORKERGROUP -nworkers, is_master, libE_specs, _ = parse_args() # None of these will be used here +nworkers, is_master, libE_specs, _ = parse_args() # None of these will be used here workerfile = 'worker.py' @@ -37,6 +38,8 @@ os.chdir(home) # EXECUTE BALSAM JOB +# By this point, script_test_balsam.py has been submitted as an app and job to Balsam +# This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' subprocess.check_call(runstr.split()) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 9677908a0..dd99e0a9b 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -1,4 +1,6 @@ -# This script is meant to be submitted to Balsam for execution +# This script is submitted as an app to Balsam, and jobs are launched +# based on this app. This submission is via 'balsam launch' executed +# in the test_balsam.py script. import os import numpy as np From 48810710e098c57da1f697b7ac973f5bae419355 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 11:50:09 -0500 Subject: [PATCH 109/644] build script back to original settings, new subprocess call function attempt --- conda/run_travis_locally/build_mpich_libE.sh | 4 ++-- conda/test_balsam.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 6a560a77f..aab538e09 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -5,8 +5,8 @@ # -x echo commands set -x # problem with this - loads of conda internal commands shown - overwhelming. -export PYTHON_VERSION=3.6 # default - override with -p -export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b +export PYTHON_VERSION=3.7 # default - override with -p +export LIBE_BRANCH="develop" # default - override with -b export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 83b2435fe..c0933cece 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -14,7 +14,8 @@ # MODIFY BALSAM WORKERGROUP -nworkers, is_master, libE_specs, _ = parse_args() # None of these will be used here +# None of these will be used here. Preventative measure against run-tests bugs +nworkers, is_master, libE_specs, _ = parse_args() workerfile = 'worker.py' @@ -42,4 +43,4 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' -subprocess.check_call(runstr.split()) +subprocess.check_output(runstr.split()) From d97fa1b02959381a5303c551936a7cac091f0350 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 15:07:07 -0500 Subject: [PATCH 110/644] source testing script for env var availability --- .travis.yml | 2 +- conda/run_travis_locally/build_mpich_libE.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2e0bffe4..3dab30afc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ before_script: # Run test (-z show output) script: - - libensemble/tests/run-tests.sh -z + - source libensemble/tests/run-tests.sh -z # Coverage after_success: diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index aab538e09..960744597 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -98,7 +98,7 @@ pip install -e . || return python conda/install-balsam.py export BALSAM_DB_PATH=~/test-balsam -libensemble/tests/run-tests.sh -z +source libensemble/tests/run-tests.sh -z echo -e "\n\nScript completed...\n\n" set +ex From 69a2bfc8ce52833958f6f2caaa3829415a606744 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 15:27:34 -0500 Subject: [PATCH 111/644] cleanup and comments --- .travis.yml | 7 --- conda/configure-balsam-test.sh | 23 +++++-- conda/test_balsam.py | 63 ++++++++++--------- .../regression_tests/script_test_balsam.py | 24 +------ 4 files changed, 52 insertions(+), 65 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3dab30afc..c01b6bc76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,13 +85,6 @@ install: - mpiexec --version # Show MPI library details - pip install -e . - python conda/install-balsam.py - # - if [[ "{ $TRAVIS_PYTHON_VERSION | cut -c 3 } -ge 6" ]]; then - # git clone https://github.com/balsam-alcf/balsam.git ../balsam; - # cd ../balsam; pip install -e .; cd ../libensemble; - # mv ./conda/test_balsam.py ./libensemble/tests/regression_tests; - # ./conda/balsam-setup.sh; - # fi - before_script: - flake8 libensemble diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index 7b1d6b99f..bce2ed440 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -1,11 +1,11 @@ #!/bin/bash -export BALSAM_DB_PATH='~/test-balsam' -sudo chown -R postgres:postgres /var/run/postgresql -sudo chmod a+w /var/run/postgresql -balsam init ~/test-balsam -sudo chmod -R 700 ~/test-balsam/balsamdb -source balsamactivate test-balsam +# Run this script between each repeated local run of the Balsam +# job_control_hworld_balsam test. Besides ensuring that postgres and the +# generated Balsam db have the proper permissions, this script flushes previous +# apps and jobs in the db before submitting the test_balsam app/job script. +# +# Most of this comes from scaling_tests/forces/balsam_local.sh export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py export NUM_WORKERS=2 @@ -14,9 +14,20 @@ export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD export SCRIPT_BASENAME=script_test_balsam +# Set proper permissions, initialize Balsam DB, activate DB +export BALSAM_DB_PATH='~/test-balsam' +sudo chown -R postgres:postgres /var/run/postgresql +sudo chmod a+w /var/run/postgresql +balsam init ~/test-balsam +sudo chmod -R 700 ~/test-balsam/balsamdb +source balsamactivate test-balsam + +# Refresh DB balsam rm apps --all --force balsam rm jobs --all --force +# Submit script_test_balsam as app balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" +# Submit job based on script_test_balsam app balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes diff --git a/conda/test_balsam.py b/conda/test_balsam.py index c0933cece..07e92c759 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -3,44 +3,49 @@ import balsam from libensemble.tests.regression_tests.common import parse_args -# Balsam is meant for HPC systems that commonly distribute jobs across many -# nodes. Due to the nature of testing Balsam on local or CI systems which usually -# only contain a single node, we need to change Balsam's default worker setup -# so multiple workers can be run on a single node (until this feature is [hopefully] added!). -# For our purposes, we append ten workers to Balsam's WorkerGroup - # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 -# MODIFY BALSAM WORKERGROUP +new_lines = [" for idx in range(10):\n", + " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", + " self.workers.append(w)\n"] + + +def modify_Balsam_worker(): + # Balsam is meant for HPC systems that commonly distribute jobs across many + # nodes. Due to the nature of testing Balsam on local or CI systems which usually + # only contain a single node, we need to change Balsam's default worker setup + # so multiple workers can be run on a single node (until this feature is [hopefully] added!). + # For our purposes, we append ten workers to Balsam's WorkerGroup -# None of these will be used here. Preventative measure against run-tests bugs -nworkers, is_master, libE_specs, _ = parse_args() + workerfile = 'worker.py' + home = os.getcwd() + balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' + os.chdir(balsam_worker_path) -workerfile = 'worker.py' + with open(workerfile, 'r') as f: + lines = f.readlines() -home = os.getcwd() -balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' -os.chdir(balsam_worker_path) + if lines[-3] != new_lines[0]: + lines = lines[:-2] # effectively inserting new_lines[0] above + lines.extend(new_lines) -with open(workerfile, 'r') as f: - lines = f.readlines() + with open(workerfile, 'w') as f: + for line in lines: + f.write(line) -if lines[-3] != " for idx in range(10):\n": - lines = lines[:-2] # Will re-add these lines - lines.extend([" for idx in range(10):\n", - " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", - " self.workers.append(w)\n"]) + os.chdir(home) -with open(workerfile, 'w') as f: - for line in lines: - f.write(line) -os.chdir(home) +def run_Balsam_job(): + # Executes Balsam Job + # By this point, script_test_balsam.py has been submitted as an app and job to Balsam + # This line launches the queued job in the Balsam database + runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' + subprocess.check_output(runstr.split()) -# EXECUTE BALSAM JOB -# By this point, script_test_balsam.py has been submitted as an app and job to Balsam -# This line launches the queued job in the Balsam database -runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' -subprocess.check_output(runstr.split()) +if __name__ == '__main__': + nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention + modify_Balsam_worker() + run_Balsam_job() diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index dd99e0a9b..ba3a15ff9 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -21,11 +21,7 @@ def build_simfunc(): import subprocess print('Balsam job launched in: {}'.format(os.getcwd())) - # Build simfunc - # buildstring='mpif90 -o my_simjob.x my_simjob.f90' # On cray need to use ftn buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' - # buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' - # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) @@ -40,15 +36,8 @@ def build_simfunc(): logical_cores = multiprocessing.cpu_count() cores_all_jobs = nworkers*cores_per_job -if cores_all_jobs > logical_cores: - use_auto_resources = False - mess_resources = 'Oversubscribing - auto_resources set to False' -else: - use_auto_resources = True - mess_resources = 'Auto_resources set to True' - if is_master: - print('\nCores req: {} Cores avail: {}\n {}\n'.format(cores_all_jobs, logical_cores, mess_resources)) + print('\nCores req: {} Cores avail: {}\n'.format(cores_all_jobs, logical_cores)) sim_app = './my_simjob.x' if not os.path.isfile(sim_app): @@ -106,16 +95,5 @@ def build_simfunc(): # Repeat N times for N workers and insert Completed at start for generator calc_desc_list = ['Completed'] + calc_desc_list_in*nworkers - # script_name = os.path.splitext(os.path.basename(__file__))[0] - # short_name = script_name.split("test_", 1).pop() - # summary_file_name = short_name + '.libe_summary.txt' - # with open(summary_file_name,'r') as f: - # i=0 - # for line in f: - # if "Status:" in line: - # _, file_status = line.partition("Status:")[::2] - # print("Expected: {} Filestatus: {}".format(calc_desc_list[i], file_status.strip())) - # assert calc_desc_list[i] == file_status.strip(), "Status does not match file" - # i+=1 print("\n\n\nRun completed.") From 10b79bec87a198e9bda8d9a5e47e690910406181 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 14 Aug 2019 15:32:07 -0500 Subject: [PATCH 112/644] trying different calling method --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 07e92c759..304827d50 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -42,7 +42,7 @@ def run_Balsam_job(): # By this point, script_test_balsam.py has been submitted as an app and job to Balsam # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' - subprocess.check_output(runstr.split()) + subprocess.call(runstr.split()) if __name__ == '__main__': From 3aed4ccf6ad9bfb3996c79f169232c6c08bdfb20 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 09:41:19 -0500 Subject: [PATCH 113/644] reorganize test_balsam, adds more descriptive print statements --- conda/run_travis_locally/build_mpich_libE.sh | 2 +- conda/test_balsam.py | 58 +++++++++----------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 960744597..34dddabbf 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -6,7 +6,7 @@ set -x # problem with this - loads of conda internal commands shown - overwhelming. export PYTHON_VERSION=3.7 # default - override with -p -export LIBE_BRANCH="develop" # default - override with -b +export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 304827d50..1097027e0 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -10,42 +10,38 @@ " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", " self.workers.append(w)\n"] +nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention -def modify_Balsam_worker(): - # Balsam is meant for HPC systems that commonly distribute jobs across many - # nodes. Due to the nature of testing Balsam on local or CI systems which usually - # only contain a single node, we need to change Balsam's default worker setup - # so multiple workers can be run on a single node (until this feature is [hopefully] added!). - # For our purposes, we append ten workers to Balsam's WorkerGroup - workerfile = 'worker.py' - home = os.getcwd() - balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' - os.chdir(balsam_worker_path) +# Balsam is meant for HPC systems that commonly distribute jobs across many +# nodes. Due to the nature of testing Balsam on local or CI systems which usually +# only contain a single node, we need to change Balsam's default worker setup +# so multiple workers can be run on a single node (until this feature is [hopefully] added!). +# For our purposes, we append ten workers to Balsam's WorkerGroup +print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd())) +workerfile = 'worker.py' +home = os.getcwd() +balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' +os.chdir(balsam_worker_path) - with open(workerfile, 'r') as f: - lines = f.readlines() +with open(workerfile, 'r') as f: + lines = f.readlines() - if lines[-3] != new_lines[0]: - lines = lines[:-2] # effectively inserting new_lines[0] above - lines.extend(new_lines) +if lines[-3] != new_lines[0]: + lines = lines[:-2] # effectively inserting new_lines[0] above + lines.extend(new_lines) - with open(workerfile, 'w') as f: - for line in lines: - f.write(line) +with open(workerfile, 'w') as f: + for line in lines: + f.write(line) - os.chdir(home) +print("Modified worker file in {}".format(os.getcwd())) +os.chdir(home) -def run_Balsam_job(): - # Executes Balsam Job - # By this point, script_test_balsam.py has been submitted as an app and job to Balsam - # This line launches the queued job in the Balsam database - runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' - subprocess.call(runstr.split()) - - -if __name__ == '__main__': - nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention - modify_Balsam_worker() - run_Balsam_job() +# Executes Balsam Job +# By this point, script_test_balsam.py has been submitted as an app and job to Balsam +# This line launches the queued job in the Balsam database +runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' +print('Executing Balsam job with command: {}'.format(runstr)) +subprocess.call(runstr.split()) From de3a552cf68bc80d3e9a16fd847c4853360da23b Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 10:45:37 -0500 Subject: [PATCH 114/644] trying sleep --- conda/test_balsam.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 1097027e0..2810af145 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,6 +1,7 @@ import subprocess import os import balsam +import time from libensemble.tests.regression_tests.common import parse_args # TESTSUITE_COMMS: local @@ -12,6 +13,8 @@ nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention +print('Sleeping for a couple seconds (just in case)') +time.sleep(5) # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which usually @@ -45,3 +48,5 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) subprocess.call(runstr.split()) + +os.chdir('~/test-balsam/data/') From 5eece5aa7397ec8c78d35beabc7a15f9724916fe Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 12:13:49 -0500 Subject: [PATCH 115/644] modify Balsam worker moved to common. Adding test output from DB output directory --- conda/test_balsam.py | 32 +++++--------------- libensemble/tests/regression_tests/common.py | 32 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 2810af145..c2c128826 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -2,15 +2,11 @@ import os import balsam import time -from libensemble.tests.regression_tests.common import parse_args +from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 -new_lines = [" for idx in range(10):\n", - " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", - " self.workers.append(w)\n"] - nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention print('Sleeping for a couple seconds (just in case)') @@ -22,25 +18,7 @@ # so multiple workers can be run on a single node (until this feature is [hopefully] added!). # For our purposes, we append ten workers to Balsam's WorkerGroup print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd())) -workerfile = 'worker.py' -home = os.getcwd() -balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' -os.chdir(balsam_worker_path) - -with open(workerfile, 'r') as f: - lines = f.readlines() - -if lines[-3] != new_lines[0]: - lines = lines[:-2] # effectively inserting new_lines[0] above - lines.extend(new_lines) - -with open(workerfile, 'w') as f: - for line in lines: - f.write(line) - -print("Modified worker file in {}".format(os.getcwd())) -os.chdir(home) - +modify_Balsam_worker() # Executes Balsam Job # By this point, script_test_balsam.py has been submitted as an app and job to Balsam @@ -49,4 +27,8 @@ print('Executing Balsam job with command: {}'.format(runstr)) subprocess.call(runstr.split()) -os.chdir('~/test-balsam/data/') +os.chdir('~/test-balsam/data/libe_test-balsam/job_script_test_balsam_*') +with open('job_script_test_balsam.out', 'r') as f: + lines = f.readlines() +for line in lines: + print(line) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 1b21883aa..41fc4bbd7 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -170,3 +170,35 @@ def build_simfunc(): buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c' # subprocess.run(buildstring.split(),check=True) #Python3.5+ subprocess.check_call(buildstring.split()) + + +def modify_Balsam_worker(): + # Balsam is meant for HPC systems that commonly distribute jobs across many + # nodes. Due to the nature of testing Balsam on local or CI systems which usually + # only contain a single node, we need to change Balsam's default worker setup + # so multiple workers can be run on a single node (until this feature is [hopefully] added!). + # For our purposes, we append ten workers to Balsam's WorkerGroup + import balsam + + new_lines = [" for idx in range(10):\n", + " w = Worker(1, host_type='DEFAULT', num_nodes=1)\n", + " self.workers.append(w)\n"] + + workerfile = 'worker.py' + home = os.getcwd() + balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' + os.chdir(balsam_worker_path) + + with open(workerfile, 'r') as f: + lines = f.readlines() + + if lines[-3] != new_lines[0]: + lines = lines[:-2] # effectively inserting new_lines[0] above + lines.extend(new_lines) + + with open(workerfile, 'w') as f: + for line in lines: + f.write(line) + + print("Modified worker file in {}".format(os.getcwd())) + os.chdir(home) From 65f1bd63fee076303ca40959acc503b7a3cdac31 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 12:22:18 -0500 Subject: [PATCH 116/644] flake8 --- conda/test_balsam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index c2c128826..cee4ed886 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,6 +1,5 @@ import subprocess import os -import balsam import time from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker From 39819bff8220716cc58a30817b6e20c3650cd865 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 13:18:27 -0500 Subject: [PATCH 117/644] print call_process error messages (if any) --- conda/test_balsam.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index cee4ed886..16e7cb540 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -24,10 +24,17 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -subprocess.call(runstr.split()) +try: + subprocess.check_output(runstr.split()) +except subprocess.CalledProcessError as e: + print e.output + +curdir = os.getcwd() os.chdir('~/test-balsam/data/libe_test-balsam/job_script_test_balsam_*') with open('job_script_test_balsam.out', 'r') as f: lines = f.readlines() for line in lines: print(line) + +os.chdir(curdir) From d9858b45cc8b53ff7401ebc3a19c9e0e7ba4e730 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 13:29:00 -0500 Subject: [PATCH 118/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 16e7cb540..42e081494 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -27,7 +27,7 @@ try: subprocess.check_output(runstr.split()) except subprocess.CalledProcessError as e: - print e.output + print(e.output) curdir = os.getcwd() From 634943cda3c4d95113ec3442cbf4d3ca4abfe511 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 14:28:19 -0500 Subject: [PATCH 119/644] remove sleep, attempt reading file in home libe dir --- conda/test_balsam.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 42e081494..3e29f2f49 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -8,9 +8,6 @@ nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention -print('Sleeping for a couple seconds (just in case)') -time.sleep(5) - # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which usually # only contain a single node, we need to change Balsam's default worker setup @@ -29,12 +26,8 @@ except subprocess.CalledProcessError as e: print(e.output) -curdir = os.getcwd() - -os.chdir('~/test-balsam/data/libe_test-balsam/job_script_test_balsam_*') -with open('job_script_test_balsam.out', 'r') as f: +print('Attempting to print Balsam job output') +with open('../../../job_script_test_balsam.out', 'r') as f: lines = f.readlines() for line in lines: print(line) - -os.chdir(curdir) From bfa6ce2bd17dab3426c3f4c0df6d06210ad60afa Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 14:33:46 -0500 Subject: [PATCH 120/644] flake8 --- conda/test_balsam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 3e29f2f49..740414475 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,6 +1,5 @@ import subprocess import os -import time from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker # TESTSUITE_COMMS: local From 8f978f042632485c081fb2702d3d0ecc964d3933 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 15:08:20 -0500 Subject: [PATCH 121/644] newer calling function --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 740414475..dc1385624 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -21,7 +21,7 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) try: - subprocess.check_output(runstr.split()) + subprocess.run(runstr.split(), check=True) except subprocess.CalledProcessError as e: print(e.output) From e47bfd8e5017a587b49b414447ff699a5360df99 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 15:21:01 -0500 Subject: [PATCH 122/644] attempting without try block --- conda/test_balsam.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index dc1385624..cd667d566 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -20,10 +20,12 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -try: - subprocess.run(runstr.split(), check=True) -except subprocess.CalledProcessError as e: - print(e.output) +# try: +# subprocess.run(runstr.split(), check=True) +# except subprocess.CalledProcessError as cpe: +# print(cpe.output) + +subprocess.run(runstr.split()) print('Attempting to print Balsam job output') with open('../../../job_script_test_balsam.out', 'r') as f: From 1deefc8050a42f22c46dc061a581d43287dc6fe3 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 15:42:06 -0500 Subject: [PATCH 123/644] Trying longer timeout limit. More error checking, subprocess instead of os.system --- conda/install-balsam.py | 3 ++- conda/test_balsam.py | 22 ++++++++++------------ libensemble/tests/run-tests.sh | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 40489521d..2e18a9938 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -14,4 +14,5 @@ if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'): os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - os.system('./conda/configure-balsam-test.sh') + # os.system('./conda/configure-balsam-test.sh') + subprocess.run('./conda/configure-balsam-test.sh'.split()) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index cd667d566..1c2dce9b7 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -20,15 +20,13 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -# try: -# subprocess.run(runstr.split(), check=True) -# except subprocess.CalledProcessError as cpe: -# print(cpe.output) - -subprocess.run(runstr.split()) - -print('Attempting to print Balsam job output') -with open('../../../job_script_test_balsam.out', 'r') as f: - lines = f.readlines() -for line in lines: - print(line) +try: + subprocess.run(runstr.split(), check=True) +except subprocess.CalledProcessError as cpe: + print(cpe.output) +else: + print('Attempting to print Balsam job output') + with open('../../../job_script_test_balsam.out', 'r') as f: + lines = f.readlines() + for line in lines: + print(line) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index a48bff814..9a021e11e 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 60s" + TIMEOUT="timeout 90s" fi #Build any sim/gen source code dependencies here ..... From e103c1e2c2564f5ad098a17e0f464cd0538bc623 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 16:00:32 -0500 Subject: [PATCH 124/644] more timing changes --- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- libensemble/tests/run-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index ba3a15ff9..cbafad8ad 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -64,7 +64,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 30} +exit_criteria = {'elapsed_wallclock_time': 60} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 9a021e11e..7b363dc81 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 90s" + TIMEOUT="timeout --preserve-status 60s" fi #Build any sim/gen source code dependencies here ..... From 15b7b9e7dba7f8e3aee6a817354626996f479ca7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 16:02:27 -0500 Subject: [PATCH 125/644] adds verbosity option to timeout line --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 7b363dc81..0e31e0e30 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout --preserve-status 60s" + TIMEOUT="timeout --preserve-status --verbose 60s" fi #Build any sim/gen source code dependencies here ..... From 94ac6e0a1142ed371bbea4d5cf3bdf04e6387661 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 16:12:50 -0500 Subject: [PATCH 126/644] removes 'verbose', apparently not supported on the included version of timeout --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 0e31e0e30..7b363dc81 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout --preserve-status --verbose 60s" + TIMEOUT="timeout --preserve-status 60s" fi #Build any sim/gen source code dependencies here ..... From a814aa872d081b0888de642090a0473e653a476d Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 16:35:12 -0500 Subject: [PATCH 127/644] timeout increased again, removes preserve status --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 7b363dc81..9a021e11e 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout --preserve-status 60s" + TIMEOUT="timeout 90s" fi #Build any sim/gen source code dependencies here ..... From 7ea782674d8051fb5a97545391f30f98e80851d3 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 15 Aug 2019 16:55:16 -0500 Subject: [PATCH 128/644] adjusts balsam job script test timing, better checks for output file --- conda/test_balsam.py | 11 ++++++++--- .../tests/regression_tests/script_test_balsam.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 1c2dce9b7..5bfe42e21 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -24,9 +24,14 @@ subprocess.run(runstr.split(), check=True) except subprocess.CalledProcessError as cpe: print(cpe.output) -else: - print('Attempting to print Balsam job output') - with open('../../../job_script_test_balsam.out', 'r') as f: + +print('Attempting to print Balsam job output') +outfile = '../../../job_script_test_balsam.out' +if os.path.isfile(outfile): + print('Balsam job output found!') + with open(outfile, 'r') as f: lines = f.readlines() for line in lines: print(line) +else: + print('No job output found! Check the Balsam DB?') diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index cbafad8ad..ba3a15ff9 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -64,7 +64,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 60} +exit_criteria = {'elapsed_wallclock_time': 30} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 2eb280076c2e845d01fa2afd6f4aedc6996a2e33 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 16 Aug 2019 01:13:50 -0500 Subject: [PATCH 129/644] do not allocate memory for sim_id checking --- libensemble/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/history.py b/libensemble/history.py index 22f3907e9..eb27fe833 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -153,7 +153,7 @@ def update_history_x_in(self, gen_worker, O): # gen method is building sim_id or adjusting values in existing sim_id rows. # Ensure there aren't any gaps in the generated sim_id values: - assert np.all(np.in1d(np.arange(self.index, np.max(O['sim_id'])+1), O['sim_id'])), "The generator function has produced sim_id that are not in order." + assert all(i in O['sim_id'] for i in range(self.index, np.max(O['sim_id'])+1)), "The generator function has produced sim_id that are not in order." num_new = len(np.setdiff1d(O['sim_id'], self.H['sim_id'])) From 3956d32f8ae82f2a492a5bff7eacfbc94a6d923c Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 16 Aug 2019 03:15:23 -0500 Subject: [PATCH 130/644] adding debug statements removes error? --- libensemble/gen_funcs/support.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index e5084828d..51a6ad065 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,6 +11,8 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ + print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", + flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From 54b6875533f3ae7dd67b5415639792f9e8713a5f Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 16 Aug 2019 03:22:30 -0500 Subject: [PATCH 131/644] placate flake8 --- libensemble/gen_funcs/support.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 51a6ad065..ed8335a57 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,8 +11,7 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ - print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", - flush=True) + print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From 0afb244f7d733b94b357b9f6ad6e269470d4a14f Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Fri, 16 Aug 2019 03:35:42 -0500 Subject: [PATCH 132/644] guards the print statement --- libensemble/gen_funcs/support.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index ed8335a57..0ae971e7a 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,7 +11,8 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ - print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", flush=True) + if 'sim_id' in O.dtype.names: + print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From e35e4d7e89849bee0df13bdcf84965ee128ee8d9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 09:35:56 -0500 Subject: [PATCH 133/644] trying additional timing adjustment for timeout --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 9a021e11e..a52a08ca2 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 90s" + TIMEOUT="timeout 120s" fi #Build any sim/gen source code dependencies here ..... From 75ef199179fd86f216687539de64fd7c38d8a351 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 09:55:03 -0500 Subject: [PATCH 134/644] trying 10 whole minutes... trying job execution without check --- conda/test_balsam.py | 16 +++++++++++----- libensemble/tests/run-tests.sh | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 5bfe42e21..f7744ae63 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,5 +1,6 @@ import subprocess import os +import time from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker # TESTSUITE_COMMS: local @@ -18,12 +19,17 @@ # Executes Balsam Job # By this point, script_test_balsam.py has been submitted as an app and job to Balsam # This line launches the queued job in the Balsam database -runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' +runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 &' print('Executing Balsam job with command: {}'.format(runstr)) -try: - subprocess.run(runstr.split(), check=True) -except subprocess.CalledProcessError as cpe: - print(cpe.output) +# try: +# subprocess.run(runstr.split(), check=True) +# except subprocess.CalledProcessError as cpe: +# print(cpe.output) + +subprocess.run(runstr.split()) + +print('Sleeping until the job should complete') +time.sleep(31) print('Attempting to print Balsam job output') outfile = '../../../job_script_test_balsam.out' diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index a52a08ca2..4bc69eedf 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 120s" + TIMEOUT="timeout 600s" fi #Build any sim/gen source code dependencies here ..... From 7c12b509fe41c171910ad46ac12b13cd03e61d0f Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 10:07:31 -0500 Subject: [PATCH 135/644] Attempting Popen so test_balsam doesn't wait on balsam launcher to complete --- conda/test_balsam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index f7744ae63..239e8df76 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -19,16 +19,16 @@ # Executes Balsam Job # By this point, script_test_balsam.py has been submitted as an app and job to Balsam # This line launches the queued job in the Balsam database -runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 &' +runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) # try: # subprocess.run(runstr.split(), check=True) # except subprocess.CalledProcessError as cpe: # print(cpe.output) -subprocess.run(runstr.split()) +subprocess.Popen(runstr.split()) -print('Sleeping until the job should complete') +print('Sleeping until job should complete') time.sleep(31) print('Attempting to print Balsam job output') From 50f7d457759cccea241fa9775e1e5b96198d39d3 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 10:42:01 -0500 Subject: [PATCH 136/644] returns to normal timeout interval. testing cycle of checking for balsam output --- conda/test_balsam.py | 30 +++++++++++++++--------------- libensemble/tests/run-tests.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 239e8df76..47589455f 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -21,23 +21,23 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -# try: -# subprocess.run(runstr.split(), check=True) -# except subprocess.CalledProcessError as cpe: -# print(cpe.output) subprocess.Popen(runstr.split()) -print('Sleeping until job should complete') -time.sleep(31) +print('Beginning cycle of checking for Balsam output') + +sleeptime = 0 -print('Attempting to print Balsam job output') outfile = '../../../job_script_test_balsam.out' -if os.path.isfile(outfile): - print('Balsam job output found!') - with open(outfile, 'r') as f: - lines = f.readlines() - for line in lines: - print(line) -else: - print('No job output found! Check the Balsam DB?') +while sleeptime != 56: + if os.path.isfile(outfile): + print('{}: Balsam job output found!'.format(sleeptime)) + with open(outfile, 'r') as f: + lines = f.readlines() + for line in lines: + print(line) + break + else: + print('{}: No job output found. Checking again.'.format(sleeptime)) + sleeptime += 1 + time.sleep(1) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 4bc69eedf..a48bff814 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 600s" + TIMEOUT="timeout 60s" fi #Build any sim/gen source code dependencies here ..... From 34a6204bb8a0392dae5b283f51d64ae0b09f618f Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:06:08 -0500 Subject: [PATCH 137/644] Trying stdout piping, adjust exit_criteria, check for output every 2 seconds --- conda/test_balsam.py | 8 ++++---- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 47589455f..aa952f6a8 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -22,14 +22,14 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -subprocess.Popen(runstr.split()) +subprocess.Popen(runstr.split(), stderr=stdout, stdout=subprocess.PIPE) print('Beginning cycle of checking for Balsam output') sleeptime = 0 outfile = '../../../job_script_test_balsam.out' -while sleeptime != 56: +while sleeptime != 60: if os.path.isfile(outfile): print('{}: Balsam job output found!'.format(sleeptime)) with open(outfile, 'r') as f: @@ -39,5 +39,5 @@ break else: print('{}: No job output found. Checking again.'.format(sleeptime)) - sleeptime += 1 - time.sleep(1) + sleeptime += 2 + time.sleep(2) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index ba3a15ff9..cbafad8ad 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -64,7 +64,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 30} +exit_criteria = {'elapsed_wallclock_time': 60} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 71b1e05eb6c69ce377796a5a305ffe6197ece42e Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:15:19 -0500 Subject: [PATCH 138/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index aa952f6a8..b10ccf05f 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -22,7 +22,7 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -subprocess.Popen(runstr.split(), stderr=stdout, stdout=subprocess.PIPE) +subprocess.Popen(runstr.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE) print('Beginning cycle of checking for Balsam output') From d58db725219da35e1a544907c8816de8638190b2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:19:31 -0500 Subject: [PATCH 139/644] using communicate to check for process errors or output --- conda/test_balsam.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index b10ccf05f..71d2a82d8 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -22,7 +22,7 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -subprocess.Popen(runstr.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE) +bp = subprocess.Popen(runstr.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE) print('Beginning cycle of checking for Balsam output') @@ -38,6 +38,8 @@ print(line) break else: - print('{}: No job output found. Checking again.'.format(sleeptime)) + print('{}: No job output found. Checking again and checking possible stdout or stderr.'.format(sleeptime)) + outs, errs = bp.communicate() + print('{} {}'.format(out, errs)) sleeptime += 2 time.sleep(2) From 01f0d06fb4eb036c8fc5f6e3b92355b9f24925de Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:20:36 -0500 Subject: [PATCH 140/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 71d2a82d8..f4dfa246c 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -40,6 +40,6 @@ else: print('{}: No job output found. Checking again and checking possible stdout or stderr.'.format(sleeptime)) outs, errs = bp.communicate() - print('{} {}'.format(out, errs)) + print('{} {}'.format(outs, errs)) sleeptime += 2 time.sleep(2) From 54974579ec6ad7632698e5619ecbabfa5b327797 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:36:04 -0500 Subject: [PATCH 141/644] undoing pip and stderr reading --- conda/test_balsam.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index f4dfa246c..6f11c47ce 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -22,7 +22,7 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -bp = subprocess.Popen(runstr.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE) +bp = subprocess.Popen(runstr.split()) print('Beginning cycle of checking for Balsam output') @@ -38,8 +38,6 @@ print(line) break else: - print('{}: No job output found. Checking again and checking possible stdout or stderr.'.format(sleeptime)) - outs, errs = bp.communicate() - print('{} {}'.format(outs, errs)) + print('{}: No job output found. Checking again.'.format(sleeptime)) sleeptime += 2 time.sleep(2) From d68ed3c95cd79b18fa9414abd064868dcce5d4e3 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 11:47:21 -0500 Subject: [PATCH 142/644] timing adjustments --- conda/test_balsam.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 6f11c47ce..058ae9bd7 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -22,14 +22,14 @@ runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) -bp = subprocess.Popen(runstr.split()) +subprocess.Popen(runstr.split()) print('Beginning cycle of checking for Balsam output') sleeptime = 0 outfile = '../../../job_script_test_balsam.out' -while sleeptime != 60: +while sleeptime != 58: if os.path.isfile(outfile): print('{}: Balsam job output found!'.format(sleeptime)) with open(outfile, 'r') as f: @@ -39,5 +39,5 @@ break else: print('{}: No job output found. Checking again.'.format(sleeptime)) - sleeptime += 2 - time.sleep(2) + sleeptime += 1 + time.sleep(1) From 29234762fd10540c509c6cb26fee767491b95f43 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 14:05:46 -0500 Subject: [PATCH 143/644] Improved mechanism to read Balsam DB data dir, wait for various output files. --- conda/test_balsam.py | 23 ++++++++++++++----- .../regression_tests/script_test_balsam.py | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 058ae9bd7..b5f3598f7 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -21,16 +21,25 @@ # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' print('Executing Balsam job with command: {}'.format(runstr)) - subprocess.Popen(runstr.split()) -print('Beginning cycle of checking for Balsam output') +basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam') sleeptime = 0 -outfile = '../../../job_script_test_balsam.out' +while len(os.listdir(basedb)) == 0 and sleeptime != 58: + print('{}: Waiting for Job directory'.format(sleeptime)) + sleeptime += 1 + time.sleep(1) + +# Find output script in Balsam DB +jobdirname = os.listdir(basedb)[0] +jobdir = os.path.join(basedb, jobdirname) +outscript = os.path.join(jobdir, 'job_script_test_balsam.out') + +print('Beginning cycle of checking for Balsam output') while sleeptime != 58: - if os.path.isfile(outfile): + if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) with open(outfile, 'r') as f: lines = f.readlines() @@ -39,5 +48,7 @@ break else: print('{}: No job output found. Checking again.'.format(sleeptime)) - sleeptime += 1 - time.sleep(1) + sleeptime += 2 + time.sleep(2) + +print('Test completed.') diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index cbafad8ad..ba3a15ff9 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -64,7 +64,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 60} +exit_criteria = {'elapsed_wallclock_time': 30} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 5f4b1c8983d82b6c72ba321344dfa358a1d740f4 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 14:08:16 -0500 Subject: [PATCH 144/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index b5f3598f7..4c33cad7f 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -41,7 +41,7 @@ while sleeptime != 58: if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) - with open(outfile, 'r') as f: + with open(outscript, 'r') as f: lines = f.readlines() for line in lines: print(line) From 54a798c97f3754ef62ea6b55d3fcac0e0eef28bb Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 14:42:22 -0500 Subject: [PATCH 145/644] check for additional directory before continuing --- conda/run_travis_locally/build_mpich_libE.sh | 2 +- conda/test_balsam.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 34dddabbf..8a606502e 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -3,7 +3,7 @@ # Note for other MPIs may need to install some packages from source (eg. petsc) # -x echo commands -set -x # problem with this - loads of conda internal commands shown - overwhelming. +# set -x # problem with this - loads of conda internal commands shown - overwhelming. export PYTHON_VERSION=3.7 # default - override with -p export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 4c33cad7f..ad892af1d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -27,17 +27,21 @@ sleeptime = 0 +while not os.path.isdir(basedb) and sleeptime != 58: + print('{}: Waiting for Workflow directory'.format(sleeptime)) + sleeptime += 1 + time.sleep(1) + while len(os.listdir(basedb)) == 0 and sleeptime != 58: - print('{}: Waiting for Job directory'.format(sleeptime)) + print('{}: Waiting for Job Directory'.format(sleeptime)) sleeptime += 1 time.sleep(1) -# Find output script in Balsam DB jobdirname = os.listdir(basedb)[0] jobdir = os.path.join(basedb, jobdirname) outscript = os.path.join(jobdir, 'job_script_test_balsam.out') -print('Beginning cycle of checking for Balsam output') +print('Beginning cycle of checking for Balsam output: {}'.format(outscript)) while sleeptime != 58: if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) From d25d970c72c2f10682a11b8fae8cac89f6747585 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 15:08:30 -0500 Subject: [PATCH 146/644] print every output line as it is written, temporarily disable timeout --- conda/test_balsam.py | 11 ++++++----- libensemble/tests/run-tests.sh | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index ad892af1d..16dea588d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -45,11 +45,12 @@ while sleeptime != 58: if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) - with open(outscript, 'r') as f: - lines = f.readlines() - for line in lines: - print(line) - break + with open(outscript, 'r') as f + while True: + line = f.readline() + if not line: + break + print(line) else: print('{}: No job output found. Checking again.'.format(sleeptime)) sleeptime += 2 diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index a48bff814..700fe8de9 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,8 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 60s" + # TIMEOUT="timeout 60s" + TIMEOUT="" fi #Build any sim/gen source code dependencies here ..... From c354daafa7d6ca10405250f1a851eb1176b94c50 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 15:09:43 -0500 Subject: [PATCH 147/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 16dea588d..37bb38c2d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -45,7 +45,7 @@ while sleeptime != 58: if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) - with open(outscript, 'r') as f + with open(outscript, 'r') as f: while True: line = f.readline() if not line: From 94dbe5798e8e1c207d5d0f5eb1e599327e863601 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 15:18:52 -0500 Subject: [PATCH 148/644] Fixes outscript logic not doing anything until 58 seconds in! --- conda/test_balsam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 37bb38c2d..97ca1455a 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -27,12 +27,12 @@ sleeptime = 0 -while not os.path.isdir(basedb) and sleeptime != 58: +while not os.path.isdir(basedb) and sleeptime < 58: print('{}: Waiting for Workflow directory'.format(sleeptime)) sleeptime += 1 time.sleep(1) -while len(os.listdir(basedb)) == 0 and sleeptime != 58: +while len(os.listdir(basedb)) == 0 and sleeptime < 58: print('{}: Waiting for Job Directory'.format(sleeptime)) sleeptime += 1 time.sleep(1) @@ -42,7 +42,7 @@ outscript = os.path.join(jobdir, 'job_script_test_balsam.out') print('Beginning cycle of checking for Balsam output: {}'.format(outscript)) -while sleeptime != 58: +while sleeptime < 58: if os.path.isfile(outscript): print('{}: Balsam job output found!'.format(sleeptime)) with open(outscript, 'r') as f: From 8611dfd538a5474c4606b8bb58b05e288b464905 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 15:44:23 -0500 Subject: [PATCH 149/644] prevent infinite loop, only print new lines --- conda/test_balsam.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 97ca1455a..1544d286d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -44,13 +44,17 @@ print('Beginning cycle of checking for Balsam output: {}'.format(outscript)) while sleeptime < 58: if os.path.isfile(outscript): + sleeptime = 58 print('{}: Balsam job output found!'.format(sleeptime)) with open(outscript, 'r') as f: + lastline = None while True: line = f.readline() + if line != lastline: + lastline = line + print(line) if not line: break - print(line) else: print('{}: No job output found. Checking again.'.format(sleeptime)) sleeptime += 2 From 2de69929073d0bf2ec3df006cbc7a1c5a98c6774 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 16:14:07 -0500 Subject: [PATCH 150/644] reorganize outscript reading logic --- conda/test_balsam.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 1544d286d..67f6e4de0 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -44,20 +44,25 @@ print('Beginning cycle of checking for Balsam output: {}'.format(outscript)) while sleeptime < 58: if os.path.isfile(outscript): - sleeptime = 58 print('{}: Balsam job output found!'.format(sleeptime)) - with open(outscript, 'r') as f: - lastline = None - while True: - line = f.readline() - if line != lastline: - lastline = line - print(line) - if not line: - break + break else: print('{}: No job output found. Checking again.'.format(sleeptime)) sleeptime += 2 time.sleep(2) +fileout = [] +with open(outscript, 'r') as f: + lastline = None + while True: + line = f.readline() + if line != lastline: + lastline = line + fileout.append(line) + if not line: + break + +for line in fileout: + print(line) + print('Test completed.') From 92607fef880989cc5de3f166c9ae41a8999fdcca Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 16:43:20 -0500 Subject: [PATCH 151/644] added continues for perpetual outscript opening until fileout len achieved --- conda/test_balsam.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 67f6e4de0..94c48a108 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -52,15 +52,17 @@ time.sleep(2) fileout = [] -with open(outscript, 'r') as f: - lastline = None - while True: +lastline = None +while True: + with open(outscript, 'r') as f: line = f.readline() if line != lastline: lastline = line fileout.append(line) - if not line: + continue + if not line and len(fileout) > 20: break + continue for line in fileout: print(line) From eacacf738073d841e33d77ee8c1a9076b5717475 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 16 Aug 2019 17:14:07 -0500 Subject: [PATCH 152/644] Trying new outscript output by line --- conda/test_balsam.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 94c48a108..aba550a38 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -1,6 +1,7 @@ import subprocess import os import time +import sys from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker # TESTSUITE_COMMS: local @@ -51,20 +52,16 @@ sleeptime += 2 time.sleep(2) -fileout = [] -lastline = None +print('Writing output file to terminal') +last = 0 while True: with open(outscript, 'r') as f: - line = f.readline() - if line != lastline: - lastline = line - fileout.append(line) - continue - if not line and len(fileout) > 20: - break - continue - -for line in fileout: - print(line) + f.seek(last) + new = f.read() + last = f.tell() + print(new) + sys.stdout.flush() + if new == 'Job 4 done on worker 1': + break print('Test completed.') From 8df8e97a0a484176b411c1a100e61e7ac591b08d Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 09:18:09 -0500 Subject: [PATCH 153/644] adds logic for concluding output printing when last line encountered --- conda/test_balsam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index aba550a38..c35d2d17c 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -54,6 +54,7 @@ print('Writing output file to terminal') last = 0 +lastline = 'Job 4 done on worker 1' while True: with open(outscript, 'r') as f: f.seek(last) @@ -61,7 +62,7 @@ last = f.tell() print(new) sys.stdout.flush() - if new == 'Job 4 done on worker 1': + if new[-len(lastline):] == lastline: break print('Test completed.') From 66ed4878af368dfeaad4c97e5ce324449915aeb7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 09:39:52 -0500 Subject: [PATCH 154/644] saving print for once final line in log detected --- conda/test_balsam.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index c35d2d17c..e6ef2a5d3 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -24,10 +24,11 @@ print('Executing Balsam job with command: {}'.format(runstr)) subprocess.Popen(runstr.split()) +# Location of Balsam DB location defined in configure-balsam-test.sh basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam') +# Periodically wait for Workflow and Job directory within Balsam DB sleeptime = 0 - while not os.path.isdir(basedb) and sleeptime < 58: print('{}: Waiting for Workflow directory'.format(sleeptime)) sleeptime += 1 @@ -38,6 +39,7 @@ sleeptime += 1 time.sleep(1) +# Periodically check for Balsam general output jobdirname = os.listdir(basedb)[0] jobdir = os.path.join(basedb, jobdirname) outscript = os.path.join(jobdir, 'job_script_test_balsam.out') @@ -52,17 +54,32 @@ sleeptime += 2 time.sleep(2) -print('Writing output file to terminal') -last = 0 +# Output from Balsam Job (script_test_balsam.py) +# print('Writing output file to terminal') +# last = 0 +# lastline = 'Job 4 done on worker 1' +# while True: +# with open(outscript, 'r') as f: +# f.seek(last) +# new = f.read() +# last = f.tell() +# print(new) +# sys.stdout.flush() +# if new[-len(lastline):] == lastline: +# break + + lastline = 'Job 4 done on worker 1' while True: with open(outscript, 'r') as f: - f.seek(last) new = f.read() - last = f.tell() - print(new) + time.sleep(0.5) + if new[-len(lastline):] == lastline: + break + +with open(outscript, 'r') as f: + lines = f.read() + print(lines) sys.stdout.flush() - if new[-len(lastline):] == lastline: - break print('Test completed.') From e9b1df5a97accc371723491973799abe559a1e4f Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 09:58:04 -0500 Subject: [PATCH 155/644] Adds newline character to lastline --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index e6ef2a5d3..e2d67509d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -69,7 +69,7 @@ # break -lastline = 'Job 4 done on worker 1' +lastline = 'Job 4 done on worker 1\n' while True: with open(outscript, 'r') as f: new = f.read() From 7cef85e311d79fac7a97f982738ae45e7e666d74 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 10:30:36 -0500 Subject: [PATCH 156/644] Moved some prints to prevent redundancy. Attempting to prevent constant blank lines and repeating output, --- conda/run_travis_locally/build_mpich_libE.sh | 2 +- conda/test_balsam.py | 25 ++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 8a606502e..748059c87 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -5,7 +5,7 @@ # -x echo commands # set -x # problem with this - loads of conda internal commands shown - overwhelming. -export PYTHON_VERSION=3.7 # default - override with -p +export PYTHON_VERSION=3.6 # default - override with -p export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows diff --git a/conda/test_balsam.py b/conda/test_balsam.py index e2d67509d..522b99e49 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -29,13 +29,13 @@ # Periodically wait for Workflow and Job directory within Balsam DB sleeptime = 0 +print('{}: Waiting for Workflow directory'.format(sleeptime)) while not os.path.isdir(basedb) and sleeptime < 58: - print('{}: Waiting for Workflow directory'.format(sleeptime)) sleeptime += 1 time.sleep(1) +print('{}: Waiting for Job Directory'.format(sleeptime)) while len(os.listdir(basedb)) == 0 and sleeptime < 58: - print('{}: Waiting for Job Directory'.format(sleeptime)) sleeptime += 1 time.sleep(1) @@ -68,18 +68,19 @@ # if new[-len(lastline):] == lastline: # break - -lastline = 'Job 4 done on worker 1\n' -while True: +lastposition = 0 +lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] +while sleeptime < 58: with open(outscript, 'r') as f: + f.seek(lastposition) new = f.read() - time.sleep(0.5) - if new[-len(lastline):] == lastline: - break + last = f.tell() + if len(new) > 0: + print(new) + sys.stdout.flush() + if new[-len(lastline):] in lastlines: + break + time.sleep(1) -with open(outscript, 'r') as f: - lines = f.read() - print(lines) - sys.stdout.flush() print('Test completed.') From b47f26d582c503ee689fcefe8714146ac196ca8f Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 10:37:15 -0500 Subject: [PATCH 157/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 522b99e49..efc65be23 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -78,7 +78,7 @@ if len(new) > 0: print(new) sys.stdout.flush() - if new[-len(lastline):] in lastlines: + if new[-len(lastlines[0]):] in lastlines: break time.sleep(1) From 9a83a98ef9807307c6ab9e5328260c4032ce4a5e Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 10:50:09 -0500 Subject: [PATCH 158/644] fixes lastposition not being updated --- conda/test_balsam.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index efc65be23..d7f1fc1db 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -54,33 +54,20 @@ sleeptime += 2 time.sleep(2) -# Output from Balsam Job (script_test_balsam.py) -# print('Writing output file to terminal') -# last = 0 -# lastline = 'Job 4 done on worker 1' -# while True: -# with open(outscript, 'r') as f: -# f.seek(last) -# new = f.read() -# last = f.tell() -# print(new) -# sys.stdout.flush() -# if new[-len(lastline):] == lastline: -# break - lastposition = 0 lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] while sleeptime < 58: with open(outscript, 'r') as f: f.seek(lastposition) new = f.read() - last = f.tell() + lastposition = f.tell() if len(new) > 0: print(new) sys.stdout.flush() if new[-len(lastlines[0]):] in lastlines: break time.sleep(1) + sleeptime += 1 print('Test completed.') From db3cd3c4fb96c34bced0d52ef58f7304797732b7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 11:31:27 -0500 Subject: [PATCH 159/644] Reenables timeout in run-tests. All-around commenting and cleanup --- conda/configure-balsam-test.sh | 7 +++--- conda/install-balsam.py | 7 +++++- conda/run_travis_locally/build_mpich_libE.sh | 13 +---------- conda/test_balsam.py | 23 ++++++++++--------- .../regression_tests/script_test_balsam.py | 22 +++++++++--------- libensemble/tests/run-tests.sh | 3 +-- 6 files changed, 35 insertions(+), 40 deletions(-) diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index bce2ed440..344c0c59a 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -1,9 +1,10 @@ #!/bin/bash # Run this script between each repeated local run of the Balsam -# job_control_hworld_balsam test. Besides ensuring that postgres and the -# generated Balsam db have the proper permissions, this script flushes previous -# apps and jobs in the db before submitting the test_balsam app/job script. +# job_control_hworld_balsam test in the base libensemble directory. +# Besides ensuring that postgres and the generated Balsam db have the proper +# permissions, this script flushes previous apps and jobs in the db before +# submitting the test_balsam app/job script. # # Most of this comes from scaling_tests/forces/balsam_local.sh diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 2e18a9938..ac340fca8 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -1,12 +1,17 @@ #! /usr/bin/env python +# Installing Balsam consists of both cloning and pip installing the latest version +# of Balsam from source but also moving the Balsam-relevant test to the regression_tests +# directory. This way, run-tests won't run a non-relevant test if Balsam isn't +# installed or the necessary Python version isn't installed. + import sys import os import subprocess balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' -if int(sys.version[2]) >= 6: +if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ subprocess.check_call(balsamclone.split()) os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index 748059c87..cd7e00254 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -6,7 +6,7 @@ # set -x # problem with this - loads of conda internal commands shown - overwhelming. export PYTHON_VERSION=3.6 # default - override with -p -export LIBE_BRANCH="experimental/balsam-on-travis" # default - override with -b +export LIBE_BRANCH="develop" # default - override with -b export SYSTEM="Linux" # default - override with -s # Options for miniconda - Linux, MacOSX, Windows export MPI=MPICH @@ -41,12 +41,6 @@ done echo -e "\nBuilding libE on ${SYSTEM} with python $PYTHON_VERSION and branch ${LIBE_BRANCH}\n" -# mkdir pg; cd pg; -# wget http://sbp.enterprisedb.com/getfile.jsp?fileid=11966 -# tar xf getfile.jsp?fileid=11966 -# export PATH="$PATH:/home/travis/pg/pgsql/bin" -# cd ~ - sudo pip install --upgrade pip sudo /etc/init.d/postgresql stop 9.2 sudo /etc/init.d/postgresql start 9.6 @@ -78,11 +72,6 @@ else conda install gcc_linux-64 || return fi conda install nlopt petsc4py petsc mumps-mpi=5.1.2=h5bebb2f_1007 mpi4py scipy $MPI -#conda install numpy || return #scipy includes numpy -# conda install scipy || return -# conda install mpi4py || return -# conda install petsc4py petsc || return -# conda install nlopt || return # pip install these as the conda installs downgrade pytest on python3.4 pip install pytest || return diff --git a/conda/test_balsam.py b/conda/test_balsam.py index d7f1fc1db..6e66a7d8e 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -7,12 +7,15 @@ # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 +# This test is NOT submitted as a job to Balsam. That would be script_test_balsam.py +# This test executes that job through the 'runstr' line defined further down. + nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which usually # only contain a single node, we need to change Balsam's default worker setup -# so multiple workers can be run on a single node (until this feature is [hopefully] added!). +# so multiple workers can be run on a single node. # For our purposes, we append ten workers to Balsam's WorkerGroup print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd())) modify_Balsam_worker() @@ -44,21 +47,19 @@ jobdir = os.path.join(basedb, jobdirname) outscript = os.path.join(jobdir, 'job_script_test_balsam.out') -print('Beginning cycle of checking for Balsam output: {}'.format(outscript)) -while sleeptime < 58: - if os.path.isfile(outscript): - print('{}: Balsam job output found!'.format(sleeptime)) - break - else: - print('{}: No job output found. Checking again.'.format(sleeptime)) - sleeptime += 2 - time.sleep(2) +# Periodically wait for Balsam Job output +print('{}: Beginning cycle of checking for Balsam output: {}'.format(sleeptime, outscript)) +while not os.path.isfile(outscript) and sleeptime < 58: + sleeptime += 2 + time.sleep(2) +# Print sections of Balsam output to screen every second until complete +print('{}: Balsam job output found! Printing to screen.'.format(sleeptime)) lastposition = 0 lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] while sleeptime < 58: with open(outscript, 'r') as f: - f.seek(lastposition) + f.seek(lastposition) # Prevents outputting already printed sections new = f.read() lastposition = f.tell() if len(new) > 0: diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index ba3a15ff9..d8c4a43a1 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -1,6 +1,6 @@ -# This script is submitted as an app to Balsam, and jobs are launched -# based on this app. This submission is via 'balsam launch' executed -# in the test_balsam.py script. +# This script is submitted as an app and job to Balsam, and jobs +# are launched based on this app. This submission is via ' +# balsam launch' executed in the test_balsam.py script. import os import numpy as np @@ -17,16 +17,13 @@ mpi4py.rc.recv_mprobe = False # Disable matching probes - +# Slighty different due to not executing in /regression_tests def build_simfunc(): import subprocess print('Balsam job launched in: {}'.format(os.getcwd())) buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' subprocess.check_call(buildstring.split()) - -# This script is meant to be launched by Balsam - libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} nworkers = MPI.COMM_WORLD.Get_size() - 1 @@ -37,7 +34,8 @@ def build_simfunc(): cores_all_jobs = nworkers*cores_per_job if is_master: - print('\nCores req: {} Cores avail: {}\n'.format(cores_all_jobs, logical_cores)) + print('\nCores req: {} Cores avail: {}\n'.format(cores_all_jobs, + logical_cores)) sim_app = './my_simjob.x' if not os.path.isfile(sim_app): @@ -67,8 +65,8 @@ def build_simfunc(): exit_criteria = {'elapsed_wallclock_time': 30} # Perform the run -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs) +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, + persis_info, libE_specs=libE_specs) if is_master: print('\nChecking expected job status against Workers ...\n') @@ -77,7 +75,9 @@ def build_simfunc(): # manager kill - but should show in the summary file. # Repeat expected lists nworkers times and compare with list of status's # received from workers - calc_status_list_in = np.asarray([WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED, 0]) + calc_status_list_in = np.asarray([WORKER_DONE, WORKER_KILL_ON_ERR, + WORKER_KILL_ON_TIMEOUT, + JOB_FAILED, 0]) calc_status_list = np.repeat(calc_status_list_in, nworkers) # For debug diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 700fe8de9..a48bff814 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,8 +378,7 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - # TIMEOUT="timeout 60s" - TIMEOUT="" + TIMEOUT="timeout 60s" fi #Build any sim/gen source code dependencies here ..... From 21f05f3f01ea45e3a171d156b6ab89bbf91cc802 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 11:37:15 -0500 Subject: [PATCH 160/644] flake8 --- conda/install-balsam.py | 2 +- libensemble/tests/regression_tests/script_test_balsam.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index ac340fca8..cd6c1a3d2 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,7 +11,7 @@ balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' -if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ +if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ subprocess.check_call(balsamclone.split()) os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index d8c4a43a1..e604aa699 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -17,6 +17,7 @@ mpi4py.rc.recv_mprobe = False # Disable matching probes + # Slighty different due to not executing in /regression_tests def build_simfunc(): import subprocess @@ -24,6 +25,7 @@ def build_simfunc(): buildstring = 'mpicc -o my_simjob.x libensemble/tests/unit_tests/simdir/my_simjob.c' subprocess.check_call(buildstring.split()) + libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} nworkers = MPI.COMM_WORLD.Get_size() - 1 From 000b2cffd7e391657b514bce1567c8888ffe85a5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 12:04:25 -0500 Subject: [PATCH 161/644] try adding progress to file/dir waiters. try disabling timeout again --- conda/test_balsam.py | 3 +++ libensemble/tests/run-tests.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 6e66a7d8e..621836578 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -35,11 +35,13 @@ print('{}: Waiting for Workflow directory'.format(sleeptime)) while not os.path.isdir(basedb) and sleeptime < 58: sleeptime += 1 + print('{}'.format(sleeptime), end=" ") time.sleep(1) print('{}: Waiting for Job Directory'.format(sleeptime)) while len(os.listdir(basedb)) == 0 and sleeptime < 58: sleeptime += 1 + print('{}'.format(sleeptime), end=" ") time.sleep(1) # Periodically check for Balsam general output @@ -51,6 +53,7 @@ print('{}: Beginning cycle of checking for Balsam output: {}'.format(sleeptime, outscript)) while not os.path.isfile(outscript) and sleeptime < 58: sleeptime += 2 + print('{}'.format(sleeptime), end=" ") time.sleep(2) # Print sections of Balsam output to screen every second until complete diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index a48bff814..700fe8de9 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -378,7 +378,8 @@ if [ "$root_found" = true ]; then TIMEOUT="" if [ -x "$(command -v timeout)" ] ; then - TIMEOUT="timeout 60s" + # TIMEOUT="timeout 60s" + TIMEOUT="" fi #Build any sim/gen source code dependencies here ..... From e64c962ccdd5a563a5d060727c6c48a72b1ad6ed Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 14:08:42 -0500 Subject: [PATCH 162/644] attempt to use typical job_control_hworld sim_f --- libensemble/tests/regression_tests/script_test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index e604aa699..9c6034f11 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -11,7 +11,7 @@ from libensemble.balsam_controller import BalsamJobController from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE -from libensemble.sim_funcs.job_control_hworld_balsam import job_control_hworld as sim_f +from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import per_worker_stream From 2edfdccef9cd65b4000d3fe9abd6adc851d55155 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 19 Aug 2019 14:21:18 -0500 Subject: [PATCH 163/644] removes duplicate code, adjusts timeout of job_control_hworld (which will be used instead) --- libensemble/sim_funcs/job_control_hworld.py | 2 +- .../sim_funcs/job_control_hworld_balsam.py | 121 ------------------ 2 files changed, 1 insertion(+), 122 deletions(-) delete mode 100644 libensemble/sim_funcs/job_control_hworld_balsam.py diff --git a/libensemble/sim_funcs/job_control_hworld.py b/libensemble/sim_funcs/job_control_hworld.py index 9cb7403d6..7b772fb0d 100644 --- a/libensemble/sim_funcs/job_control_hworld.py +++ b/libensemble/sim_funcs/job_control_hworld.py @@ -72,7 +72,7 @@ def job_control_hworld(H, persis_info, sim_specs, libE_specs): # pref send this in X as a sim_in from calling script global sim_count sim_count += 1 - timeout = 3.0 + timeout = 6.0 if sim_count == 1: args_for_sim = 'sleep 1' # Should finish elif sim_count == 2: diff --git a/libensemble/sim_funcs/job_control_hworld_balsam.py b/libensemble/sim_funcs/job_control_hworld_balsam.py deleted file mode 100644 index a8dd89e8c..000000000 --- a/libensemble/sim_funcs/job_control_hworld_balsam.py +++ /dev/null @@ -1,121 +0,0 @@ -from libensemble.balsam_controller import BalsamJobController -from libensemble.message_numbers import UNSET_TAG, WORKER_KILL_ON_ERR, MAN_SIGNAL_FINISH, WORKER_DONE, JOB_FAILED, WORKER_KILL_ON_TIMEOUT -import numpy as np - -__all__ = ['job_control_hworld'] - -# Alt send values through X -sim_count = 0 - - -def polling_loop(comm, jobctl, job, timeout_sec=3.0, delay=0.3): - import time - - calc_status = UNSET_TAG # Sim func determines status of libensemble calc - returned to worker - - while job.runtime < timeout_sec: - time.sleep(delay) - - # print('Probing manager at time: ', job.runtime) - jobctl.manager_poll(comm) - if jobctl.manager_signal == 'finish': - jobctl.kill(job) - calc_status = MAN_SIGNAL_FINISH # Worker will pick this up and close down - print('Job {} killed by manager on worker {}'.format(job.id, jobctl.workerID)) - break - - # print('Polling job at time', job.runtime) - job.poll() - if job.finished: - break - elif job.state == 'RUNNING': - print('Job {} still running on worker {} ....'.format(job.id, jobctl.workerID)) - - # Check output file for error - # print('Checking output file for error at time:', job.runtime) - if job.stdout_exists(): - if 'Error' in job.read_stdout(): - print("Found (deliberate) Error in ouput file - cancelling job {} on worker {}".format(job.id, jobctl.workerID)) - jobctl.kill(job) - calc_status = WORKER_KILL_ON_ERR - break - - # After exiting loop - if job.finished: - print('Job {} done on worker {}'.format(job.id, jobctl.workerID)) - # Fill in calc_status if not already - if calc_status == UNSET_TAG: - if job.state == 'FINISHED': # Means finished succesfully - calc_status = WORKER_DONE - elif job.state == 'FAILED': - calc_status = JOB_FAILED - # elif job.state == 'USER_KILLED': - # calc_status = WORKER_KILL - else: - # assert job.state == 'RUNNING', "job.state expected to be RUNNING. Returned: " + str(job.state) - print("Job {} timed out - killing on worker {}".format(job.id, jobctl.workerID)) - jobctl.kill(job) - if job.finished: - print('Job {} done on worker {}'.format(job.id, jobctl.workerID)) - calc_status = WORKER_KILL_ON_TIMEOUT - - return job, calc_status - - -def job_control_hworld(H, persis_info, sim_specs, libE_specs): - """ Test of launching and polling job and exiting on job finish""" - jobctl = BalsamJobController.controller - cores = sim_specs['cores'] - comm = libE_specs['comm'] - - args_for_sim = 'sleep 1' - # pref send this in X as a sim_in from calling script - global sim_count - sim_count += 1 - timeout = 6.0 - if sim_count == 1: - args_for_sim = 'sleep 1' # Should finish - elif sim_count == 2: - args_for_sim = 'sleep 1 Error' # Worker kill on error - elif sim_count == 3: - args_for_sim = 'sleep 3' # Worker kill on timeout - timeout = 1.0 - elif sim_count == 4: - args_for_sim = 'sleep 1 Fail' # Manager kill - if signal received else completes - elif sim_count == 5: - args_for_sim = 'sleep 18' # Manager kill - if signal received else completes - timeout = 20.0 - - job = jobctl.launch(calc_type='sim', num_procs=cores, app_args=args_for_sim, hyperthreads=True, wait_on_run=True) - job, calc_status = polling_loop(comm, jobctl, job, timeout) - - # assert job.finished, "job.finished should be True. Returned " + str(job.finished) - # assert job.state == 'FINISHED', "job.state should be FINISHED. Returned " + str(job.state) - - # This is temp - return something - so doing six_hump_camel_func again... - batch = len(H['x']) - O = np.zeros(batch, dtype=sim_specs['out']) - for i, x in enumerate(H['x']): - O['f'][i] = six_hump_camel_func(x) - - # This is just for testing at calling script level - status of each job - O['cstat'] = calc_status - - # v = np.random.uniform(0, 10) - # print('About to sleep for :' + str(v)) - # time.sleep(v) - - return O, persis_info, calc_status - - -def six_hump_camel_func(x): - """ - Definition of the six-hump camel - """ - x1 = x[0] - x2 = x[1] - term1 = (4-2.1*x1**2+(x1**4)/3) * x1**2 - term2 = x1*x2 - term3 = (-4+4*x2**2) * x2**2 - - return term1 + term2 + term3 From 268d40470e0a5785d4723c8b68f1912bc05a6c2d Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni Date: Tue, 20 Aug 2019 18:50:32 +0530 Subject: [PATCH 164/644] removes debug message --- libensemble/gen_funcs/support.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 0ae971e7a..e5084828d 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -11,8 +11,6 @@ def sendrecv_mgr_worker_msg(comm, O, status=None): def send_mgr_worker_msg(comm, O): """Send message from worker to manager. """ - if 'sim_id' in O.dtype.names: - print(25*"-", "Sending {}(SimID: {})".format(O[0][0], O['sim_id']), 25*"-", flush=True) D = {'calc_out': O, 'libE_info': {'persistent': True}, 'calc_status': UNSET_TAG, From 3b5f3c0e6662432f96ee4ef2200354a160e00145 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 20 Aug 2019 08:27:34 -0500 Subject: [PATCH 165/644] Removing 3.4 --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8dacfbeda..92bca6f08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python sudo: required dist: xenial python: - - 3.4 - 3.5 - 3.6 - 3.7 @@ -25,7 +24,6 @@ matrix: language: generic python: 3 - # matrix: # allow_failures: # - env: MPI=openmpi From 907b94ae6d1f3a00bd203a1a654a38dfef348d79 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 10:44:43 -0500 Subject: [PATCH 166/644] Adds function to modify Balsam's newapp routine to add coverage runtime. --- conda/install-balsam.py | 3 +- conda/test_balsam.py | 7 +++-- libensemble/tests/regression_tests/common.py | 30 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index cd6c1a3d2..5aee31a31 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -8,6 +8,7 @@ import sys import os import subprocess +from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' @@ -19,5 +20,5 @@ if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'): os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - # os.system('./conda/configure-balsam-test.sh') + modify_Balsam_pyCoverage() # Need to add "coverage" before jobs added subprocess.run('./conda/configure-balsam-test.sh'.split()) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 621836578..50a4844b3 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -68,10 +68,11 @@ if len(new) > 0: print(new) sys.stdout.flush() - if new[-len(lastlines[0]):] in lastlines: + #if new[-len(lastlines[0]):] in lastlines: + if any(new.endswith(line) for line in lastlines): break time.sleep(1) sleeptime += 1 - -print('Test completed.') +print('Test completed. Importing coverage into local directory.') +# Coming soon! diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 41fc4bbd7..30fd275f9 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -202,3 +202,33 @@ def modify_Balsam_worker(): print("Modified worker file in {}".format(os.getcwd())) os.chdir(home) + +def modify_Balsam_pyCoverage(): + # Tracking line coverage of our code through our tests requires running a test + # with the format 'python -m coverage run test.py args'. Balsam explicitely + # configures Python executable runs with 'python test.py args' with no current + # capability for specifying runtime Python options. This hack attempts to resolve + # this for our purposes only. + import balsam + + old_line = " path = ' '.join((exe, script_path, args))" + new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode', script_path, args))" + + commandfile = 'cli_commands.py' + home = os.getcwd() + balsam_scripts_path = os.path.dirname(balsam.__file__) + '/scripts' + os.chdir(balsam_scripts_path) + + with open(commandfile, 'r') as f: + lines = f.readlines() + + for i in range(len(lines)): + if lines[i] == old_line: + lines[i] = new_line + + with open(commandfile, 'w') as f: + for line in lines: + f.write(line) + + print("Modified cli_commands file in {}".format(os.getcwd())) + os.chdir(home) From a73942d337bf732711dd497a4ddc9749c4320c24 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 11:18:48 -0500 Subject: [PATCH 167/644] Moves Balsam modification line --- conda/configure-balsam-test.sh | 3 +++ conda/install-balsam.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index 344c0c59a..6eacd80aa 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -8,6 +8,9 @@ # # Most of this comes from scaling_tests/forces/balsam_local.sh +# Can't run this line in calling Python file. Balsam installation hasn't been +# noticed by the Python runtime yet. +python -c 'from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage; modify_Balsam_pyCoverage()' export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 5aee31a31..140ecf2c0 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -8,7 +8,7 @@ import sys import os import subprocess -from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage + balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' @@ -20,5 +20,4 @@ if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'): os.rename('./conda/test_balsam.py', './libensemble/tests/regression_tests/test_balsam.py') - modify_Balsam_pyCoverage() # Need to add "coverage" before jobs added subprocess.run('./conda/configure-balsam-test.sh'.split()) From c14c87c0d060c656a67f0ae2bbc6e1858930523b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 11:48:01 -0500 Subject: [PATCH 168/644] newlines, stdout flushes for progress --- conda/test_balsam.py | 3 +++ libensemble/tests/regression_tests/common.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 50a4844b3..5c455d0e8 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -36,12 +36,14 @@ while not os.path.isdir(basedb) and sleeptime < 58: sleeptime += 1 print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() time.sleep(1) print('{}: Waiting for Job Directory'.format(sleeptime)) while len(os.listdir(basedb)) == 0 and sleeptime < 58: sleeptime += 1 print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() time.sleep(1) # Periodically check for Balsam general output @@ -54,6 +56,7 @@ while not os.path.isfile(outscript) and sleeptime < 58: sleeptime += 2 print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() time.sleep(2) # Print sections of Balsam output to screen every second until complete diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 30fd275f9..b62503db9 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -211,8 +211,8 @@ def modify_Balsam_pyCoverage(): # this for our purposes only. import balsam - old_line = " path = ' '.join((exe, script_path, args))" - new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode', script_path, args))" + old_line = " path = ' '.join((exe, script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode', script_path, args))\n" commandfile = 'cli_commands.py' home = os.getcwd() From 6a79da820eaf14625c4aec6ab9ef36a8c7423691 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 14:36:22 -0500 Subject: [PATCH 169/644] First attempt to move coverage from Balsam to tests directory --- conda/test_balsam.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 5c455d0e8..3cdca52f4 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -60,7 +60,7 @@ time.sleep(2) # Print sections of Balsam output to screen every second until complete -print('{}: Balsam job output found! Printing to screen.'.format(sleeptime)) +print('{}: Attempting to print output.'.format(sleeptime)) lastposition = 0 lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] while sleeptime < 58: @@ -77,5 +77,14 @@ time.sleep(1) sleeptime += 1 -print('Test completed. Importing coverage into local directory.') -# Coming soon! +print('Test completed. Importing any coverage from Balsam.') +here = os.getcwd() + +# Move coverage files from Balsam DB to ./regression_tests (for concatenation) +for file in os.listdir(jobdir): + if file[:10] == '.coverage.': + print('Coverage found: {}'.format(file)) + coveragefile = os.path.join(jobdir, file) + os.rename(coveragefile, here) + +print('test_balsam processing complete.') From d325287afc3bce3f3f0b3998bda206b64b212ce7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 15:01:09 -0500 Subject: [PATCH 170/644] specifies .coveragerc file explicitly, fixes file move path --- conda/test_balsam.py | 5 +++-- libensemble/tests/regression_tests/common.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 3cdca52f4..415bb4440 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -84,7 +84,8 @@ for file in os.listdir(jobdir): if file[:10] == '.coverage.': print('Coverage found: {}'.format(file)) - coveragefile = os.path.join(jobdir, file) - os.rename(coveragefile, here) + balsam_cov = os.path.join(jobdir, file) + here_cov = os.path.join(here, file) + os.rename(balsam_cov, here_cov) print('test_balsam processing complete.') diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index b62503db9..da45fce3c 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -212,7 +212,7 @@ def modify_Balsam_pyCoverage(): import balsam old_line = " path = ' '.join((exe, script_path, args))\n" - new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode', script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' home = os.getcwd() From d115586b3d9c7e8a1b5908ed8b5e39c4312436db Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 15:16:19 -0500 Subject: [PATCH 171/644] flake8 --- conda/test_balsam.py | 2 +- libensemble/tests/regression_tests/common.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 415bb4440..826c5dace 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -71,7 +71,7 @@ if len(new) > 0: print(new) sys.stdout.flush() - #if new[-len(lastlines[0]):] in lastlines: + # if new[-len(lastlines[0]):] in lastlines: if any(new.endswith(line) for line in lastlines): break time.sleep(1) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index da45fce3c..e23706f13 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -203,6 +203,7 @@ def modify_Balsam_worker(): print("Modified worker file in {}".format(os.getcwd())) os.chdir(home) + def modify_Balsam_pyCoverage(): # Tracking line coverage of our code through our tests requires running a test # with the format 'python -m coverage run test.py args'. Balsam explicitely From fa9956900690a5c7816361dcf552818210a4c336 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 16:05:25 -0500 Subject: [PATCH 172/644] don't need to source run-tests.sh --- .travis.yml | 2 +- conda/run_travis_locally/build_mpich_libE.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b36a6fa0b..b1cee2631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -97,7 +97,7 @@ before_script: # Run test (-z show output) script: - - source libensemble/tests/run-tests.sh -z + - ./libensemble/tests/run-tests.sh -z # Coverage after_success: diff --git a/conda/run_travis_locally/build_mpich_libE.sh b/conda/run_travis_locally/build_mpich_libE.sh index ad224dd91..345b8ec97 100755 --- a/conda/run_travis_locally/build_mpich_libE.sh +++ b/conda/run_travis_locally/build_mpich_libE.sh @@ -87,7 +87,7 @@ pip install -e . || return python conda/install-balsam.py export BALSAM_DB_PATH=~/test-balsam -source libensemble/tests/run-tests.sh -z +./libensemble/tests/run-tests.sh -z echo -e "\n\nScript completed...\n\n" set +ex From d9d0aab2676e4ed6ad5408b354e89b53d51e6e28 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 16:17:50 -0500 Subject: [PATCH 173/644] fixes coverage filename --- conda/test_balsam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 826c5dace..8b5af641d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -81,8 +81,9 @@ here = os.getcwd() # Move coverage files from Balsam DB to ./regression_tests (for concatenation) +covname = '.cov_reg_out.' for file in os.listdir(jobdir): - if file[:10] == '.coverage.': + if file[:len(covname)] == covname: # Filename starts with... print('Coverage found: {}'.format(file)) balsam_cov = os.path.join(jobdir, file) here_cov = os.path.join(here, file) From 86f4e0d1ace43b08a655aeeae32388cf783a706f Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 21 Aug 2019 16:23:36 -0500 Subject: [PATCH 174/644] flake8... --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 8b5af641d..0c1248c26 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -83,7 +83,7 @@ # Move coverage files from Balsam DB to ./regression_tests (for concatenation) covname = '.cov_reg_out.' for file in os.listdir(jobdir): - if file[:len(covname)] == covname: # Filename starts with... + if file[:len(covname)] == covname: # Filename starts with... print('Coverage found: {}'.format(file)) balsam_cov = os.path.join(jobdir, file) here_cov = os.path.join(here, file) From 88129ae88566bcd8bf280d0c9242e8369e3f6f16 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 22 Aug 2019 13:21:49 -0500 Subject: [PATCH 175/644] better print for coverage file moving --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 0c1248c26..2b9b1540d 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -84,9 +84,9 @@ covname = '.cov_reg_out.' for file in os.listdir(jobdir): if file[:len(covname)] == covname: # Filename starts with... - print('Coverage found: {}'.format(file)) balsam_cov = os.path.join(jobdir, file) here_cov = os.path.join(here, file) + print('Moved {} from {} to {}.'.format(file, jobdir, here)) os.rename(balsam_cov, here_cov) print('test_balsam processing complete.') From 59cc1fb7f3aa11bd4929bdee28211cc3fa129202 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 22 Aug 2019 14:14:50 -0500 Subject: [PATCH 176/644] experiment with appending balsam script coverage to run-tests coverage --- conda/test_balsam.py | 2 +- libensemble/tests/regression_tests/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 2b9b1540d..c7fe7a981 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -86,7 +86,7 @@ if file[:len(covname)] == covname: # Filename starts with... balsam_cov = os.path.join(jobdir, file) here_cov = os.path.join(here, file) - print('Moved {} from {} to {}.'.format(file, jobdir, here)) + print('Moved {} from {} to {}'.format(file, jobdir, here)) os.rename(balsam_cov, here_cov) print('test_balsam processing complete.') diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index e23706f13..bda680b8a 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -213,7 +213,7 @@ def modify_Balsam_pyCoverage(): import balsam old_line = " path = ' '.join((exe, script_path, args))\n" - new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run --append --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' home = os.getcwd() From 749a91c8d41bf9ec8dc9a11a209cf6c47b6b2eaf Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 22 Aug 2019 14:30:02 -0500 Subject: [PATCH 177/644] removes append --- libensemble/tests/regression_tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index bda680b8a..e23706f13 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -213,7 +213,7 @@ def modify_Balsam_pyCoverage(): import balsam old_line = " path = ' '.join((exe, script_path, args))\n" - new_line = " path = ' '.join((exe, '-m coverage run --append --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' home = os.getcwd() From 00f18ecd33ad8b1918b604a21d1f3e903d71ae78 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 09:57:48 -0500 Subject: [PATCH 178/644] misc small changes --- conda/test_balsam.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index c7fe7a981..b92ec6dd5 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -71,22 +71,21 @@ if len(new) > 0: print(new) sys.stdout.flush() - # if new[-len(lastlines[0]):] in lastlines: if any(new.endswith(line) for line in lastlines): break time.sleep(1) sleeptime += 1 -print('Test completed. Importing any coverage from Balsam.') +print('{}: Importing any coverage from Balsam.'.format(sleeptime)) here = os.getcwd() # Move coverage files from Balsam DB to ./regression_tests (for concatenation) covname = '.cov_reg_out.' for file in os.listdir(jobdir): - if file[:len(covname)] == covname: # Filename starts with... + if file.startswith(covname): balsam_cov = os.path.join(jobdir, file) here_cov = os.path.join(here, file) print('Moved {} from {} to {}'.format(file, jobdir, here)) os.rename(balsam_cov, here_cov) -print('test_balsam processing complete.') +print('Test complete.') From acde8df9cfb122d616a5c2278c780bde8ea4df45 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 11:16:44 -0500 Subject: [PATCH 179/644] exploring coverage not collecting coverage info (based on warning message) --- conda/test_balsam.py | 14 ++++----- .../tests/regression_tests/.bal_coveragerc | 30 +++++++++++++++++++ libensemble/tests/regression_tests/common.py | 2 +- 3 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 libensemble/tests/regression_tests/.bal_coveragerc diff --git a/conda/test_balsam.py b/conda/test_balsam.py index b92ec6dd5..019c478d5 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -80,12 +80,12 @@ here = os.getcwd() # Move coverage files from Balsam DB to ./regression_tests (for concatenation) -covname = '.cov_reg_out.' -for file in os.listdir(jobdir): - if file.startswith(covname): - balsam_cov = os.path.join(jobdir, file) - here_cov = os.path.join(here, file) - print('Moved {} from {} to {}'.format(file, jobdir, here)) - os.rename(balsam_cov, here_cov) +# covname = '.cov_reg_out.' +# for file in os.listdir(jobdir): +# if file.startswith(covname): +# balsam_cov = os.path.join(jobdir, file) +# here_cov = os.path.join(here, file) +# print('Moved {} from {} to {}'.format(file, jobdir, here)) +# os.rename(balsam_cov, here_cov) print('Test complete.') diff --git a/libensemble/tests/regression_tests/.bal_coveragerc b/libensemble/tests/regression_tests/.bal_coveragerc new file mode 100644 index 000000000..ba9b70e2b --- /dev/null +++ b/libensemble/tests/regression_tests/.bal_coveragerc @@ -0,0 +1,30 @@ +[run] +source = . + +omit = + */tests/* + */unit_tests/* + */unit_tests_nompi/* + */regression_tests/* + */branin_*/* + +branch = true + +data_file = .cov_reg_out + +parallel = true + +[report] +omit = + */__init__.py/* + */setup.py/* + */tests/* + */unit_tests/* + */unit_tests_nompi/* + */regression_tests/* + +exclude_lines = + if __name__ == .__main__.: + +[html] +directory = cov_reg diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index e23706f13..075ea3263 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -213,7 +213,7 @@ def modify_Balsam_pyCoverage(): import balsam old_line = " path = ' '.join((exe, script_path, args))\n" - new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.coveragerc', script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.bal_coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' home = os.getcwd() From 156c556da3696395bc4111cdc7636950eeff0e5c Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 12:16:32 -0500 Subject: [PATCH 180/644] adding explicit libensemble location --- libensemble/tests/regression_tests/.bal_coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/.bal_coveragerc b/libensemble/tests/regression_tests/.bal_coveragerc index ba9b70e2b..59fbb96d7 100644 --- a/libensemble/tests/regression_tests/.bal_coveragerc +++ b/libensemble/tests/regression_tests/.bal_coveragerc @@ -1,5 +1,5 @@ [run] -source = . +source = /home/travis/libensemble/ omit = */tests/* From 209975636d15b175230e3d62e346c1821c1c3d68 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 13:11:09 -0500 Subject: [PATCH 181/644] un-commented code block --- conda/test_balsam.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 019c478d5..1f63a2926 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -79,13 +79,13 @@ print('{}: Importing any coverage from Balsam.'.format(sleeptime)) here = os.getcwd() -# Move coverage files from Balsam DB to ./regression_tests (for concatenation) -# covname = '.cov_reg_out.' -# for file in os.listdir(jobdir): -# if file.startswith(covname): -# balsam_cov = os.path.join(jobdir, file) -# here_cov = os.path.join(here, file) -# print('Moved {} from {} to {}'.format(file, jobdir, here)) -# os.rename(balsam_cov, here_cov) +Move coverage files from Balsam DB to ./regression_tests (for concatenation) +covname = '.cov_reg_out.' +for file in os.listdir(jobdir): + if file.startswith(covname): + balsam_cov = os.path.join(jobdir, file) + here_cov = os.path.join(here, file) + print('Moved {} from {} to {}'.format(file, jobdir, here)) + os.rename(balsam_cov, here_cov) print('Test complete.') From 653bd9ede801a8fa66dfc7fce429ac403dceb534 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 13:20:11 -0500 Subject: [PATCH 182/644] flake8 --- conda/test_balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index 1f63a2926..b92ec6dd5 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -79,7 +79,7 @@ print('{}: Importing any coverage from Balsam.'.format(sleeptime)) here = os.getcwd() -Move coverage files from Balsam DB to ./regression_tests (for concatenation) +# Move coverage files from Balsam DB to ./regression_tests (for concatenation) covname = '.cov_reg_out.' for file in os.listdir(jobdir): if file.startswith(covname): From 93638082cc5f787c6d3914befe5838230b94fcd7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 14:01:22 -0500 Subject: [PATCH 183/644] trying travis-server libe path --- libensemble/tests/regression_tests/.bal_coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/.bal_coveragerc b/libensemble/tests/regression_tests/.bal_coveragerc index 59fbb96d7..e3a4097db 100644 --- a/libensemble/tests/regression_tests/.bal_coveragerc +++ b/libensemble/tests/regression_tests/.bal_coveragerc @@ -1,5 +1,5 @@ [run] -source = /home/travis/libensemble/ +source = /home/travis/build/Libensemble/libensemble omit = */tests/* From 0a8cabf61cf514119bfe4d3325e685c2f8d742a4 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 15:32:10 -0500 Subject: [PATCH 184/644] Attempts to make base libensemble path more dynamic for coverage. Misc housekeeping changes --- conda/test_balsam.py | 22 ++++++----- .../tests/regression_tests/.bal_coveragerc | 2 +- libensemble/tests/regression_tests/common.py | 39 +++++++++---------- .../regression_tests/script_test_balsam.py | 7 ++-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/conda/test_balsam.py b/conda/test_balsam.py index b92ec6dd5..d64697aa0 100644 --- a/conda/test_balsam.py +++ b/conda/test_balsam.py @@ -2,6 +2,7 @@ import os import time import sys +import libensemble from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker # TESTSUITE_COMMS: local @@ -12,15 +13,17 @@ nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention +# Set libensemble base directory to environment variable for Balsam coverage +# orientation purposes. Otherwise coverage in the Balsam data directory is +# (unsuccessfully) collected. +libepath = os.path.dirname(libensemble.__file__) +os.environ['LIBE_PATH'] = libepath + # Balsam is meant for HPC systems that commonly distribute jobs across many -# nodes. Due to the nature of testing Balsam on local or CI systems which usually -# only contain a single node, we need to change Balsam's default worker setup -# so multiple workers can be run on a single node. -# For our purposes, we append ten workers to Balsam's WorkerGroup +# nodes. For our purposes, we append (hack) ten workers to Balsam's WorkerGroup print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd())) modify_Balsam_worker() -# Executes Balsam Job # By this point, script_test_balsam.py has been submitted as an app and job to Balsam # This line launches the queued job in the Balsam database runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' @@ -46,13 +49,13 @@ sys.stdout.flush() time.sleep(1) -# Periodically check for Balsam general output +# Job directory now exists jobdirname = os.listdir(basedb)[0] jobdir = os.path.join(basedb, jobdirname) outscript = os.path.join(jobdir, 'job_script_test_balsam.out') # Periodically wait for Balsam Job output -print('{}: Beginning cycle of checking for Balsam output: {}'.format(sleeptime, outscript)) +print('{}: Checking for Balsam output file: {}'.format(sleeptime, outscript)) while not os.path.isfile(outscript) and sleeptime < 58: sleeptime += 2 print('{}'.format(sleeptime), end=" ") @@ -60,12 +63,12 @@ time.sleep(2) # Print sections of Balsam output to screen every second until complete -print('{}: Attempting to print output.'.format(sleeptime)) +print('{}: Output file exists. Waiting for Balsam Job Output.'.format(sleeptime)) lastposition = 0 lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] while sleeptime < 58: with open(outscript, 'r') as f: - f.seek(lastposition) # Prevents outputting already printed sections + f.seek(lastposition) # (should) prevent outputting already printed sections new = f.read() lastposition = f.tell() if len(new) > 0: @@ -73,6 +76,7 @@ sys.stdout.flush() if any(new.endswith(line) for line in lastlines): break + print('{}'.format(sleeptime), end=" ") time.sleep(1) sleeptime += 1 diff --git a/libensemble/tests/regression_tests/.bal_coveragerc b/libensemble/tests/regression_tests/.bal_coveragerc index e3a4097db..40a5b152f 100644 --- a/libensemble/tests/regression_tests/.bal_coveragerc +++ b/libensemble/tests/regression_tests/.bal_coveragerc @@ -1,5 +1,5 @@ [run] -source = /home/travis/build/Libensemble/libensemble +source = $LIBE_PATH omit = */tests/* diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 075ea3263..4b5662489 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -174,10 +174,11 @@ def build_simfunc(): def modify_Balsam_worker(): # Balsam is meant for HPC systems that commonly distribute jobs across many - # nodes. Due to the nature of testing Balsam on local or CI systems which usually - # only contain a single node, we need to change Balsam's default worker setup - # so multiple workers can be run on a single node (until this feature is [hopefully] added!). - # For our purposes, we append ten workers to Balsam's WorkerGroup + # nodes. Due to the nature of testing Balsam on local or CI systems which + # usually only contain a single node, we need to change Balsam's default + # worker setup so multiple workers can be run on a single node (until this + # feature is [hopefully] added!).For our purposes, we append ten workers + # to Balsam's WorkerGroup import balsam new_lines = [" for idx in range(10):\n", @@ -185,51 +186,47 @@ def modify_Balsam_worker(): " self.workers.append(w)\n"] workerfile = 'worker.py' - home = os.getcwd() - balsam_worker_path = os.path.dirname(balsam.__file__) + '/launcher' - os.chdir(balsam_worker_path) + balsam_path = os.path.dirname(balsam.__file__) + '/launcher' + balsam_worker_path = os.path.join(balsam_path, workerfile) - with open(workerfile, 'r') as f: + with open(balsam_worker_path, 'r') as f: lines = f.readlines() if lines[-3] != new_lines[0]: lines = lines[:-2] # effectively inserting new_lines[0] above lines.extend(new_lines) - with open(workerfile, 'w') as f: + with open(balsam_worker_path, 'w') as f: for line in lines: f.write(line) - print("Modified worker file in {}".format(os.getcwd())) - os.chdir(home) + print("Modified worker file in {}".format(balsam_path)) def modify_Balsam_pyCoverage(): # Tracking line coverage of our code through our tests requires running a test # with the format 'python -m coverage run test.py args'. Balsam explicitely - # configures Python executable runs with 'python test.py args' with no current - # capability for specifying runtime Python options. This hack attempts to resolve - # this for our purposes only. + # configures Python runs with 'python test.py args' with no current + # capability for specifying runtime Python options. This hack attempts to + # resolve this for our purposes only. import balsam old_line = " path = ' '.join((exe, script_path, args))\n" new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.bal_coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' - home = os.getcwd() - balsam_scripts_path = os.path.dirname(balsam.__file__) + '/scripts' - os.chdir(balsam_scripts_path) + balsam_path = os.path.dirname(balsam.__file__) + '/scripts' + balsam_commands_path = os.path.join(balsam_path, commandfile) - with open(commandfile, 'r') as f: + with open(balsam_commands_path, 'r') as f: lines = f.readlines() for i in range(len(lines)): if lines[i] == old_line: lines[i] = new_line - with open(commandfile, 'w') as f: + with open(balsam_commands_path, 'w') as f: for line in lines: f.write(line) - print("Modified cli_commands file in {}".format(os.getcwd())) - os.chdir(home) + print("Modified cli_commands file in {}".format(balsam_scripts_path)) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam.py index 9c6034f11..d606bd099 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam.py @@ -1,6 +1,5 @@ -# This script is submitted as an app and job to Balsam, and jobs -# are launched based on this app. This submission is via ' -# balsam launch' executed in the test_balsam.py script. +# This script is submitted as an app and job to Balsam. The job submission is +# via 'balsam launch' executed in the test_balsam.py script. import os import numpy as np @@ -18,7 +17,7 @@ mpi4py.rc.recv_mprobe = False # Disable matching probes -# Slighty different due to not executing in /regression_tests +# Slighty different due to working directory not being /regression_tests def build_simfunc(): import subprocess print('Balsam job launched in: {}'.format(os.getcwd())) From 748bbe1ae3638d66baf586f5cff214afdacb5b14 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 26 Aug 2019 15:40:07 -0500 Subject: [PATCH 185/644] flake8 --- libensemble/tests/regression_tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 4b5662489..12518c49b 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -229,4 +229,4 @@ def modify_Balsam_pyCoverage(): for line in lines: f.write(line) - print("Modified cli_commands file in {}".format(balsam_scripts_path)) + print("Modified cli_commands file in {}".format(balsam_path)) From c6c2079eedf615cc7087ca497f4d05267a6aab28 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 27 Aug 2019 10:43:14 -0500 Subject: [PATCH 186/644] Renamed Balsam tests to include hworld, dramatic refactoring throughout --- conda/configure-balsam-test.sh | 10 +- conda/install-balsam.py | 32 +++-- conda/test_balsam.py | 95 --------------- conda/test_balsam_hworld.py | 109 ++++++++++++++++++ libensemble/tests/regression_tests/common.py | 21 ++-- ...balsam.py => script_test_balsam_hworld.py} | 8 +- 6 files changed, 144 insertions(+), 131 deletions(-) delete mode 100644 conda/test_balsam.py create mode 100644 conda/test_balsam_hworld.py rename libensemble/tests/regression_tests/{script_test_balsam.py => script_test_balsam_hworld.py} (91%) diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index 6eacd80aa..a284fd92e 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -4,19 +4,19 @@ # job_control_hworld_balsam test in the base libensemble directory. # Besides ensuring that postgres and the generated Balsam db have the proper # permissions, this script flushes previous apps and jobs in the db before -# submitting the test_balsam app/job script. +# submitting the test_balsam_hworld app/job script. # # Most of this comes from scaling_tests/forces/balsam_local.sh # Can't run this line in calling Python file. Balsam installation hasn't been # noticed by the Python runtime yet. python -c 'from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage; modify_Balsam_pyCoverage()' -export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py +export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam_hworld.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam export LIBE_WALLCLOCK=3 export THIS_DIR=$PWD -export SCRIPT_BASENAME=script_test_balsam +export SCRIPT_BASENAME=script_test_balsam_hworld # Set proper permissions, initialize Balsam DB, activate DB export BALSAM_DB_PATH='~/test-balsam' @@ -30,8 +30,8 @@ source balsamactivate test-balsam balsam rm apps --all --force balsam rm jobs --all --force -# Submit script_test_balsam as app +# Submit script_test_balsam_hworld as app balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" -# Submit job based on script_test_balsam app +# Submit job based on script_test_balsam_hworld app balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 140ecf2c0..853c84f8e 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -1,23 +1,33 @@ #! /usr/bin/env python -# Installing Balsam consists of both cloning and pip installing the latest version -# of Balsam from source but also moving the Balsam-relevant test to the regression_tests -# directory. This way, run-tests won't run a non-relevant test if Balsam isn't -# installed or the necessary Python version isn't installed. +# Configuring Balsam for libEnsemble consists of installing the latest version +# of Balsam from source and moving Balsam-relevant tests to the regression_tests +# directory. This way, run-tests won't run a non-relevant test if Balsam or the +# necessary Python version aren't installed. import sys import os import subprocess -balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' - -if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ +def install_balsam(): + # Installs Balsam in a directory on the same level as the current directory. + here = os.getcwd() + balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' subprocess.check_call(balsamclone.split()) os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) - os.chdir('../libensemble') - if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'): - os.rename('./conda/test_balsam.py', - './libensemble/tests/regression_tests/test_balsam.py') + os.chdir(here) + + +def move_test_balsam(balsam_test): + # Moves specified test from /conda to /regression_tests + reg_dir_with_btest = os.path.join('./libensemble/tests/regression_tests', balsam_test) + if not os.path.isfile(reg_dir_with_btest): + os.rename('./conda/{}'.format(balsam_test), reg_dir_with_btest) + + +if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ + install_balsam() + move_test_balsam('test_balsam_hworld.py') subprocess.run('./conda/configure-balsam-test.sh'.split()) diff --git a/conda/test_balsam.py b/conda/test_balsam.py deleted file mode 100644 index d64697aa0..000000000 --- a/conda/test_balsam.py +++ /dev/null @@ -1,95 +0,0 @@ -import subprocess -import os -import time -import sys -import libensemble -from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker - -# TESTSUITE_COMMS: local -# TESTSUITE_NPROCS: 3 - -# This test is NOT submitted as a job to Balsam. That would be script_test_balsam.py -# This test executes that job through the 'runstr' line defined further down. - -nworkers, is_master, libE_specs, _ = parse_args() # None used. Bug-prevention - -# Set libensemble base directory to environment variable for Balsam coverage -# orientation purposes. Otherwise coverage in the Balsam data directory is -# (unsuccessfully) collected. -libepath = os.path.dirname(libensemble.__file__) -os.environ['LIBE_PATH'] = libepath - -# Balsam is meant for HPC systems that commonly distribute jobs across many -# nodes. For our purposes, we append (hack) ten workers to Balsam's WorkerGroup -print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd())) -modify_Balsam_worker() - -# By this point, script_test_balsam.py has been submitted as an app and job to Balsam -# This line launches the queued job in the Balsam database -runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' -print('Executing Balsam job with command: {}'.format(runstr)) -subprocess.Popen(runstr.split()) - -# Location of Balsam DB location defined in configure-balsam-test.sh -basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam') - -# Periodically wait for Workflow and Job directory within Balsam DB -sleeptime = 0 -print('{}: Waiting for Workflow directory'.format(sleeptime)) -while not os.path.isdir(basedb) and sleeptime < 58: - sleeptime += 1 - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() - time.sleep(1) - -print('{}: Waiting for Job Directory'.format(sleeptime)) -while len(os.listdir(basedb)) == 0 and sleeptime < 58: - sleeptime += 1 - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() - time.sleep(1) - -# Job directory now exists -jobdirname = os.listdir(basedb)[0] -jobdir = os.path.join(basedb, jobdirname) -outscript = os.path.join(jobdir, 'job_script_test_balsam.out') - -# Periodically wait for Balsam Job output -print('{}: Checking for Balsam output file: {}'.format(sleeptime, outscript)) -while not os.path.isfile(outscript) and sleeptime < 58: - sleeptime += 2 - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() - time.sleep(2) - -# Print sections of Balsam output to screen every second until complete -print('{}: Output file exists. Waiting for Balsam Job Output.'.format(sleeptime)) -lastposition = 0 -lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] -while sleeptime < 58: - with open(outscript, 'r') as f: - f.seek(lastposition) # (should) prevent outputting already printed sections - new = f.read() - lastposition = f.tell() - if len(new) > 0: - print(new) - sys.stdout.flush() - if any(new.endswith(line) for line in lastlines): - break - print('{}'.format(sleeptime), end=" ") - time.sleep(1) - sleeptime += 1 - -print('{}: Importing any coverage from Balsam.'.format(sleeptime)) -here = os.getcwd() - -# Move coverage files from Balsam DB to ./regression_tests (for concatenation) -covname = '.cov_reg_out.' -for file in os.listdir(jobdir): - if file.startswith(covname): - balsam_cov = os.path.join(jobdir, file) - here_cov = os.path.join(here, file) - print('Moved {} from {} to {}'.format(file, jobdir, here)) - os.rename(balsam_cov, here_cov) - -print('Test complete.') diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py new file mode 100644 index 000000000..900301bd8 --- /dev/null +++ b/conda/test_balsam_hworld.py @@ -0,0 +1,109 @@ +import subprocess +import os +import time +import sys +import libensemble +from libensemble.tests.regression_tests.common import modify_Balsam_worker + +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 3 + +# This test is NOT submitted as a job to Balsam. Instead, script_test_balsam_hworld.py +# This test executes that job through the 'runstr' line in run_Balsam_job() + + +def run_Balsam_job(): + runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1' + print('Executing Balsam job with command: {}'.format(runstr)) + subprocess.Popen(runstr.split()) + + +def wait_for_job_dir(basedb): + sleeptime = 0 + + while not os.path.isdir(basedb) and sleeptime < 15: + time.sleep(1) + sleeptime += 1 + + print('Waiting for Job Directory'.format(sleeptime)) + while len(os.listdir(basedb)) == 0 and sleeptime < 15: + print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() + time.sleep(1) + sleeptime += 1 + + jobdirname = os.listdir(basedb)[0] + jobdir = os.path.join(basedb, jobdirname) + return jobdir + + +def wait_for_job_output(jobdir): + sleeptime = 0 + + output = os.path.join(jobdir, 'job_script_test_balsam_hworld.out') + print('Checking for Balsam output file: {}'.format(output)) + + while not os.path.isfile(output) and sleeptime < 30: + print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() + time.sleep(2) + sleeptime += 2 + + return output + + +def print_job_output(outscript): + sleeptime = 0 + + print('Output file found. Waiting for complete Balsam Job Output.') + lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] + lastposition = 0 + + while sleeptime < 60: + with open(outscript, 'r') as f: + f.seek(lastposition) + new = f.read() + lastposition = f.tell() + + if len(new) > 0: + print(new) + else: + print('{}'.format(sleeptime), end=" ") + sys.stdout.flush() + + if any(new.endswith(line) for line in lastlines): + break + + time.sleep(1) + sleeptime += 1 + + +def move_job_coverage(jobdir): + # Move coverage files from Balsam DB to ./regression_tests (for concatenation) + here = os.getcwd() + covname = '.cov_reg_out.' + + for file in os.listdir(jobdir): + if file.startswith(covname): + balsam_cov = os.path.join(jobdir, file) + here_cov = os.path.join(here, file) + os.rename(balsam_cov, here_cov) + + +if __name__ == '__main__': + + # For Balsam-specific Coverage config file, to not evaluate Balsam data dir + libepath = os.path.dirname(libensemble.__file__) + os.environ['LIBE_PATH'] = libepath + + basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam') + + modify_Balsam_worker() + run_Balsam_job() + + jobdir = wait_for_job_dir(basedb) + output = wait_for_job_output(jobdir) + print_job_output(output) + move_job_coverage(jobdir) + + print('Test complete.') diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 12518c49b..65f8ff953 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -176,9 +176,7 @@ def modify_Balsam_worker(): # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which # usually only contain a single node, we need to change Balsam's default - # worker setup so multiple workers can be run on a single node (until this - # feature is [hopefully] added!).For our purposes, we append ten workers - # to Balsam's WorkerGroup + # worker setup so multiple workers can be run on a single node. import balsam new_lines = [" for idx in range(10):\n", @@ -200,19 +198,18 @@ def modify_Balsam_worker(): for line in lines: f.write(line) - print("Modified worker file in {}".format(balsam_path)) - def modify_Balsam_pyCoverage(): - # Tracking line coverage of our code through our tests requires running a test - # with the format 'python -m coverage run test.py args'. Balsam explicitely - # configures Python runs with 'python test.py args' with no current - # capability for specifying runtime Python options. This hack attempts to - # resolve this for our purposes only. + # Tracking line coverage through our tests requires running the Python module + # 'coverage' directly. Balsam explicitely configures Python runs with + # 'python [script].py args' with no current capability for specifying + # modules. This hack specifies the coverage module and some options. import balsam old_line = " path = ' '.join((exe, script_path, args))\n" - new_line = " path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.bal_coveragerc', script_path, args))\n" + new_line = " path = ' '.join((exe, '-m coverage run " + \ + "--parallel-mode --rcfile=./libensemble/tests/regression_tests/" + \ + ".bal_coveragerc', script_path, args))\n" commandfile = 'cli_commands.py' balsam_path = os.path.dirname(balsam.__file__) + '/scripts' @@ -228,5 +225,3 @@ def modify_Balsam_pyCoverage(): with open(balsam_commands_path, 'w') as f: for line in lines: f.write(line) - - print("Modified cli_commands file in {}".format(balsam_path)) diff --git a/libensemble/tests/regression_tests/script_test_balsam.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py similarity index 91% rename from libensemble/tests/regression_tests/script_test_balsam.py rename to libensemble/tests/regression_tests/script_test_balsam_hworld.py index d606bd099..fd9430229 100644 --- a/libensemble/tests/regression_tests/script_test_balsam.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -1,5 +1,5 @@ # This script is submitted as an app and job to Balsam. The job submission is -# via 'balsam launch' executed in the test_balsam.py script. +# via 'balsam launch' executed in the test_balsam_hworld.py script. import os import numpy as np @@ -71,17 +71,11 @@ def build_simfunc(): if is_master: print('\nChecking expected job status against Workers ...\n') - - # Expected list: Last is zero as will not be entered into H array on - # manager kill - but should show in the summary file. - # Repeat expected lists nworkers times and compare with list of status's - # received from workers calc_status_list_in = np.asarray([WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED, 0]) calc_status_list = np.repeat(calc_status_list_in, nworkers) - # For debug print("Expecting: {}".format(calc_status_list)) print("Received: {}\n".format(H['cstat'])) From 0a29517982a6df9e290c57840407885ba9653634 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 09:55:07 -0500 Subject: [PATCH 187/644] Printing stuff --- libensemble/gen_funcs/support.py | 5 +++++ libensemble/libE_manager.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index e5084828d..75da6303a 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -16,6 +16,10 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } + if O['sim_id'] > 200 or O['sim_id'] < 0 : + import sys + sys.exit('Bad send') + comm.send(EVAL_GEN_TAG, D) @@ -27,4 +31,5 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() + print("Next to receive:", calc_in['sim_id'],flush=True) return tag, Work, calc_in diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 2a1c24a46..dc18028ec 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -252,6 +252,11 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: + print('Outside_recieve:', D_recv['calc_out']['sim_id'],flush=True) + if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): + import ipdb; ipdb.set_trace() + print("HA", D_recv['calc_out']['sim_id'], self.index) + self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ "Gen must return work when is is the only thing active." From c3821af8a7a709c8d394b1464ed38c66af7ca2d3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 10:41:24 -0500 Subject: [PATCH 188/644] Commenting-out prints and removing local from test --- libensemble/gen_funcs/support.py | 8 ++++---- libensemble/libE_manager.py | 8 ++++---- .../test_6-hump_camel_persistent_aposmm.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 75da6303a..d8774c2a1 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -16,9 +16,9 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - if O['sim_id'] > 200 or O['sim_id'] < 0 : - import sys - sys.exit('Bad send') + # if O['sim_id'] > 200 or O['sim_id'] < 0: + # import sys + # sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) @@ -31,5 +31,5 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() - print("Next to receive:", calc_in['sim_id'],flush=True) + # print("Next to receive:", calc_in['sim_id'], flush=True) return tag, Work, calc_in diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index dc18028ec..07610ae65 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -252,10 +252,10 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: - print('Outside_recieve:', D_recv['calc_out']['sim_id'],flush=True) - if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): - import ipdb; ipdb.set_trace() - print("HA", D_recv['calc_out']['sim_id'], self.index) + # print('Outside_recieve:', D_recv['calc_out']['sim_id'], flush=True) + # if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): + # import ipdb; ipdb.set_trace() + # print("HA", D_recv['calc_out']['sim_id'], self.index) self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index c6d0a1dc5..6e17ea272 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_COMMS: mpi tcp # TESTSUITE_NPROCS: 3 4 import sys From 477161cccf96745169a5b70d0010d0b6bbb6ef7f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 10:47:14 -0500 Subject: [PATCH 189/644] Reverting and leaving example with (broken) prints --- libensemble/gen_funcs/support.py | 8 ++++---- libensemble/libE_manager.py | 8 ++++---- .../test_6-hump_camel_persistent_aposmm.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index d8774c2a1..75da6303a 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -16,9 +16,9 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - # if O['sim_id'] > 200 or O['sim_id'] < 0: - # import sys - # sys.exit('Bad send') + if O['sim_id'] > 200 or O['sim_id'] < 0 : + import sys + sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) @@ -31,5 +31,5 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() - # print("Next to receive:", calc_in['sim_id'], flush=True) + print("Next to receive:", calc_in['sim_id'],flush=True) return tag, Work, calc_in diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 07610ae65..dc18028ec 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -252,10 +252,10 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: - # print('Outside_recieve:', D_recv['calc_out']['sim_id'], flush=True) - # if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): - # import ipdb; ipdb.set_trace() - # print("HA", D_recv['calc_out']['sim_id'], self.index) + print('Outside_recieve:', D_recv['calc_out']['sim_id'],flush=True) + if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): + import ipdb; ipdb.set_trace() + print("HA", D_recv['calc_out']['sim_id'], self.index) self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 6e17ea272..7921717df 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi tcp +# TESTSUITE_COMMS: mpi local tcp # TESTSUITE_NPROCS: 3 4 import sys @@ -66,7 +66,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 1000} +exit_criteria = {'sim_max': 200} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 165af8d8428ca6d5e9be13b5f5ba3402ed5d9091 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 11:27:10 -0500 Subject: [PATCH 190/644] Fixing 1d FutureWarnings --- .../tests/regression_tests/test_inverse_bayes_example.py | 2 +- libensemble/tests/unit_tests/setup.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index fbe0100af..bcbec1a3d 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -37,7 +37,7 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': [('x', float, 2), ('batch', int), ('subbatch', int), - ('prior', float, 1), ('prop', float, 1), ('weight', float, 1)], + ('prior', float), ('prop', float), ('weight', float)], 'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'subbatch_size': 3, diff --git a/libensemble/tests/unit_tests/setup.py b/libensemble/tests/unit_tests/setup.py index 592c52ed5..3f3f8173b 100644 --- a/libensemble/tests/unit_tests/setup.py +++ b/libensemble/tests/unit_tests/setup.py @@ -8,7 +8,11 @@ def make_criteria_and_specs_0(simx=10, n=1): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x_on_cube'], 'out': [('f', float), ('fvec', float, 3)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x_on_cube', float, n), ('priority', float), ('local_pt', bool), ('local_min', bool), ('num_active_runs', int)], 'ub': np.ones(n), 'lb': np.zeros(n), 'nu': 0} + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('priority', float), ('local_pt', bool), ('local_min', bool), ('num_active_runs', int)], 'ub': np.ones(n), 'lb': np.zeros(n), 'nu': 0} + if n == 1: + gen_specs['out'] += [('x_on_cube', float)] + else: + gen_specs['out'] += [('x_on_cube', float, n)] exit_criteria = {'sim_max': simx} return sim_specs, gen_specs, exit_criteria From e9e75d85fe2fcb35175baafceb596864358d9073 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 13:46:56 -0500 Subject: [PATCH 191/644] flake8 --- libensemble/tests/unit_tests/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/unit_tests/setup.py b/libensemble/tests/unit_tests/setup.py index 3f3f8173b..66114bcfe 100644 --- a/libensemble/tests/unit_tests/setup.py +++ b/libensemble/tests/unit_tests/setup.py @@ -9,7 +9,7 @@ def make_criteria_and_specs_0(simx=10, n=1): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x_on_cube'], 'out': [('f', float), ('fvec', float, 3)], } gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('priority', float), ('local_pt', bool), ('local_min', bool), ('num_active_runs', int)], 'ub': np.ones(n), 'lb': np.zeros(n), 'nu': 0} - if n == 1: + if n == 1: gen_specs['out'] += [('x_on_cube', float)] else: gen_specs['out'] += [('x_on_cube', float, n)] From 1eca03c8b041ef892db6934cdd5ade11dba10829 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 28 Aug 2019 14:57:04 -0500 Subject: [PATCH 192/644] Update history comparison script --- postproc_scripts/compare_npy.py | 19 ++++++++++++++++--- postproc_scripts/readme.rst | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/postproc_scripts/compare_npy.py b/postproc_scripts/compare_npy.py index cb274ffbc..0489ff512 100755 --- a/postproc_scripts/compare_npy.py +++ b/postproc_scripts/compare_npy.py @@ -1,16 +1,29 @@ #!/usr/bin/env python +'''Script to compare libEnsemble history arrays in files. + +If two *.npy files are provided they are compared with each other. +If one *.npy file is provided if is compared with a hard-coded expected file +(by default located at ../expected.npy) +Default NumPy tolerances are used for comparison (rtol=1e-05, atol=1e-08) and +Nans compare as equal. Variable fields (such as those containing a time) +are ignored. +''' + import sys import numpy as np -if len(sys.argv) > 1: +if len(sys.argv) > 2: + results = np.load(sys.argv[1]) + exp_results = np.load(sys.argv[2]) +elif len(sys.argv) > 1: results = np.load(sys.argv[1]) + exp_results_file = "../expected.npy" + exp_results = np.load(exp_results_file) else: print('You need to supply an .npy file - aborting') sys.exit() -exp_results_file = "../expected.npy" -exp_results = np.load(exp_results_file) exclude_fields = ['gen_worker', 'sim_worker', 'gen_time', 'given_time'] # list of fields to ignore locate_mismatch = True diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index b2a102d36..1bed83f25 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -21,4 +21,4 @@ Results analysis scripts ./print_npy.py run_libe_forces_results_History_length=1000_evals=8.npy done -* **compare_npy.py**: Compares a provided ``*.npy`` file with an expected results file (by default located at ../expected.npy). A tolerance is given on floating point results, nans are compared as equal, and variable fields (such as those containing a time) are ignored. These fields may need to be modified depending on user's history array. +* **compare_npy.py**: Compares either two provided ``*.npy`` files or one provided ``*.npy`` file with an expected results file (by default located at ../expected.npy). A tolerance is given on floating point results and NANs are compared as equal. Variable fields (such as those containing a time) are ignored. These fields may need to be modified depending on user's history array. From 66bd418ab6b483e48d0d3e60754599c1efbcc7c2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 28 Aug 2019 16:01:02 -0500 Subject: [PATCH 193/644] Removing local until we can resolve the issue with nested multiprocessing --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index c6d0a1dc5..6e17ea272 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_COMMS: mpi tcp # TESTSUITE_NPROCS: 3 4 import sys From 09752d69e681518082eba8df5ca668874af5964a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 29 Aug 2019 08:28:02 -0500 Subject: [PATCH 194/644] Update .travis.yml Agreeing with `develop` version --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df3df5dc4..b1cee2631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,6 @@ install: - pip install -e . - python conda/install-balsam.py - before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh From d3c7928e8bf9a63b6049083c2c59178ae405b1db Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 29 Aug 2019 12:39:53 -0500 Subject: [PATCH 195/644] Putting in a pause and still having the issue. --- libensemble/gen_funcs/persistent_aposmm.py | 1 + libensemble/libE_manager.py | 4 ++-- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 5746b5364..b1a19347b 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -223,6 +223,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) + time.sleep(5) tag = None max_active_runs = gen_specs.get('max_active_runs', np.inf) waiting_starting_inds = [] diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index dc18028ec..481c6c9ae 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -253,9 +253,9 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: print('Outside_recieve:', D_recv['calc_out']['sim_id'],flush=True) - if (D_recv['calc_out']['sim_id'] > 200) or (D_recv['calc_out']['sim_id'] < 0): + if D_recv['calc_out']['sim_id'] != self.hist.index: import ipdb; ipdb.set_trace() - print("HA", D_recv['calc_out']['sim_id'], self.index) + print("HA", D_recv['calc_out']['sim_id'], self.hist.index) self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 7921717df..f3e4e99ae 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -66,7 +66,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 200} +exit_criteria = {'sim_max': 100} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 77d3b803be48029da35fa474b2e03554eb8161f0 Mon Sep 17 00:00:00 2001 From: shudson Date: Fri, 30 Aug 2019 15:51:25 -0500 Subject: [PATCH 196/644] Add srun option to mpi_controller --- libensemble/mpi_controller.py | 3 ++- libensemble/resources.py | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index 31991bbf1..ec3c7ccfd 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -87,7 +87,8 @@ def __init__(self, auto_resources=True, central_mode=False, 'aprun': ['aprun', '-e {env}', '-L {hostlist}', '-n {num_procs}', '-N {ranks_per_node}'], - 'jsrun': ['jsrun', '--np {num_procs}'] + 'jsrun': ['jsrun', '--np {num_procs}'], # Need to add more + 'srun': ['srun', '-w {hostlist}', '-n {num_procs}'] # Need to add more } self.mpi_launch_type = MPIResources.get_MPI_variant() self.mpi_command = mpi_commands[self.mpi_launch_type] diff --git a/libensemble/resources.py b/libensemble/resources.py index 9d143bba2..a5254499c 100644 --- a/libensemble/resources.py +++ b/libensemble/resources.py @@ -150,13 +150,23 @@ def get_MPI_variant(): except OSError: pass - # Explore mpi4py.MPI.get_vendor() and mpi4py.MPI.Get_library_version() for mpi4py - try_mpich = subprocess.Popen(['mpirun', '-npernode'], stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - stdout, _ = try_mpich.communicate() - if 'unrecognized argument npernode' in stdout.decode(): - return 'mpich' - return 'openmpi' + try: + # Explore mpi4py.MPI.get_vendor() and mpi4py.MPI.Get_library_version() for mpi4py + try_mpich = subprocess.Popen(['mpirun', '-npernode'], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + stdout, _ = try_mpich.communicate() + if 'unrecognized argument npernode' in stdout.decode(): + return 'mpich' + return 'openmpi' + except Exception as e: + print('Testing: Error on MPI command: {}'.format(e)) + pass + + try: + subprocess.check_call(['srun', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return 'srun' + except OSError: + pass # --------------------------------------------------------------------------- From 77f310e9ea813bb766759b8570f710a519e27ee8 Mon Sep 17 00:00:00 2001 From: shudson Date: Fri, 30 Aug 2019 15:53:22 -0500 Subject: [PATCH 197/644] Correction to get_slurm_nodelist --- libensemble/env_resources.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 9d1f118da..3271c8127 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -112,13 +112,13 @@ def get_slurm_nodelist(node_list_env): fullstr = os.environ[node_list_env] if not fullstr: return [] - splitstr = fullstr.split('-', 1) + splitstr = fullstr.split('[', 1) prefix = splitstr[0] - nidstr = splitstr[1].strip("[]") + nidstr = splitstr[1].strip("]") for nidgroup in nidstr.split(','): a, b, nnum_len = EnvResources._range_split(nidgroup) for nid in range(a, b): - nidlst.append(prefix + '-' + str(nid).zfill(nnum_len)) + nidlst.append(prefix + str(nid).zfill(nnum_len)) return sorted(nidlst) @staticmethod From 32241157a445979ec46158502291e84affe86f4f Mon Sep 17 00:00:00 2001 From: shudson Date: Fri, 30 Aug 2019 17:30:15 -0500 Subject: [PATCH 198/644] Fix single node case and add tests --- libensemble/env_resources.py | 2 ++ .../tests/unit_tests/test_env_resources.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 3271c8127..2225bf795 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -113,6 +113,8 @@ def get_slurm_nodelist(node_list_env): if not fullstr: return [] splitstr = fullstr.split('[', 1) + if len(splitstr) == 1: + return splitstr prefix = splitstr[0] nidstr = splitstr[1].strip("]") for nidgroup in nidstr.split(','): diff --git a/libensemble/tests/unit_tests/test_env_resources.py b/libensemble/tests/unit_tests/test_env_resources.py index 59169701e..b63bbd0fa 100644 --- a/libensemble/tests/unit_tests/test_env_resources.py +++ b/libensemble/tests/unit_tests/test_env_resources.py @@ -38,6 +38,13 @@ def test_slurm_nodelist_single(): assert nodelist == exp_out, "Nodelist returned does not match expected" +def test_slurm_nodelist_single_nodash(): + os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "nid00056" + exp_out = ["nid00056"] + nodelist = EnvResources.get_slurm_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST") + assert nodelist == exp_out, "Nodelist returned does not match expected" + + def test_slurm_nodelist_knl_seq(): os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0009-0012]" exp_out = ['knl-0009', 'knl-0010', 'knl-0011', 'knl-0012'] @@ -59,6 +66,13 @@ def test_slurm_nodelist_groups(): assert nodelist == exp_out, "Nodelist returned does not match expected" +def test_slurm_nodelist_groups_nodash(): + os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "nid0[0020-0022,0137-0139,1234]" + exp_out = ['nid00020', 'nid00021', 'nid00022', 'nid00137', 'nid00138', 'nid00139', 'nid01234'] + nodelist = EnvResources.get_slurm_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST") + assert nodelist == exp_out, "Nodelist returned does not match expected" + + def test_slurm_nodelist_groups_longprefix(): os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "super-[000020-000022,000137-000139,001234,023456-023458]" exp_out = ['super-000020', 'super-000021', 'super-000022', 'super-000137', 'super-000138', 'super-000139', @@ -191,9 +205,11 @@ def test_abbrev_nodenames_cobalt(): test_slurm_nodelist_empty() test_slurm_nodelist_single() + test_slurm_nodelist_single_nodash() test_slurm_nodelist_knl_seq() test_slurm_nodelist_bdw_seq() test_slurm_nodelist_groups() + test_slurm_nodelist_groups_nodash() test_slurm_nodelist_groups_longprefix() test_slurm_nodelist_reverse_grp() From 80ff1d59d5b64e37d7d2e7a28e73c1e33e2eac9f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 4 Sep 2019 09:52:29 -0500 Subject: [PATCH 199/644] Trying to debug too many files open error on OSX+Python-multiprocessing --- libensemble/gen_funcs/persistent_aposmm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 5746b5364..9e0f80474 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -259,6 +259,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if sim_id_to_child_indices.get(sim_id_recv): for child_idx in sim_id_to_child_indices[sim_id_recv]: + print(np.sum([i.is_running for i in local_opters])) x_new = local_opters[child_idx].iterate(*data_to_give_processes) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x From bb167d9e3bfc142d3050235328e2d9efff667b12 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 4 Sep 2019 09:57:28 -0500 Subject: [PATCH 200/644] Trying to join and then terminate processes --- libensemble/gen_funcs/persistent_aposmm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 9e0f80474..70daab13d 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -427,6 +427,7 @@ def iterate(self, f, grad=None): x_new = self.comm_queue.get() if isinstance(x_new, ConvergedMsg): self.process.join() + self.process.terminate() self.is_running = False return x_new From ad3659a0dc9038cc46f7c475044c2aa2c0faae97 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 10:41:41 -0500 Subject: [PATCH 201/644] experimental sleep on worker -> manager message to prevent data corruption --- libensemble/gen_funcs/support.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 75da6303a..231daf410 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -1,5 +1,7 @@ from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, UNSET_TAG, EVAL_GEN_TAG +import time + def sendrecv_mgr_worker_msg(comm, O, status=None): """Send message from worker to manager and receive response. @@ -16,11 +18,12 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - if O['sim_id'] > 200 or O['sim_id'] < 0 : + if O['sim_id'] > 200 or O['sim_id'] < 0 : import sys sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) + time.sleep(0.01) def get_mgr_worker_msg(comm, status=None): From 9d282a95b227b10f974d07de9bb88392fc41423b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 4 Sep 2019 10:52:24 -0500 Subject: [PATCH 202/644] Removing terminate (which should be unnecessary) --- libensemble/gen_funcs/persistent_aposmm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 70daab13d..9e0f80474 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -427,7 +427,6 @@ def iterate(self, f, grad=None): x_new = self.comm_queue.get() if isinstance(x_new, ConvergedMsg): self.process.join() - self.process.terminate() self.is_running = False return x_new From 7e038fba6b9d5d46e4cd867ddadaf16d496e3fef Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 14:57:36 -0500 Subject: [PATCH 203/644] comments on sleep, reenables local (for testing) --- libensemble/gen_funcs/support.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 231daf410..2529ac2ce 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -23,8 +23,7 @@ def send_mgr_worker_msg(comm, O): sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) - time.sleep(0.01) - + time.sleep(0.01) # May prevent queue corruption. Investigate def get_mgr_worker_msg(comm, status=None): """Get message to worker from manager. diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 97ab7a3f8..f3e4e99ae 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi tcp +# TESTSUITE_COMMS: mpi local tcp # TESTSUITE_NPROCS: 3 4 import sys From d935007644277c47c7c6acbe6675894f30dc5917 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 14:59:39 -0500 Subject: [PATCH 204/644] Adds ulimit adjustment to travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b1cee2631..4cc24d54c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,8 @@ before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - source setbalsampath.sh + # Allow 10000 files to be open at once (critical for persistent_aposmm) + - ulimit -Sn 10000 # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; From ebf39c5b4902b9473e7f5ccb97873736cb57a113 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 15:15:54 -0500 Subject: [PATCH 205/644] flake8, comment out debug statements --- libensemble/gen_funcs/persistent_aposmm.py | 1 - libensemble/gen_funcs/support.py | 10 +++++----- libensemble/libE_manager.py | 8 ++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index b1a19347b..5746b5364 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -223,7 +223,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) - time.sleep(5) tag = None max_active_runs = gen_specs.get('max_active_runs', np.inf) waiting_starting_inds = [] diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 2529ac2ce..f7e14b69c 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -1,5 +1,4 @@ from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, UNSET_TAG, EVAL_GEN_TAG - import time @@ -18,13 +17,14 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - if O['sim_id'] > 200 or O['sim_id'] < 0 : - import sys - sys.exit('Bad send') + # if O['sim_id'] > 200 or O['sim_id'] < 0: + # import sys + # sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) time.sleep(0.01) # May prevent queue corruption. Investigate + def get_mgr_worker_msg(comm, status=None): """Get message to worker from manager. """ @@ -33,5 +33,5 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() - print("Next to receive:", calc_in['sim_id'],flush=True) + # print("Next to receive:", calc_in['sim_id'], flush=True) return tag, Work, calc_in diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 36c98b89f..3a08afcbe 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -294,10 +294,10 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: - print('Outside_recieve:', D_recv['calc_out']['sim_id'],flush=True) - if D_recv['calc_out']['sim_id'] != self.hist.index: - import ipdb; ipdb.set_trace() - print("HA", D_recv['calc_out']['sim_id'], self.hist.index) + # print('Outside_recieve:', D_recv['calc_out']['sim_id'], flush=True) + # if D_recv['calc_out']['sim_id'] != self.hist.index: + # import ipdb; ipdb.set_trace() + # print("HA", D_recv['calc_out']['sim_id'], self.hist.index) self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ From 105da229390baae999b11506d1518cb987b0a8e9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 15:32:24 -0500 Subject: [PATCH 206/644] update exit_criteria --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index f3e4e99ae..c6d0a1dc5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -66,7 +66,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 100} +exit_criteria = {'sim_max': 1000} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, From 770e814ba98a39737b5b0b1e45e1ecccfbbc9f5e Mon Sep 17 00:00:00 2001 From: shudson Date: Fri, 6 Sep 2019 12:53:38 -0500 Subject: [PATCH 207/644] Add srun options --- libensemble/mpi_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index ec3c7ccfd..c5e5f2ec8 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -88,7 +88,8 @@ def __init__(self, auto_resources=True, central_mode=False, '-L {hostlist}', '-n {num_procs}', '-N {ranks_per_node}'], 'jsrun': ['jsrun', '--np {num_procs}'], # Need to add more - 'srun': ['srun', '-w {hostlist}', '-n {num_procs}'] # Need to add more + 'srun': ['srun', '-w {hostlist}', '-n {num_procs}', + '--ntasks-per-node {ranks_per_node}'] # Need to add more } self.mpi_launch_type = MPIResources.get_MPI_variant() self.mpi_command = mpi_commands[self.mpi_launch_type] From 5801e947c85704cf221ffb72dd42c84d113bccc0 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 6 Sep 2019 15:52:01 -0500 Subject: [PATCH 208/644] Trying to communicate only in batches --- .../alloc_funcs/persistent_aposmm_alloc.py | 53 +++++ libensemble/gen_funcs/persistent_aposmm.py | 198 ++++++------------ .../test_6-hump_camel_persistent_aposmm.py | 2 +- 3 files changed, 122 insertions(+), 131 deletions(-) create mode 100644 libensemble/alloc_funcs/persistent_aposmm_alloc.py diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py new file mode 100644 index 000000000..652d85f8e --- /dev/null +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -0,0 +1,53 @@ +import numpy as np + +from libensemble.alloc_funcs.support import avail_worker_ids, sim_work, gen_work, count_persis_gens + + +def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): + """ + This allocation function will give simulation work if possible, but + otherwise start up to 1 persistent generator. If all points requested by + the persistent generator have been returned from the simulation evaluation, + then this information is given back to the persistent generator. + + :See: + ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` + """ + + Work = {} + gen_count = count_persis_gens(W) + + # If i is in persistent mode, and any of its calculated values have + # returned, give them back to i. Otherwise, give nothing to i + for i in avail_worker_ids(W, persistent=True): + gen_inds = (H['gen_worker'] == i) + returned_but_not_given = np.logical_and(H['returned'][gen_inds], ~H['given_back'][gen_inds]) + if np.any(returned_but_not_given): + # If a point is a sample point, only return it if all other sample + # points requested at the same time are returned. + last_time_gen_gave_batch = np.unique(H['gen_time'][returned_but_not_given]) + assert(len(last_time_gen_gave_batch)) == 1, "It shouldn't be possible for persistent aposmm to give two batches." + inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] + + if np.all(H[inds_of_last_batch_from_gen]['returned']): + gen_work(Work, i, + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], + np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) + + H['given_back'][inds_of_last_batch_from_gen] = True + + task_avail = ~H['given'] + for i in avail_worker_ids(W, persistent=False): + if np.any(task_avail): + # perform sim evaluations (if they exist in History). + sim_ids_to_send = np.nonzero(task_avail)[0][0] # oldest point + sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), persis_info[i]) + task_avail[sim_ids_to_send] = False + + elif gen_count == 0: + # Finally, call a persistent generator as there is nothing else to do. + gen_count += 1 + gen_work(Work, i, gen_specs['in'], [], persis_info[i], + persistent=True) + + return Work, persis_info diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 5746b5364..b65044a3c 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -190,25 +190,16 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - # {{{ multiprocessing initialization - - # from libensemble.util.forkpdb import ForkablePdb - # ForkablePdb().set_trace() + n, n_s, c_flag, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + # Initialize stuff for localopt children local_opters = [] parent_can_read_from_queue = Event() comm_queue = Queue() - - # }}} - - n, n_s, c_flag, rk_const, ld, mu, nu, total_runs, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) - sim_id_to_child_indices = {} child_id_to_run_id = {} run_order = {} - - # {{{ setting the data needed by the local optimization method - + total_runs = 0 if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', @@ -217,137 +208,74 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) - # }}} - for _ in range(gen_specs['initial_sample_size']): - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, - comm, local_H, sim_id_to_child_indices) + # Send our initial sample.W e don't need to check that n_s is large enough: + # the alloc_func only returns when the initial sample has function values. + send_k_sample_points_for_evaluation(gen_specs['initial_sample_size'],gen_specs, + persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) tag = None - max_active_runs = gen_specs.get('max_active_runs', np.inf) - waiting_starting_inds = [] - while 1: tag, Work, calc_in = get_mgr_worker_msg(comm) - if calc_in: - if fields_to_pass == ['f', 'grad']: - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in - data_to_give_processes = (f_x_recv, grad_f_x_recv) - else: - assert(fields_to_pass == ['f']) - if len(calc_in[0]) == 3: - (x_recv, f_x_recv, sim_id_recv), = calc_in - else: - # FIXME: *BIG BIG!*, even if we are passing grad_f even - # though we are not using it. - (x_recv, f_x_recv, grad_f_x_recv, sim_id_recv), = calc_in - data_to_give_processes = (f_x_recv, ) - if tag in [STOP_TAG, PERSIS_STOP]: - # FIXME: This has to be a clean exit. - - print('[Parent]: The optimal points are:\n', - local_H[np.where(local_H['local_min'])]['x'], flush=True) - - for p in local_opters: - if p.is_running: - p.destroy() + clean_up_and_stop(local_H, local_opters) break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - if sim_id_to_child_indices.get(sim_id_recv): - for child_idx in sim_id_to_child_indices[sim_id_recv]: - x_new = local_opters[child_idx].iterate(*data_to_give_processes) - if isinstance(x_new, ConvergedMsg): - x_opt = x_new.x - update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) - - if waiting_starting_inds: - ind = waiting_starting_inds[0] - waiting_starting_inds = waiting_starting_inds[1:] - - # {{{ initing a local opt run - - local_opter = LocalOptInterfacer(gen_specs, - local_H[ind]['x_on_cube'], - parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv if 'grad' in - fields_to_pass else None) - local_opters.append(local_opter) - x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) - add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) - - run_order[total_runs] = [local_H[-1]['sim_id']] - child_id_to_run_id[len(local_opters)-1] = total_runs - total_runs += 1 + counter = 0 + for row in calc_in: + if sim_id_to_child_indices.get(row['sim_id']): + # Point came from a child local opt run + for child_idx in sim_id_to_child_indices[row['sim_id']]: + x_new = local_opters[child_idx].iterate(row[fields_to_pass]) + if isinstance(x_new, ConvergedMsg): + x_opt = x_new.x + update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) + else: + add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + counter += 1 + run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) - # }}} - - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - else: - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) - else: - add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) - - run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) - if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) - else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (child_idx, ) - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) - else: - if n_s < gen_specs['initial_sample_size']: - continue - n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - - starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, ld, mu, nu) - - for ind in starting_inds: + starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, ld, mu, nu) + for ind in starting_inds: + if len([p for p in local_opters if p.is_running]) < gen_specs.get('max_active_runs', np.inf): local_H['started_run'][ind] = 1 - if len([p for p in local_opters if p.is_running]) < max_active_runs: + # Initialize a local opt run + local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, + local_H[ind]['f'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) - # {{{ initing a local opt run + local_opters.append(local_opter) - local_opter = LocalOptInterfacer(gen_specs, - local_H[ind]['x_on_cube'], - parent_can_read_from_queue, comm_queue, - local_H[ind]['f'], grad_f_x_recv if 'grad' in fields_to_pass else None) - local_opters.append(local_opter) - x_new = local_opter.iterate(*local_H[ind][fields_to_pass]) + x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal - add_to_local_H(local_H, (x_new, ), gen_specs, c_flag, local_flag=1, on_cube=True) - assert np.allclose(local_H[-1]['x_on_cube'], x_new) + add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + counter += 1 - run_order[total_runs] = [local_H[-1]['sim_id']] - child_id_to_run_id[len(local_opters)-1] = total_runs - total_runs += 1 - - if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) - else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + run_order[total_runs] = [ind, local_H[-1]['sim_id']] + child_id_to_run_id[len(local_opters)-1] = total_runs + total_runs += 1 - # }}} - - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + if local_H[-1]['sim_id'] in sim_id_to_child_indices: + sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) else: - waiting_starting_inds.append(ind) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + - if len(starting_inds) == 0 and len(waiting_starting_inds) == 0: - send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + if counter == 0: + send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) + else: + send_mgr_worker_msg(comm, local_H[-counter:][['x', 'sim_id']]) comm_queue.close() comm_queue.join_thread() @@ -406,7 +334,7 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, grad0=None): self.parent_can_read.wait() assert np.allclose(comm_queue.get(), x0) - def iterate(self, f, grad=None): + def iterate(self, data): """ Returns an instance of either :class:`numpy.ndarray` corresponding to the next iterative guess or :class:`ConvergedMsg` when the solver is converged. @@ -415,10 +343,11 @@ def iterate(self, f, grad=None): :param grad: A numpy array of the function's gradient. """ self.parent_can_read.clear() - if grad is None: - self.comm_queue.put((f, )) + + if 'grad' in data.dtype.names: + self.comm_queue.put((data['f'], data['grad'])) else: - self.comm_queue.put((f, grad)) + self.comm_queue.put((data['f'], )) self.child_can_read.set() self.parent_can_read.wait() @@ -427,6 +356,8 @@ def iterate(self, f, grad=None): if isinstance(x_new, ConvergedMsg): self.process.join() self.is_running = False + else: + x_new = np.atleast_2d(x_new) return x_new @@ -1005,8 +936,6 @@ def initialize_APOSMM(H, gen_specs, libE_info): else: nu = 0 - total_runs = 0 - comm = libE_info['comm'] local_H_fields = [('f', float), @@ -1032,17 +961,26 @@ def initialize_APOSMM(H, gen_specs, libE_info): local_H = np.empty(0, dtype=local_H_fields) - return n, n_s, c_flag, rk_c, ld, mu, nu, total_runs, comm, local_H + return n, n_s, c_flag, rk_c, ld, mu, nu, comm, local_H -def send_one_sample_point_for_evaluation(gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): - sampled_points = persis_info['rand_stream'].uniform(0, 1, (1, n)) +def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): + sampled_points = persis_info['rand_stream'].uniform(0, 1, (k, n)) add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) - assert local_H['sim_id'][-1] not in sim_id_to_child_indices - sim_id_to_child_indices[local_H['sim_id'][-1]] = None + assert local_H['sim_id'][-k] not in sim_id_to_child_indices + # sim_id_to_child_indices[local_H['sim_id'][-k:]] = None + + send_mgr_worker_msg(comm, local_H[-k:][['x', 'sim_id']]) + +def clean_up_and_stop(local_H, local_opters): + # FIXME: This has to be a clean exit. - send_mgr_worker_msg(comm, local_H[-1:][['x', 'sim_id']]) + print('[Parent]: The optimal points are:\n', + local_H[np.where(local_H['local_min'])]['x'], flush=True) + for p in local_opters: + if p.is_running: + p.destroy() def display_exception(e): print(e.__doc__) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index c6d0a1dc5..74076c4b1 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -22,7 +22,7 @@ from math import gamma, pi, sqrt from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time From 077ba932aff8f62a4b51d7e65ae281c2bd4b306d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 6 Sep 2019 16:27:41 -0500 Subject: [PATCH 209/644] Now having each child use their own queue, and doing a handshake check with x_on_cube --- .../alloc_funcs/persistent_aposmm_alloc.py | 2 +- libensemble/gen_funcs/persistent_aposmm.py | 32 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 652d85f8e..2edb3fe39 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -31,7 +31,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if np.all(H[inds_of_last_batch_from_gen]['returned']): gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'),('x_on_cube')], np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) H['given_back'][inds_of_last_batch_from_gen] = True diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index b65044a3c..a818b7a53 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -201,10 +201,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): run_order = {} total_runs = 0 if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: - fields_to_pass = ['f', 'grad'] + fields_to_pass = ['x_on_cube', 'f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: - fields_to_pass = ['f'] + fields_to_pass = ['x_on_cube', 'f'] else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) @@ -251,7 +251,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 # Initialize a local opt run - local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], parent_can_read_from_queue, comm_queue, + local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], local_H[ind]['f'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) local_opters.append(local_opter) @@ -275,7 +275,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) else: - send_mgr_worker_msg(comm, local_H[-counter:][['x', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[-counter:][['x', 'x_on_cube', 'sim_id']]) comm_queue.close() comm_queue.join_thread() @@ -289,7 +289,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): class LocalOptInterfacer(object): - def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, grad0=None): + def __init__(self, gen_specs, x0, parent_can_read, f0, grad0=None): """ :param x0: A numpy array of the initial guess solution. This guess should be scaled to a unit cube. @@ -301,9 +301,9 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, grad0=None): immediately after creating the class. """ - self.parent_can_read = parent_can_read + self.parent_can_read = Event() - self.comm_queue = comm_queue + self.comm_queue = Queue() self.child_can_read = Event() self.x0 = x0 @@ -326,13 +326,13 @@ def __init__(self, gen_specs, x0, parent_can_read, comm_queue, f0, grad0=None): self.parent_can_read.clear() self.process = Process(target=run_local_opt, args=(gen_specs, - comm_queue, x0, f0, self.child_can_read, - parent_can_read)) + self.comm_queue, x0, f0, self.child_can_read, + self.parent_can_read)) self.process.start() self.is_running = True self.parent_can_read.wait() - assert np.allclose(comm_queue.get(), x0) + assert np.allclose(self.comm_queue.get(), x0) def iterate(self, data): """ @@ -345,9 +345,9 @@ def iterate(self, data): self.parent_can_read.clear() if 'grad' in data.dtype.names: - self.comm_queue.put((data['f'], data['grad'])) + self.comm_queue.put((data['x_on_cube'], data['f'], data['grad'])) else: - self.comm_queue.put((data['f'], )) + self.comm_queue.put((data['x_on_cube'], data['f'], )) self.child_can_read.set() self.parent_can_read.wait() @@ -386,12 +386,14 @@ def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, gen parent_can_read.set() child_can_read.wait() if gen_specs['localopt_method'] in ['LD_MMA']: - f_recv, grad_recv = comm_queue.get() + x_recv, f_recv, grad_recv = comm_queue.get() grad[:] = grad_recv else: assert gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] - f_recv, = comm_queue.get() + x_recv, f_recv = comm_queue.get() + + assert np.array_equal(x,x_recv), "The point I gave is not the point I got back!" child_can_read.clear() @@ -970,7 +972,7 @@ def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, co assert local_H['sim_id'][-k] not in sim_id_to_child_indices # sim_id_to_child_indices[local_H['sim_id'][-k:]] = None - send_mgr_worker_msg(comm, local_H[-k:][['x', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[-k:][['x', 'x_on_cube', 'sim_id']]) def clean_up_and_stop(local_H, local_opters): # FIXME: This has to be a clean exit. From b80a2c9404fe4c6dcd370d53027b1521f5a56537 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 6 Sep 2019 16:31:59 -0500 Subject: [PATCH 210/644] flake8 --- .../alloc_funcs/persistent_aposmm_alloc.py | 4 +-- libensemble/gen_funcs/persistent_aposmm.py | 28 +++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 2edb3fe39..a4bc95772 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -24,14 +24,14 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info returned_but_not_given = np.logical_and(H['returned'][gen_inds], ~H['given_back'][gen_inds]) if np.any(returned_but_not_given): # If a point is a sample point, only return it if all other sample - # points requested at the same time are returned. + # points requested at the same time are returned. last_time_gen_gave_batch = np.unique(H['gen_time'][returned_but_not_given]) assert(len(last_time_gen_gave_batch)) == 1, "It shouldn't be possible for persistent aposmm to give two batches." inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] if np.all(H[inds_of_last_batch_from_gen]['returned']): gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'),('x_on_cube')], + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'), ('x_on_cube')], np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) H['given_back'][inds_of_last_batch_from_gen] = True diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index a818b7a53..4e62b8f68 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -194,8 +194,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Initialize stuff for localopt children local_opters = [] - parent_can_read_from_queue = Event() - comm_queue = Queue() sim_id_to_child_indices = {} child_id_to_run_id = {} run_order = {} @@ -208,10 +206,9 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) - - # Send our initial sample.W e don't need to check that n_s is large enough: + # Send our initial sample. We don't need to check that n_s is large enough: # the alloc_func only returns when the initial sample has function values. - send_k_sample_points_for_evaluation(gen_specs['initial_sample_size'],gen_specs, + send_k_sample_points_for_evaluation(gen_specs['initial_sample_size'], gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) @@ -220,7 +217,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if tag in [STOP_TAG, PERSIS_STOP]: - clean_up_and_stop(local_H, local_opters) + clean_up_and_stop(local_H, local_opters) break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -251,12 +248,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): local_H['started_run'][ind] = 1 # Initialize a local opt run - local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], + local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], local_H[ind]['f'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) local_opters.append(local_opter) - x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal + x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) counter += 1 @@ -270,15 +267,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) - if counter == 0: send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) - else: + sim_id_to_child_indices) + else: send_mgr_worker_msg(comm, local_H[-counter:][['x', 'x_on_cube', 'sim_id']]) - comm_queue.close() - comm_queue.join_thread() + # comm_queue.close() + # comm_queue.join_thread() for local_opter in local_opters: if local_opter.is_running: @@ -356,7 +352,7 @@ def iterate(self, data): if isinstance(x_new, ConvergedMsg): self.process.join() self.is_running = False - else: + else: x_new = np.atleast_2d(x_new) return x_new @@ -393,7 +389,7 @@ def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, gen 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] x_recv, f_recv = comm_queue.get() - assert np.array_equal(x,x_recv), "The point I gave is not the point I got back!" + assert np.array_equal(x, x_recv), "The point I gave is not the point I got back!" child_can_read.clear() @@ -974,6 +970,7 @@ def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, co send_mgr_worker_msg(comm, local_H[-k:][['x', 'x_on_cube', 'sim_id']]) + def clean_up_and_stop(local_H, local_opters): # FIXME: This has to be a clean exit. @@ -984,6 +981,7 @@ def clean_up_and_stop(local_H, local_opters): if p.is_running: p.destroy() + def display_exception(e): print(e.__doc__) print(e.args) From ce8d1c4977f54f1dbcc0aace4a7d1f36436423ca Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 08:46:00 -0500 Subject: [PATCH 211/644] Removing some unused gen_specs --- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 74076c4b1..7f0e7a20e 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -51,14 +51,11 @@ 'sample_points': np.round(minima, 1), 'localopt_method': 'LD_MMA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'xtol_rel': 1e-3, + 'xtol_rel': 1e-6, 'num_active_gens': 1, 'local_min': True, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, - 'grtol': 1e-4, - 'gatol': 1e-4, - 'tol': 1e-5, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} From f4f56dcb60296e7a445cd03d6a10c0bde37c4e74 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 08:46:54 -0500 Subject: [PATCH 212/644] Not returning to persistent_aposmm until initial sample is complete, and then returning after each evaluation --- .../alloc_funcs/persistent_aposmm_alloc.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index a4bc95772..97a82548b 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -20,21 +20,20 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # If i is in persistent mode, and any of its calculated values have # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): + if sum(H['returned']) < gen_specs['initial_sample_size']: + # Don't return if the initial sample is not complete + continue + gen_inds = (H['gen_worker'] == i) returned_but_not_given = np.logical_and(H['returned'][gen_inds], ~H['given_back'][gen_inds]) if np.any(returned_but_not_given): - # If a point is a sample point, only return it if all other sample - # points requested at the same time are returned. - last_time_gen_gave_batch = np.unique(H['gen_time'][returned_but_not_given]) - assert(len(last_time_gen_gave_batch)) == 1, "It shouldn't be possible for persistent aposmm to give two batches." - inds_of_last_batch_from_gen = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] - - if np.all(H[inds_of_last_batch_from_gen]['returned']): - gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'), ('x_on_cube')], - np.atleast_1d(inds_of_last_batch_from_gen), persis_info[i], persistent=True) - - H['given_back'][inds_of_last_batch_from_gen] = True + inds_to_give = np.where(returned_but_not_given)[0] + + gen_work(Work, i, + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'), ('x_on_cube')], + np.atleast_1d(inds_to_give), persis_info[i], persistent=True) + + H['given_back'][inds_to_give] = True task_avail = ~H['given'] for i in avail_worker_ids(W, persistent=False): From 3dfe4e943a47843b254369621a2ab9bfb9dc1ae4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 08:47:33 -0500 Subject: [PATCH 213/644] close() and join_thread() for queue items --- libensemble/gen_funcs/persistent_aposmm.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 4e62b8f68..63e762b51 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -217,7 +217,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = get_mgr_worker_msg(comm) if tag in [STOP_TAG, PERSIS_STOP]: - clean_up_and_stop(local_H, local_opters) + clean_up_and_stop(local_H, local_opters, run_order) break n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) @@ -273,9 +273,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: send_mgr_worker_msg(comm, local_H[-counter:][['x', 'x_on_cube', 'sim_id']]) - # comm_queue.close() - # comm_queue.join_thread() - for local_opter in local_opters: if local_opter.is_running: raise RuntimeError("[Parent]: Atleast one child process is still active, even after" @@ -285,7 +282,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): class LocalOptInterfacer(object): - def __init__(self, gen_specs, x0, parent_can_read, f0, grad0=None): + def __init__(self, gen_specs, x0, f0, grad0=None): """ :param x0: A numpy array of the initial guess solution. This guess should be scaled to a unit cube. @@ -302,9 +299,9 @@ def __init__(self, gen_specs, x0, parent_can_read, f0, grad0=None): self.comm_queue = Queue() self.child_can_read = Event() - self.x0 = x0 - self.f0 = f0 - self.grad0 = grad0 + self.x0 = x0.copy() + self.f0 = f0.copy() + self.grad0 = grad0.copy() # {{{ setting the local optimization method @@ -351,27 +348,30 @@ def iterate(self, data): x_new = self.comm_queue.get() if isinstance(x_new, ConvergedMsg): self.process.join() + self.comm_queue.close() + self.comm_queue.join_thread() self.is_running = False else: x_new = np.atleast_2d(x_new) return x_new - def destroy(self): - x_new = None - while not isinstance(x_new, ConvergedMsg): + def destroy(self,previous_x): + while not isinstance(previous_x, ConvergedMsg): self.parent_can_read.clear() if self.grad0 is None: - self.comm_queue.put((0*np.ones_like(self.f0),)) + self.comm_queue.put((previous_x, 0*np.ones_like(self.f0),)) else: - self.comm_queue.put((0*np.ones_like(self.f0), np.zeros_like(self.grad0))) + self.comm_queue.put((previous_x,0*np.ones_like(self.f0), np.zeros_like(self.grad0))) self.child_can_read.set() self.parent_can_read.wait() - x_new = self.comm_queue.get() - assert isinstance(x_new, ConvergedMsg) + previous_x = self.comm_queue.get() + assert isinstance(previous_x, ConvergedMsg) self.process.join() + self.comm_queue.close() + self.comm_queue.join_thread() self.is_running = False @@ -971,15 +971,15 @@ def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, co send_mgr_worker_msg(comm, local_H[-k:][['x', 'x_on_cube', 'sim_id']]) -def clean_up_and_stop(local_H, local_opters): +def clean_up_and_stop(local_H, local_opters, run_order): # FIXME: This has to be a clean exit. print('[Parent]: The optimal points are:\n', local_H[np.where(local_H['local_min'])]['x'], flush=True) - for p in local_opters: + for i,p in enumerate(local_opters): if p.is_running: - p.destroy() + p.destroy(local_H['x_on_cube'][run_order[i][-1]]) def display_exception(e): From a98280ae12fc9ac93861f0b2ddf12bb99545eb80 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 08:48:29 -0500 Subject: [PATCH 214/644] flake8 --- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 2 +- libensemble/gen_funcs/persistent_aposmm.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 97a82548b..e92d531c1 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -21,7 +21,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): if sum(H['returned']) < gen_specs['initial_sample_size']: - # Don't return if the initial sample is not complete + # Don't return if the initial sample is not complete continue gen_inds = (H['gen_worker'] == i) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 63e762b51..c7fa02356 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -356,13 +356,13 @@ def iterate(self, data): return x_new - def destroy(self,previous_x): + def destroy(self, previous_x): while not isinstance(previous_x, ConvergedMsg): self.parent_can_read.clear() if self.grad0 is None: self.comm_queue.put((previous_x, 0*np.ones_like(self.f0),)) else: - self.comm_queue.put((previous_x,0*np.ones_like(self.f0), np.zeros_like(self.grad0))) + self.comm_queue.put((previous_x, 0*np.ones_like(self.f0), np.zeros_like(self.grad0))) self.child_can_read.set() self.parent_can_read.wait() @@ -977,7 +977,7 @@ def clean_up_and_stop(local_H, local_opters, run_order): print('[Parent]: The optimal points are:\n', local_H[np.where(local_H['local_min'])]['x'], flush=True) - for i,p in enumerate(local_opters): + for i, p in enumerate(local_opters): if p.is_running: p.destroy(local_H['x_on_cube'][run_order[i][-1]]) From 9d36229308d8dd69c965e865fa28ef1a5228f4f5 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 11:19:43 -0500 Subject: [PATCH 215/644] Now removing any trace of old runs when they are done. --- libensemble/gen_funcs/persistent_aposmm.py | 26 ++++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c7fa02356..95fcaa532 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -193,7 +193,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): n, n_s, c_flag, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) # Initialize stuff for localopt children - local_opters = [] + local_opters = {} sim_id_to_child_indices = {} child_id_to_run_id = {} run_order = {} @@ -231,6 +231,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if isinstance(x_new, ConvergedMsg): x_opt = x_new.x update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) + local_opters.pop(child_id_to_run_id[child_idx]) else: add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) counter += 1 @@ -244,14 +245,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, ld, mu, nu) for ind in starting_inds: - if len([p for p in local_opters if p.is_running]) < gen_specs.get('max_active_runs', np.inf): + if len([p for p in local_opters.values() if p.is_running]) < gen_specs.get('max_active_runs', np.inf): local_H['started_run'][ind] = 1 # Initialize a local opt run local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], local_H[ind]['f'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) - local_opters.append(local_opter) + local_opters[total_runs] = local_opter x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal @@ -259,13 +260,14 @@ def aposmm(H, persis_info, gen_specs, libE_info): counter += 1 run_order[total_runs] = [ind, local_H[-1]['sim_id']] - child_id_to_run_id[len(local_opters)-1] = total_runs - total_runs += 1 + child_id_to_run_id[total_runs] = total_runs if local_H[-1]['sim_id'] in sim_id_to_child_indices: - sim_id_to_child_indices[local_H[-1]['sim_id']] += (len(local_opters)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] += (total_runs, ) else: - sim_id_to_child_indices[local_H[-1]['sim_id']] = (len(local_opters)-1, ) + sim_id_to_child_indices[local_H[-1]['sim_id']] = (total_runs, ) + + total_runs += 1 if counter == 0: send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, @@ -273,10 +275,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): else: send_mgr_worker_msg(comm, local_H[-counter:][['x', 'x_on_cube', 'sim_id']]) - for local_opter in local_opters: - if local_opter.is_running: - raise RuntimeError("[Parent]: Atleast one child process is still active, even after" - " killing all the children.") + # for local_opter in local_opters: + # if local_opter.is_running: + # raise RuntimeError("[Parent]: Atleast one child process is still active, even after" + # " killing all the children.") return local_H, persis_info, tag @@ -977,7 +979,7 @@ def clean_up_and_stop(local_H, local_opters, run_order): print('[Parent]: The optimal points are:\n', local_H[np.where(local_H['local_min'])]['x'], flush=True) - for i, p in enumerate(local_opters): + for i, p in local_opters.items(): if p.is_running: p.destroy(local_H['x_on_cube'][run_order[i][-1]]) From 01f454c96c924780854a5acaa5dd7a53cce9e4aa Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 9 Sep 2019 11:35:38 -0500 Subject: [PATCH 216/644] removes ulimit change --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4cc24d54c..20b85977f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,7 +90,6 @@ before_script: - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - source setbalsampath.sh # Allow 10000 files to be open at once (critical for persistent_aposmm) - - ulimit -Sn 10000 # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; From fcbe6e27681fdda51497cd291caa7fed16b155d6 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 9 Sep 2019 14:12:02 -0500 Subject: [PATCH 217/644] removing child_id_to_run_id --- libensemble/gen_funcs/persistent_aposmm.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 95fcaa532..d25d22b88 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -195,7 +195,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Initialize stuff for localopt children local_opters = {} sim_id_to_child_indices = {} - child_id_to_run_id = {} run_order = {} total_runs = 0 if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: @@ -230,13 +229,13 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opters[child_idx].iterate(row[fields_to_pass]) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x - update_history_optimal(x_opt, local_H, run_order[child_id_to_run_id[child_idx]]) - local_opters.pop(child_id_to_run_id[child_idx]) + update_history_optimal(x_opt, local_H, run_order[child_idx]) + local_opters.pop(child_idx) else: add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) counter += 1 - run_order[child_id_to_run_id[child_idx]].append(local_H[-1]['sim_id']) + run_order[child_idx].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (child_idx, ) else: @@ -260,7 +259,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): counter += 1 run_order[total_runs] = [ind, local_H[-1]['sim_id']] - child_id_to_run_id[total_runs] = total_runs if local_H[-1]['sim_id'] in sim_id_to_child_indices: sim_id_to_child_indices[local_H[-1]['sim_id']] += (total_runs, ) From e32726f1b56be7d57ee9b553b8d96943728dcbaa Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 06:32:35 -0500 Subject: [PATCH 218/644] Communicating gen_specs['out'] to/from manager. --- libensemble/gen_funcs/persistent_aposmm.py | 4 ++-- .../regression_tests/test_6-hump_camel_persistent_aposmm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index d25d22b88..9c4cd11a3 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -271,7 +271,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices) else: - send_mgr_worker_msg(comm, local_H[-counter:][['x', 'x_on_cube', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[-counter:][[i[0] for i in gen_specs['out']]]) # for local_opter in local_opters: # if local_opter.is_running: @@ -968,7 +968,7 @@ def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, co assert local_H['sim_id'][-k] not in sim_id_to_child_indices # sim_id_to_child_indices[local_H['sim_id'][-k:]] = None - send_mgr_worker_msg(comm, local_H[-k:][['x', 'x_on_cube', 'sim_id']]) + send_mgr_worker_msg(comm, local_H[-k:][[i[0] for i in gen_specs['out']]]) def clean_up_and_stop(local_H, local_opters, run_order): diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 7f0e7a20e..a20bacdf5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -41,7 +41,7 @@ 'out': [('f', float), ('grad', float, n)]} gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), - ('local_min', bool)] + ('local_min', bool), ('local_pt',bool)] gen_specs = {'gen_f': gen_f, 'in': [], From c20771232d5a3400959addb5929ce828991f83fa Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 07:09:15 -0500 Subject: [PATCH 219/644] Communicating gen_specs['out'] to/from manager. --- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index e92d531c1..1e10ff619 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -29,8 +29,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if np.any(returned_but_not_given): inds_to_give = np.where(returned_but_not_given)[0] - gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id'), ('x_on_cube')], + gen_work(Work, i, [n[0] for n in sim_specs['out']] + [n[0] for n in gen_specs['out']], np.atleast_1d(inds_to_give), persis_info[i], persistent=True) H['given_back'][inds_to_give] = True From 0e147897ecc2b49b4bce5b7ebc58edc6d257d499 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 07:10:03 -0500 Subject: [PATCH 220/644] Using gen_specs['sample_points'] if available. --- libensemble/gen_funcs/persistent_aposmm.py | 50 ++++++++++++++-------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 9c4cd11a3..885afe600 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -207,9 +207,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Send our initial sample. We don't need to check that n_s is large enough: # the alloc_func only returns when the initial sample has function values. - send_k_sample_points_for_evaluation(gen_specs['initial_sample_size'], gen_specs, - persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + persis_info = add_k_sample_points_to_local_H(gen_specs['initial_sample_size'], gen_specs, + persis_info, n, c_flag, comm, local_H, + sim_id_to_child_indices) + send_mgr_worker_msg(comm, local_H[:gen_specs['initial_sample_size']][[i[0] for i in gen_specs['out']]]) tag = None while 1: @@ -221,7 +222,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) - counter = 0 + new_opt_inds_to_send_mgr = [] + new_inds_to_send_mgr = [] for row in calc_in: if sim_id_to_child_indices.get(row['sim_id']): # Point came from a child local opt run @@ -229,11 +231,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opters[child_idx].iterate(row[fields_to_pass]) if isinstance(x_new, ConvergedMsg): x_opt = x_new.x - update_history_optimal(x_opt, local_H, run_order[child_idx]) + opt_ind = update_history_optimal(x_opt, local_H, run_order[child_idx]) + new_opt_inds_to_send_mgr.append(opt_ind) local_opters.pop(child_idx) else: add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) - counter += 1 + new_inds_to_send_mgr.append(len(local_H)-1) run_order[child_idx].append(local_H[-1]['sim_id']) if local_H[-1]['sim_id'] in sim_id_to_child_indices: @@ -256,7 +259,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) - counter += 1 + new_inds_to_send_mgr.append(len(local_H)-1) run_order[total_runs] = [ind, local_H[-1]['sim_id']] @@ -267,11 +270,12 @@ def aposmm(H, persis_info, gen_specs, libE_info): total_runs += 1 - if counter == 0: - send_k_sample_points_for_evaluation(1, gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) - else: - send_mgr_worker_msg(comm, local_H[-counter:][[i[0] for i in gen_specs['out']]]) + if len(new_inds_to_send_mgr) == 0: + persis_info = add_k_sample_points_to_local_H(1, gen_specs, persis_info, n, + c_flag, comm, local_H, sim_id_to_child_indices) + new_inds_to_send_mgr.append(len(local_H)-1) + + send_mgr_worker_msg(comm, local_H[new_inds_to_send_mgr + new_opt_inds_to_send_mgr][[i[0] for i in gen_specs['out']]]) # for local_opter in local_opters: # if local_opter.is_running: @@ -769,6 +773,8 @@ def update_history_optimal(x_opt, H, run_inds): H['local_min'][opt_ind] = 1 H['num_active_runs'][run_inds] -= 1 + return opt_ind + def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): """ @@ -962,13 +968,21 @@ def initialize_APOSMM(H, gen_specs, libE_info): return n, n_s, c_flag, rk_c, ld, mu, nu, comm, local_H -def send_k_sample_points_for_evaluation(k, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): - sampled_points = persis_info['rand_stream'].uniform(0, 1, (k, n)) - add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) - assert local_H['sim_id'][-k] not in sim_id_to_child_indices - # sim_id_to_child_indices[local_H['sim_id'][-k:]] = None +def add_k_sample_points_to_local_H(k, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): + + if 'sample_points' in gen_specs: + v = np.sum(~local_H['local_pt']) # Number of sample points so far + sampled_points = gen_specs['sample_points'][v:v+k] + on_cube = False # Assume points are on original domain, not unit cube + if len(sampled_points): + add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=on_cube) + k = k-len(sampled_points) + + if k > 0: + sampled_points = persis_info['rand_stream'].uniform(0, 1, (k, n)) + add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) - send_mgr_worker_msg(comm, local_H[-k:][[i[0] for i in gen_specs['out']]]) + return persis_info def clean_up_and_stop(local_H, local_opters, run_order): From 3e20a8353de0db3547da7f5399a72548c25506e3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 07:11:01 -0500 Subject: [PATCH 221/644] Testing that all minima are found --- .../test_6-hump_camel_persistent_aposmm.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index a20bacdf5..985e49095 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -72,4 +72,13 @@ if is_master: print('[Manager]:', H[np.where(H['local_min'])]['x']) print('[Manager]: Time taken =', time() - start_time, flush=True) + + tol = 1e-5 + for m in minima: + # The minima are known on this test problem. + # We use their values to test APOSMM has identified all minima + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)),flush=True) + if np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) > tol: + libE_abort() + save_libE_output(H, persis_info, __file__, nworkers) From c705a8569243bca889564e3316ddbc41da4573b7 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 07:56:03 -0500 Subject: [PATCH 222/644] Using persis_info to speed up alloc_func --- .../alloc_funcs/persistent_aposmm_alloc.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 1e10ff619..a623b5325 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -6,45 +6,47 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): """ This allocation function will give simulation work if possible, but - otherwise start up to 1 persistent generator. If all points requested by + otherwise start a persistent APOSMM generator. If all points requested by the persistent generator have been returned from the simulation evaluation, then this information is given back to the persistent generator. + This function assumes that one persistent APOSMM will be started and never + stopped (until some exit_criterion is satisfied). + :See: - ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` + ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py`` """ Work = {} - gen_count = count_persis_gens(W) + if 'next_to_give' not in persis_info: + persis_info['next_to_give'] = 0 + - # If i is in persistent mode, and any of its calculated values have - # returned, give them back to i. Otherwise, give nothing to i + # If any persistent worker's calculated values have returned, give them back. for i in avail_worker_ids(W, persistent=True): - if sum(H['returned']) < gen_specs['initial_sample_size']: + if persis_info.get('sample_done') or sum(H['returned']) >= gen_specs['initial_sample_size']: # Don't return if the initial sample is not complete - continue + persis_info['sample_done'] = True - gen_inds = (H['gen_worker'] == i) - returned_but_not_given = np.logical_and(H['returned'][gen_inds], ~H['given_back'][gen_inds]) - if np.any(returned_but_not_given): - inds_to_give = np.where(returned_but_not_given)[0] + returned_but_not_given = np.logical_and(H['returned'], ~H['given_back']) + if np.any(returned_but_not_given): + inds_to_give = np.where(returned_but_not_given)[0] - gen_work(Work, i, [n[0] for n in sim_specs['out']] + [n[0] for n in gen_specs['out']], - np.atleast_1d(inds_to_give), persis_info[i], persistent=True) + gen_work(Work, i, [n[0] for n in sim_specs['out']] + [n[0] for n in gen_specs['out']], + np.atleast_1d(inds_to_give), persis_info[i], persistent=True) - H['given_back'][inds_to_give] = True + H['given_back'][inds_to_give] = True - task_avail = ~H['given'] for i in avail_worker_ids(W, persistent=False): - if np.any(task_avail): + if persis_info['next_to_give'] < len(H): # perform sim evaluations (if they exist in History). - sim_ids_to_send = np.nonzero(task_avail)[0][0] # oldest point - sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), persis_info[i]) - task_avail[sim_ids_to_send] = False + sim_work(Work, i, sim_specs['in'], np.atleast_1d(persis_info['next_to_give']), persis_info[i]) + persis_info['next_to_give'] += 1 - elif gen_count == 0: + elif persis_info.get('gen_started') is None: # Finally, call a persistent generator as there is nothing else to do. - gen_count += 1 + persis_info['gen_started'] = True + gen_work(Work, i, gen_specs['in'], [], persis_info[i], persistent=True) From 0bbf8d9d3fe6d6ed4adc1a86e993103feb858f10 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 08:09:53 -0500 Subject: [PATCH 223/644] flake8 --- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 3 +-- libensemble/gen_funcs/persistent_aposmm.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm.py | 7 +++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index a623b5325..3a3035e2d 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -1,6 +1,6 @@ import numpy as np -from libensemble.alloc_funcs.support import avail_worker_ids, sim_work, gen_work, count_persis_gens +from libensemble.alloc_funcs.support import avail_worker_ids, sim_work, gen_work def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): @@ -21,7 +21,6 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if 'next_to_give' not in persis_info: persis_info['next_to_give'] = 0 - # If any persistent worker's calculated values have returned, give them back. for i in avail_worker_ids(W, persistent=True): if persis_info.get('sample_done') or sum(H['returned']) >= gen_specs['initial_sample_size']: diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 885afe600..60a0925bc 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -209,7 +209,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # the alloc_func only returns when the initial sample has function values. persis_info = add_k_sample_points_to_local_H(gen_specs['initial_sample_size'], gen_specs, persis_info, n, c_flag, comm, local_H, - sim_id_to_child_indices) + sim_id_to_child_indices) send_mgr_worker_msg(comm, local_H[:gen_specs['initial_sample_size']][[i[0] for i in gen_specs['out']]]) tag = None @@ -223,7 +223,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) new_opt_inds_to_send_mgr = [] - new_inds_to_send_mgr = [] + new_inds_to_send_mgr = [] for row in calc_in: if sim_id_to_child_indices.get(row['sim_id']): # Point came from a child local opt run diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py index 985e49095..ebfd920d8 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py @@ -41,7 +41,7 @@ 'out': [('f', float), ('grad', float, n)]} gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), - ('local_min', bool), ('local_pt',bool)] + ('local_min', bool), ('local_pt', bool)] gen_specs = {'gen_f': gen_f, 'in': [], @@ -77,8 +77,7 @@ for m in minima: # The minima are known on this test problem. # We use their values to test APOSMM has identified all minima - print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)),flush=True) - if np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) > tol: - libE_abort() + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) + assert np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) < tol save_libE_output(H, persis_info, __file__, nworkers) From 7be87e1c633763d3a3a7626efff01945fe16e4fe Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 10 Sep 2019 10:03:16 -0500 Subject: [PATCH 224/644] use HOME env var instead of ~ or expanduser --- conda/configure-balsam-test.sh | 6 +++--- conda/test_balsam_hworld.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conda/configure-balsam-test.sh b/conda/configure-balsam-test.sh index a284fd92e..56377aa5f 100755 --- a/conda/configure-balsam-test.sh +++ b/conda/configure-balsam-test.sh @@ -19,11 +19,11 @@ export THIS_DIR=$PWD export SCRIPT_BASENAME=script_test_balsam_hworld # Set proper permissions, initialize Balsam DB, activate DB -export BALSAM_DB_PATH='~/test-balsam' +export BALSAM_DB_PATH=$HOME/test-balsam sudo chown -R postgres:postgres /var/run/postgresql sudo chmod a+w /var/run/postgresql -balsam init ~/test-balsam -sudo chmod -R 700 ~/test-balsam/balsamdb +balsam init $HOME/test-balsam +sudo chmod -R 700 $HOME/test-balsam/balsamdb source balsamactivate test-balsam # Refresh DB diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index 900301bd8..94ec92d50 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -95,8 +95,9 @@ def move_job_coverage(jobdir): # For Balsam-specific Coverage config file, to not evaluate Balsam data dir libepath = os.path.dirname(libensemble.__file__) os.environ['LIBE_PATH'] = libepath + home = os.environ['HOME'] - basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam') + basedb = home + '/test-balsam/data/libe_test-balsam' modify_Balsam_worker() run_Balsam_job() From a592345bb230cceb70877999066027a1ed5f7844 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 10 Sep 2019 14:57:38 -0500 Subject: [PATCH 225/644] spring cleaning --- conda/test_balsam_hworld.py | 22 ++++++++----------- .../script_test_balsam_hworld.py | 8 +------ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index 94ec92d50..844847ef6 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -1,15 +1,15 @@ import subprocess import os import time -import sys import libensemble from libensemble.tests.regression_tests.common import modify_Balsam_worker # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 -# This test is NOT submitted as a job to Balsam. Instead, script_test_balsam_hworld.py -# This test executes that job through the 'runstr' line in run_Balsam_job() +# This test is NOT submitted as a job to Balsam. script_test_balsam_hworld.py is +# the executable submitted to Balsam as a job. This test executes that job +# through the 'runstr' line in run_Balsam_job() def run_Balsam_job(): @@ -27,8 +27,7 @@ def wait_for_job_dir(basedb): print('Waiting for Job Directory'.format(sleeptime)) while len(os.listdir(basedb)) == 0 and sleeptime < 15: - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() + print('{}'.format(sleeptime), end=" ", flush=True) time.sleep(1) sleeptime += 1 @@ -44,8 +43,7 @@ def wait_for_job_output(jobdir): print('Checking for Balsam output file: {}'.format(output)) while not os.path.isfile(output) and sleeptime < 30: - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() + print('{}'.format(sleeptime), end=" ", flush=True) time.sleep(2) sleeptime += 2 @@ -66,10 +64,9 @@ def print_job_output(outscript): lastposition = f.tell() if len(new) > 0: - print(new) + print(new, flush=True) else: - print('{}'.format(sleeptime), end=" ") - sys.stdout.flush() + print('{}'.format(sleeptime), end=" ", flush=True) if any(new.endswith(line) for line in lastlines): break @@ -92,12 +89,11 @@ def move_job_coverage(jobdir): if __name__ == '__main__': - # For Balsam-specific Coverage config file, to not evaluate Balsam data dir + # Used by Balsam Coverage config file. Dont evaluate Balsam data dir libepath = os.path.dirname(libensemble.__file__) os.environ['LIBE_PATH'] = libepath - home = os.environ['HOME'] - basedb = home + '/test-balsam/data/libe_test-balsam' + basedb = os.environ['HOME'] + '/test-balsam/data/libe_test-balsam' modify_Balsam_worker() run_Balsam_job() diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index fd9430229..e86ff3b82 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -31,12 +31,6 @@ def build_simfunc(): is_master = MPI.COMM_WORLD.Get_rank() == 0 cores_per_job = 1 -logical_cores = multiprocessing.cpu_count() -cores_all_jobs = nworkers*cores_per_job - -if is_master: - print('\nCores req: {} Cores avail: {}\n'.format(cores_all_jobs, - logical_cores)) sim_app = './my_simjob.x' if not os.path.isfile(sim_app): @@ -63,7 +57,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 30} +exit_criteria = {'elapsed_wallclock_time': 15} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, From c9584caa19198ca75b27714b1790db09a42c347c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 15:32:44 -0500 Subject: [PATCH 226/644] Removing comments --- libensemble/gen_funcs/support.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index f7e14b69c..759eeeb49 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -17,9 +17,6 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - # if O['sim_id'] > 200 or O['sim_id'] < 0: - # import sys - # sys.exit('Bad send') comm.send(EVAL_GEN_TAG, D) time.sleep(0.01) # May prevent queue corruption. Investigate @@ -33,5 +30,5 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() - # print("Next to receive:", calc_in['sim_id'], flush=True) + return tag, Work, calc_in From 8591fa1e151b6a00e5888dce6b93f5d6ede87ec9 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 15:33:20 -0500 Subject: [PATCH 227/644] Update .travis.yml --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 20b85977f..b1cee2631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,6 @@ before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - source setbalsampath.sh - # Allow 10000 files to be open at once (critical for persistent_aposmm) # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; From ffae961894ebafac88912cc3eb2bfbdd419b9332 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 15:33:59 -0500 Subject: [PATCH 228/644] Update support.py --- libensemble/gen_funcs/support.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 759eeeb49..32b05218c 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -1,5 +1,4 @@ from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, UNSET_TAG, EVAL_GEN_TAG -import time def sendrecv_mgr_worker_msg(comm, O, status=None): @@ -19,7 +18,6 @@ def send_mgr_worker_msg(comm, O): } comm.send(EVAL_GEN_TAG, D) - time.sleep(0.01) # May prevent queue corruption. Investigate def get_mgr_worker_msg(comm, status=None): @@ -30,5 +28,4 @@ def get_mgr_worker_msg(comm, status=None): comm.push_back(tag, Work) return tag, None, None _, calc_in = comm.recv() - return tag, Work, calc_in From 45a8d9b773c4e27b64c19c19dc3c997fe8d0371e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 15:34:21 -0500 Subject: [PATCH 229/644] Update support.py --- libensemble/gen_funcs/support.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libensemble/gen_funcs/support.py b/libensemble/gen_funcs/support.py index 32b05218c..e5084828d 100644 --- a/libensemble/gen_funcs/support.py +++ b/libensemble/gen_funcs/support.py @@ -16,7 +16,6 @@ def send_mgr_worker_msg(comm, O): 'calc_status': UNSET_TAG, 'calc_type': EVAL_GEN_TAG } - comm.send(EVAL_GEN_TAG, D) From bb8dbef1242b06ff5578f22d686ef10445fb5e20 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 10 Sep 2019 15:35:15 -0500 Subject: [PATCH 230/644] Update libE_manager.py --- libensemble/libE_manager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 3a08afcbe..0ff9bdba8 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -294,11 +294,6 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv) if calc_type == EVAL_GEN_TAG: - # print('Outside_recieve:', D_recv['calc_out']['sim_id'], flush=True) - # if D_recv['calc_out']['sim_id'] != self.hist.index: - # import ipdb; ipdb.set_trace() - # print("HA", D_recv['calc_out']['sim_id'], self.hist.index) - self.hist.update_history_x_in(w, D_recv['calc_out']) assert len(D_recv['calc_out']) or np.any(self.W['active']), \ "Gen must return work when is is the only thing active." From 965fa05ce190a8820972899db7d3c11ce6c950ba Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 10 Sep 2019 16:41:48 -0500 Subject: [PATCH 231/644] adjusting prints --- conda/test_balsam_hworld.py | 14 ++++++++------ .../regression_tests/script_test_balsam_hworld.py | 3 +-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index 844847ef6..c5cd6fb7c 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -27,7 +27,7 @@ def wait_for_job_dir(basedb): print('Waiting for Job Directory'.format(sleeptime)) while len(os.listdir(basedb)) == 0 and sleeptime < 15: - print('{}'.format(sleeptime), end=" ", flush=True) + print(sleeptime, end=" ", flush=True) time.sleep(1) sleeptime += 1 @@ -43,9 +43,9 @@ def wait_for_job_output(jobdir): print('Checking for Balsam output file: {}'.format(output)) while not os.path.isfile(output) and sleeptime < 30: - print('{}'.format(sleeptime), end=" ", flush=True) - time.sleep(2) - sleeptime += 2 + print(sleeptime, end=" ", flush=True) + time.sleep(1) + sleeptime += 1 return output @@ -54,7 +54,9 @@ def print_job_output(outscript): sleeptime = 0 print('Output file found. Waiting for complete Balsam Job Output.') - lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n'] + lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n', + 'Run completed.\n'] + lastposition = 0 while sleeptime < 60: @@ -66,7 +68,7 @@ def print_job_output(outscript): if len(new) > 0: print(new, flush=True) else: - print('{}'.format(sleeptime), end=" ", flush=True) + print(sleeptime, end=" ", flush=True) if any(new.endswith(line) for line in lastlines): break diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index e86ff3b82..16772441b 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -3,7 +3,6 @@ import os import numpy as np -import multiprocessing import mpi4py from mpi4py import MPI @@ -57,7 +56,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 15} +exit_criteria = {'elapsed_wallclock_time': 30} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, From 3626625fc2a4e2b378362328f6e0970c4437bdb6 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 11 Sep 2019 08:22:10 -0500 Subject: [PATCH 232/644] Asserting that the allocation function must declare fields when giving rows to be sent --- libensemble/libE_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 0ff9bdba8..2b2aed25f 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -203,6 +203,9 @@ def _check_work_order(self, Work, w): work_rows = Work['libE_info']['H_rows'] if len(work_rows): work_fields = set(Work['H_fields']) + assert len(work_fields), \ + "Allocation function requested rows={} be sent to worker={}, "\ + "but requested no fields to be sent.".format(work_rows, w) hist_fields = self.hist.H.dtype.names diff_fields = list(work_fields.difference(hist_fields)) assert not diff_fields, \ From d13dc20498571043b99c319d57da1b8fdb851f73 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 11 Sep 2019 10:03:52 -0500 Subject: [PATCH 233/644] Adding tests for more of persistent_aposmm --- libensemble/gen_funcs/persistent_aposmm.py | 37 +++++--- ... test_6-hump_camel_persistent_aposmm_1.py} | 4 +- .../test_6-hump_camel_persistent_aposmm_2.py | 83 ++++++++++++++++++ .../test_6-hump_camel_persistent_aposmm_3.py | 84 +++++++++++++++++++ .../test_6-hump_camel_persistent_aposmm_4.py | 84 +++++++++++++++++++ 5 files changed, 278 insertions(+), 14 deletions(-) rename libensemble/tests/regression_tests/{test_6-hump_camel_persistent_aposmm.py => test_6-hump_camel_persistent_aposmm_1.py} (98%) create mode 100644 libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py create mode 100644 libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py create mode 100644 libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 60a0925bc..dc22ae119 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -200,7 +200,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['x_on_cube', 'f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'pounders', 'scipy_COBYLA']: + 'LN_NELDERMEAD', 'pounders', 'scipy_Nelder-Mead']: fields_to_pass = ['x_on_cube', 'f'] else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) @@ -305,7 +305,11 @@ def __init__(self, gen_specs, x0, f0, grad0=None): self.x0 = x0.copy() self.f0 = f0.copy() - self.grad0 = grad0.copy() + if grad0 is not None: + self.grad0 = grad0.copy() + else: + self.grad0 = None + # {{{ setting the local optimization method @@ -313,7 +317,7 @@ def __init__(self, gen_specs, x0, f0, grad0=None): run_local_opt = run_local_nlopt elif gen_specs['localopt_method'] in ['pounders', 'blmvm']: run_local_opt = run_local_tao - elif gen_specs['localopt_method'] in ['scipy_COBYLA']: + elif gen_specs['localopt_method'] in ['scipy_Nelder-Mead']: run_local_opt = run_local_scipy_opt else: raise NotImplementedError("Unknown local optimization method " @@ -361,6 +365,7 @@ def iterate(self, data): return x_new def destroy(self, previous_x): + while not isinstance(previous_x, ConvergedMsg): self.parent_can_read.clear() if self.grad0 is None: @@ -455,7 +460,9 @@ def scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, gen_specs # print('[Child]: I have started waiting', flush=True) child_can_read.wait() # print('[Child]: Wohooo.. I am free folks', flush=True) - f_x_recv, = comm_queue.get() + x_recv, f_x_recv, = comm_queue.get() + + assert np.array_equal(x, x_recv), "The point I gave is not the point I got back!" child_can_read.clear() return f_x_recv @@ -476,19 +483,19 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_ca # print('[Child]: Started my optimization', flush=True) res = sp_opt.minimize(lambda x: scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, gen_specs), x0, - method=method, options={'maxiter': 100, 'tol': gen_specs['tol']}) + method=method, options={'maxiter': 10, 'fatol': gen_specs['fatol'], 'xatol': gen_specs['xatol']}) if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum exit_code = 0 else: - if method == 'COBYLA': - assert res['status'] == 1, "Unknown status for COBYLA" + if method == 'Nelder-Mead': + assert res['status'] == 0, "Unknown status for Nelder-Mead" exit_code = 1 x_opt = res['x'] # FIXME: Need to do something with the exit codes. - print(exit_code) + # print(exit_code) # print('[Child]: I have converged.', flush=True) comm_queue.put(ConvergedMsg(x_opt)) @@ -507,9 +514,12 @@ def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen # print('[Child]: I have started waiting', flush=True) child_can_read.wait() # print('[Child]: Wohooo.. I am free folks', flush=True) - f_recv, = comm_queue.get() - child_can_read.clear() + x_recv, f_recv, = comm_queue.get() + + assert np.array_equal(x.array_r, x_recv), "The point I gave is not the point I got back!" + f.array[:] = f_recv + child_can_read.clear() return f @@ -522,7 +532,10 @@ def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read # print('[Child]: I have started waiting', flush=True) child_can_read.wait() # print('[Child]: Wohooo.. I am free folks', flush=True) - f_recv, grad_recv = comm_queue.get() + x_recv, f_recv, grad_recv = comm_queue.get() + + assert np.array_equal(x.array_r, x_recv), "The point I gave is not the point I got back!" + g.array[:] = grad_recv child_can_read.clear() return f_recv @@ -585,7 +598,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read exit_code = tao.getConvergedReason() # FIXME: Need to do something with the exit codes. - print(exit_code) + # print(exit_code) # print(tao.view()) # print(x_opt) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py similarity index 98% rename from libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py rename to libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index ebfd920d8..aacef9de1 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -11,8 +11,8 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi local tcp -# TESTSUITE_NPROCS: 3 4 +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 4 import sys import numpy as np diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py new file mode 100644 index 000000000..36d840fd2 --- /dev/null +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -0,0 +1,83 @@ +# """ +# Runs libEnsemble on the 6-hump camel problem. Documented here: +# https://www.sfu.ca/~ssurjano/camel6.html +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_6-hump_camel_persistent_uniform_sampling.py +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms local +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi +# TESTSUITE_NPROCS: 3 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from math import gamma, pi, sqrt +from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f +from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f +from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima +from time import time + +nworkers, is_master, libE_specs, _ = parse_args() + +if is_master: + start_time = time() + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('grad', float, n)]} + +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), + ('local_min', bool), ('local_pt', bool)] + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': gen_out, + 'batch_mode': True, + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'LN_BOBYQA', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'xtol_rel': 1e-6, + 'num_active_gens': 1, + 'local_min': True, + 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'sim_max': 1000} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_master: + print('[Manager]:', H[np.where(H['local_min'])]['x']) + print('[Manager]: Time taken =', time() - start_time, flush=True) + + tol = 1e-5 + for m in minima: + # The minima are known on this test problem. + # We use their values to test APOSMM has identified all minima + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) + assert np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) < tol + + save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py new file mode 100644 index 000000000..925b327b7 --- /dev/null +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -0,0 +1,84 @@ +# """ +# Runs libEnsemble on the 6-hump camel problem. Documented here: +# https://www.sfu.ca/~ssurjano/camel6.html +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_6-hump_camel_persistent_uniform_sampling.py +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms local +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 4 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from math import gamma, pi, sqrt +from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f +from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f +from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima +from time import time + +nworkers, is_master, libE_specs, _ = parse_args() + +if is_master: + start_time = time() + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('grad', float, n)]} + +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), + ('local_min', bool), ('local_pt', bool)] + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': gen_out, + 'batch_mode': True, + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'blmvm', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'grtol': 1e-4, + 'gatol': 1e-4, + 'num_active_gens': 1, + 'local_min': True, + 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'sim_max': 1000} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_master: + print('[Manager]:', H[np.where(H['local_min'])]['x']) + print('[Manager]: Time taken =', time() - start_time, flush=True) + + tol = 1e-5 + for m in minima: + # The minima are known on this test problem. + # We use their values to test APOSMM has identified all minima + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) + assert np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) < tol + + save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py new file mode 100644 index 000000000..b11bcdaff --- /dev/null +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -0,0 +1,84 @@ +# """ +# Runs libEnsemble on the 6-hump camel problem. Documented here: +# https://www.sfu.ca/~ssurjano/camel6.html +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_6-hump_camel_persistent_uniform_sampling.py +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms local +# python3 test_6-hump_camel_persistent_uniform_sampling.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 4 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from math import gamma, pi, sqrt +from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f +from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f +from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima +from time import time + +nworkers, is_master, libE_specs, _ = parse_args() + +if is_master: + start_time = time() + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('grad', float, n)]} + +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), + ('local_min', bool), ('local_pt', bool)] + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': gen_out, + 'batch_mode': True, + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'scipy_Nelder-Mead', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'fatol': 1e-5, + 'xatol': 1e-5, + 'num_active_gens': 1, + 'local_min': True, + 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'sim_max': 1000} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_master: + print('[Manager]:', H[np.where(H['local_min'])]['x']) + print('[Manager]: Time taken =', time() - start_time, flush=True) + + tol = 1e-3 + for m in minima: + # The minima are known on this test problem. + # We use their values to test APOSMM has identified all minima + print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) + assert np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) < tol + + save_libE_output(H, persis_info, __file__, nworkers) From 24598f0bf15f07ae0fc029c8a569550dec526410 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 11 Sep 2019 10:44:47 -0500 Subject: [PATCH 234/644] flake8 --- libensemble/gen_funcs/persistent_aposmm.py | 17 ++++++++--------- .../test_6-hump_camel_persistent_aposmm_1.py | 2 +- .../test_6-hump_camel_persistent_aposmm_2.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../test_6-hump_camel_persistent_aposmm_4.py | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index dc22ae119..d7843d3f9 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -310,7 +310,6 @@ def __init__(self, gen_specs, x0, f0, grad0=None): else: self.grad0 = None - # {{{ setting the local optimization method if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: @@ -485,12 +484,12 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_ca child_can_read, parent_can_read, gen_specs), x0, method=method, options={'maxiter': 10, 'fatol': gen_specs['fatol'], 'xatol': gen_specs['xatol']}) - if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum - exit_code = 0 - else: - if method == 'Nelder-Mead': - assert res['status'] == 0, "Unknown status for Nelder-Mead" - exit_code = 1 + # if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum + # exit_code = 0 + # else: + # if method == 'Nelder-Mead': + # assert res['status'] == 0, "Unknown status for Nelder-Mead" + # exit_code = 1 x_opt = res['x'] @@ -535,7 +534,7 @@ def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read x_recv, f_recv, grad_recv = comm_queue.get() assert np.array_equal(x.array_r, x_recv), "The point I gave is not the point I got back!" - + g.array[:] = grad_recv child_can_read.clear() return f_recv @@ -595,7 +594,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read tao.solve(x) x_opt = tao.getSolution().getArray() - exit_code = tao.getConvergedReason() + # exit_code = tao.getConvergedReason() # FIXME: Need to do something with the exit codes. # print(exit_code) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index aacef9de1..d28c1ffd6 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index 36d840fd2..2206c0ffa 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -11,8 +11,8 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi -# TESTSUITE_NPROCS: 3 +# TESTSUITE_COMMS: mpi +# TESTSUITE_NPROCS: 3 import sys import numpy as np diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 925b327b7..52803938c 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index b11bcdaff..504ca9582 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys From a1171030b8506560aa8758d3616b395f730dce51 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 10:59:24 -0500 Subject: [PATCH 235/644] Modify Balsam to print hosttype --- conda/test_balsam_hworld.py | 21 +++++++++++++++++- libensemble/tests/regression_tests/common.py | 23 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index c5cd6fb7c..cef3f1ed1 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -2,7 +2,7 @@ import os import time import libensemble -from libensemble.tests.regression_tests.common import modify_Balsam_worker +from libensemble.tests.regression_tests.common import modify_Balsam_worker, modify_Balsam_hostprint # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 @@ -98,6 +98,7 @@ def move_job_coverage(jobdir): basedb = os.environ['HOME'] + '/test-balsam/data/libe_test-balsam' modify_Balsam_worker() + modify_Balsam_hostprint() run_Balsam_job() jobdir = wait_for_job_dir(basedb) @@ -106,3 +107,21 @@ def move_job_coverage(jobdir): move_job_coverage(jobdir) print('Test complete.') + +# travis_run_before_install && travis_run_install && travis_run_before_script + +# 11-Sep-2019 14:36:27|7301| ERROR|balsam:47] Uncaught Exception : Cooley WorkerGroup needs workers_file to setup +# Traceback (most recent call last): +# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 443, in +# main(args) +# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 422, in main +# launcher = Launcher(wf_filter, timelimit_min, gpus_per_node) +# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 104, in __init__ +# self.worker_group = worker.WorkerGroup() +# File "/home/travis/build/Libensemble/balsam/balsam/launcher/worker.py", line 50, in __init__ +# self.setup() +# File "/home/travis/build/Libensemble/balsam/balsam/launcher/worker.py", line 112, in setup_COOLEY +# raise ValueError("Cooley WorkerGroup needs workers_file to setup") +# ValueError: Cooley WorkerGroup needs workers_file to setup + +# print('HOST TYPE:', self.host_type) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 65f8ff953..d8094a5fc 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -225,3 +225,26 @@ def modify_Balsam_pyCoverage(): with open(balsam_commands_path, 'w') as f: for line in lines: f.write(line) + +def modify_Balsam_hostprint(): + # Also modify Balsam to print Host type (for debugging purposes) + import balsam + + print_line = " print('HOST TYPE: ', self.host_type)\n" + + workerfile = 'worker.py' + balsam_path = os.path.dirname(balsam.__file__) + '/launcher' + balsam_worker_path = os.path.join(balsam_path, workerfile) + + with open(balsam_worker_path, 'r') as f: + lines = f.readlines() + + newlines = [] + for line in lines: + newlines.append(line) + if line == " self.host_type = JobEnv.host_type\n": + newlines.append(print_line) + + with open(balsam_worker_path, 'w') as f: + for line in newlines: + f.write(line) From a34f86da8a4788865089dde6c39621fc3d730f7b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 11:07:43 -0500 Subject: [PATCH 236/644] flake8 --- libensemble/tests/regression_tests/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index d8094a5fc..2f64cf673 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -226,6 +226,7 @@ def modify_Balsam_pyCoverage(): for line in lines: f.write(line) + def modify_Balsam_hostprint(): # Also modify Balsam to print Host type (for debugging purposes) import balsam From 32a980ba4392a9744f7439a17aa7b08ba69cb86f Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 11:29:16 -0500 Subject: [PATCH 237/644] don't append print line on subsequent runs --- libensemble/tests/regression_tests/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 2f64cf673..f804592d5 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -242,10 +242,13 @@ def modify_Balsam_hostprint(): newlines = [] for line in lines: + if line == print_line: + continue newlines.append(line) if line == " self.host_type = JobEnv.host_type\n": newlines.append(print_line) + with open(balsam_worker_path, 'w') as f: for line in newlines: f.write(line) From 975b119a16ba877c0f937e711c634d265cea9d8b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 11:41:25 -0500 Subject: [PATCH 238/644] flake8... --- libensemble/tests/regression_tests/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index f804592d5..d5e03d306 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -248,7 +248,6 @@ def modify_Balsam_hostprint(): if line == " self.host_type = JobEnv.host_type\n": newlines.append(print_line) - with open(balsam_worker_path, 'w') as f: for line in newlines: f.write(line) From 1349275d8e1c62b20987311032c88f32ca4f275b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 14:12:42 -0500 Subject: [PATCH 239/644] test commit --- conda/test_balsam_hworld.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index cef3f1ed1..5270763e5 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -123,5 +123,3 @@ def move_job_coverage(jobdir): # File "/home/travis/build/Libensemble/balsam/balsam/launcher/worker.py", line 112, in setup_COOLEY # raise ValueError("Cooley WorkerGroup needs workers_file to setup") # ValueError: Cooley WorkerGroup needs workers_file to setup - -# print('HOST TYPE:', self.host_type) From d722c7c96081a293e8e716857902c92b258f8dc0 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 11 Sep 2019 14:45:35 -0500 Subject: [PATCH 240/644] Making mpi to get more code coverage --- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_4.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index d28c1ffd6..6026f75c9 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: mpi # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 52803938c..fd82b0984 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: mpi # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 504ca9582..2d52fffd5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 4 import sys From 4cc561012d915bfce769468dbe0d76f925d5604d Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 11 Sep 2019 15:07:56 -0500 Subject: [PATCH 241/644] additional modifications to Balsam for debugging --- conda/test_balsam_hworld.py | 3 ++- libensemble/tests/regression_tests/common.py | 26 +++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index 5270763e5..bb728b3d7 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -108,7 +108,8 @@ def move_job_coverage(jobdir): print('Test complete.') -# travis_run_before_install && travis_run_install && travis_run_before_script + +# IN BALSAM LOG: # 11-Sep-2019 14:36:27|7301| ERROR|balsam:47] Uncaught Exception : Cooley WorkerGroup needs workers_file to setup # Traceback (most recent call last): diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index d5e03d306..82b2bc49e 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -228,10 +228,20 @@ def modify_Balsam_pyCoverage(): def modify_Balsam_hostprint(): - # Also modify Balsam to print Host type (for debugging purposes) + # Also modify Balsam Worker & Worker Gropu to print Host type (for debugging + # purposes). Balsam test bug may be caused by setup_COOLEY() being called + # instead of setup_DEFAULT() within Balsam's worker.py import balsam - print_line = " print('HOST TYPE: ', self.host_type)\n" + print_lines = {"host": " print('HOST TYPE: ', self.host_type)\n", + "COOLEY": " print('IN setup_COOLEY')\n", + "DEFAULT": " print('IN setup_DEFAULT')\n"} + + host_prior_lines = [" self.host_type = JobEnv.host_type\n", + " self.host_type = host_type\n"] + + setup_prior_lines = [" def setup_COOLEY(self):\n", + " def setup_DEFAULT(self):\n"] workerfile = 'worker.py' balsam_path = os.path.dirname(balsam.__file__) + '/launcher' @@ -242,11 +252,15 @@ def modify_Balsam_hostprint(): newlines = [] for line in lines: - if line == print_line: + if line in print_lines.values(): continue - newlines.append(line) - if line == " self.host_type = JobEnv.host_type\n": - newlines.append(print_line) + newlines.append(line) # Line of code from prior + if line in host_prior_lines: # + newlines.append(print_lines['host']) + elif line == setup_prior_lines[0]: + newlines.append(print_lines['COOLEY']) + elif line == setup_prior_lines[1]: + newlines.append(print_lines['DEFAULT']) with open(balsam_worker_path, 'w') as f: for line in newlines: From fd0458a7c6ff999664b8e4dd358ba14eee10f2c2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 11 Sep 2019 16:21:43 -0500 Subject: [PATCH 242/644] Showing Steve something unexpected --- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_2.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_4.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 6026f75c9..d28c1ffd6 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index 2206c0ffa..f94884265 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index fd82b0984..52803938c 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 2d52fffd5..504ca9582 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -11,7 +11,7 @@ # """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: mpi local +# TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 4 import sys From c5126a6e67143fb25c3969675dd2f10f9d591d5d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 06:44:36 -0500 Subject: [PATCH 243/644] Including multiprocessing in code coverage --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 700fe8de9..e55b3a64e 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -316,7 +316,7 @@ COV_LINE_PARALLEL='' if [ $RUN_COV_TESTS = "true" ]; then COV_LINE_SERIAL='--cov --cov-report html:cov_unit' #COV_LINE_PARALLEL='-m coverage run --parallel-mode --rcfile=../.coveragerc' #running in sub-dirs - COV_LINE_PARALLEL='-m coverage run --parallel-mode' #running in regression dir itself + COV_LINE_PARALLEL='-m coverage run --parallel-mode --concurrency=multiprocessing' #running in regression dir itself #include branch coverage? eg. flags if never jumped a statement block... [see .coveragerc file] #COV_LINE_PARALLEL='-m coverage run --branch --parallel-mode' From 1dd35fb63614f07c91b49794578643d46696e0b4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 06:44:36 -0500 Subject: [PATCH 244/644] Including multiprocessing in code coverage (cherry picked from commit c5126a6e67143fb25c3969675dd2f10f9d591d5d) --- libensemble/tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 700fe8de9..e55b3a64e 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -316,7 +316,7 @@ COV_LINE_PARALLEL='' if [ $RUN_COV_TESTS = "true" ]; then COV_LINE_SERIAL='--cov --cov-report html:cov_unit' #COV_LINE_PARALLEL='-m coverage run --parallel-mode --rcfile=../.coveragerc' #running in sub-dirs - COV_LINE_PARALLEL='-m coverage run --parallel-mode' #running in regression dir itself + COV_LINE_PARALLEL='-m coverage run --parallel-mode --concurrency=multiprocessing' #running in regression dir itself #include branch coverage? eg. flags if never jumped a statement block... [see .coveragerc file] #COV_LINE_PARALLEL='-m coverage run --branch --parallel-mode' From b6101f26a82a197dbdf986ea76e9da3ecfd74a86 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 10:01:05 -0500 Subject: [PATCH 245/644] Adding support for pounders --- libensemble/gen_funcs/persistent_aposmm.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index d7843d3f9..2cd982c3c 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -200,8 +200,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['x_on_cube', 'f', 'grad'] elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'pounders', 'scipy_Nelder-Mead']: + 'LN_NELDERMEAD', 'scipy_Nelder-Mead']: fields_to_pass = ['x_on_cube', 'f'] + elif gen_specs['localopt_method'] in ['pounders']: + fields_to_pass = ['x_on_cube', 'fvec'] else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) @@ -252,7 +254,8 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Initialize a local opt run local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], - local_H[ind]['f'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) + local_H[ind]['f'] if 'f' in fields_to_pass else local_H[ind]['fvec'], + local_H[ind]['grad'] if 'grad' in fields_to_pass else None) local_opters[total_runs] = local_opter @@ -318,9 +321,6 @@ def __init__(self, gen_specs, x0, f0, grad0=None): run_local_opt = run_local_tao elif gen_specs['localopt_method'] in ['scipy_Nelder-Mead']: run_local_opt = run_local_scipy_opt - else: - raise NotImplementedError("Unknown local optimization method " - "'{}'.".format(gen_specs['localopt_method'])) # }}} @@ -346,6 +346,8 @@ def iterate(self, data): if 'grad' in data.dtype.names: self.comm_queue.put((data['x_on_cube'], data['f'], data['grad'])) + elif 'fvec' in data.dtype.names: + self.comm_queue.put((data['x_on_cube'], data['fvec'])) else: self.comm_queue.put((data['x_on_cube'], data['f'], )) @@ -925,7 +927,7 @@ def initialize_APOSMM(H, gen_specs, libE_info): n_s = 0 - if 'single_component_at_a_time' in gen_specs and gen_specs['single_component_at_a_time']: + if gen_specs.get('single_component_at_a_time'): assert gen_specs['batch_mode'], ("Must be in batch mode when using " "'single_component_at_a_time'") c_flag = True @@ -975,6 +977,9 @@ def initialize_APOSMM(H, gen_specs, libE_info): ('pt_id', int), # Identify the same point evaluated by different sim_f's or components ] + if 'components' in gen_specs: + local_H_fields += [('fvec', float, gen_specs['components'])] + local_H = np.empty(0, dtype=local_H_fields) return n, n_s, c_flag, rk_c, ld, mu, nu, comm, local_H From 57bdfc79179255ea9160258d137cf06fd749e353 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 10:02:41 -0500 Subject: [PATCH 246/644] Adding test for pounders in persistent_aposmm --- .../test_chwirut_pounders_persistent.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py new file mode 100644 index 000000000..2b89a50e9 --- /dev/null +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -0,0 +1,80 @@ +# """ +# Runs libEnsemble with APOSMM+POUNDERS on the chwirut least squares problem. +# All 214 residual calculations for a given point are performed as a single +# simulation evaluation. +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_chwirut_pounders.py +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: local +# TESTSUITE_NPROCS: 4 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from math import gamma, pi, sqrt +from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f +from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f +from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream + +nworkers, is_master, libE_specs, _ = parse_args() + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +# Declare the run parameters/functions +m = 214 +n = 3 +budget = 10 + +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('fvec', float, m)], + 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + +gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), + ('local_min', bool), ('local_pt', bool)] + +# lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': gen_out, + 'batch_mode': True, + 'initial_sample_size': 100, + 'localopt_method': 'pounders', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'grtol': 1e-5, + 'gatol': 1e-5, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.5, + 'components': m, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n)} + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} + +persis_info = per_worker_stream({}, nworkers + 1) + +exit_criteria = {'sim_max': 500} + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_master: + assert flag == 0 + assert len(H) >= budget + + save_libE_output(H, persis_info, __file__, nworkers) + # Calculating the Jacobian at the best point (though this information was not used by pounders) + from libensemble.sim_funcs.chwirut1 import EvaluateJacobian + J = EvaluateJacobian(H['x'][np.argmin(H['f'])]) + assert np.linalg.norm(J) < 2000 + From ff0b4cded6fef92444fb0e4f949de06538d45956 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 10:03:25 -0500 Subject: [PATCH 247/644] Varying tests slightly for coverage --- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm_2.py | 4 ++-- .../regression_tests/test_6-hump_camel_persistent_aposmm_3.py | 1 - .../regression_tests/test_6-hump_camel_persistent_aposmm_4.py | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index d28c1ffd6..94a145662 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -52,9 +52,8 @@ 'localopt_method': 'LD_MMA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'xtol_rel': 1e-6, + 'ftol_rel': 1e-6, 'num_active_gens': 1, - 'local_min': True, - 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index f94884265..abe9d1dca 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -51,9 +51,9 @@ 'sample_points': np.round(minima, 1), 'localopt_method': 'LN_BOBYQA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'xtol_rel': 1e-6, + 'xtol_abs': 1e-6, + 'ftol_abs': 1e-6, 'num_active_gens': 1, - 'local_min': True, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, 'lb': np.array([-3, -2]), diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 52803938c..a8f566eca 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -54,7 +54,6 @@ 'grtol': 1e-4, 'gatol': 1e-4, 'num_active_gens': 1, - 'local_min': True, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, 'lb': np.array([-3, -2]), diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 504ca9582..e054031a5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -54,7 +54,6 @@ 'fatol': 1e-5, 'xatol': 1e-5, 'num_active_gens': 1, - 'local_min': True, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, 'lb': np.array([-3, -2]), From 08a1257255033a66f1cdd8536d960006be96474d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 10:13:59 -0500 Subject: [PATCH 248/644] phlayke-ate --- libensemble/gen_funcs/persistent_aposmm.py | 6 +++--- .../regression_tests/test_chwirut_pounders_persistent.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 2cd982c3c..7d9221484 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -202,7 +202,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'scipy_Nelder-Mead']: fields_to_pass = ['x_on_cube', 'f'] - elif gen_specs['localopt_method'] in ['pounders']: + elif gen_specs['localopt_method'] in ['pounders']: fields_to_pass = ['x_on_cube', 'fvec'] else: raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) @@ -254,7 +254,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Initialize a local opt run local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], - local_H[ind]['f'] if 'f' in fields_to_pass else local_H[ind]['fvec'], + local_H[ind]['f'] if 'f' in fields_to_pass else local_H[ind]['fvec'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) local_opters[total_runs] = local_opter @@ -978,7 +978,7 @@ def initialize_APOSMM(H, gen_specs, libE_info): ] if 'components' in gen_specs: - local_H_fields += [('fvec', float, gen_specs['components'])] + local_H_fields += [('fvec', float, gen_specs['components'])] local_H = np.empty(0, dtype=local_H_fields) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 2b89a50e9..39416371a 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -77,4 +77,3 @@ from libensemble.sim_funcs.chwirut1 import EvaluateJacobian J = EvaluateJacobian(H['x'][np.argmin(H['f'])]) assert np.linalg.norm(J) < 2000 - From 6fb4b6d686544cd32b510a2c155014cdc982490a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 12 Sep 2019 11:42:36 -0500 Subject: [PATCH 249/644] Supporting older petsc4py call --- libensemble/gen_funcs/persistent_aposmm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 7d9221484..407694fa1 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -570,7 +570,10 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read f.setSizes(m) f.setFromOptions() - tao.setResidual(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) + if hasattr(tao, 'setResidual'): + tao.setResidual(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) + else: + tao.setSeparableObjective(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) elif gen_specs['localopt_method'] == 'blmvm': g = PETSc.Vec().create(tao_comm) From 688e3e8082dcf547d7a68672eafb55cfc6ad6e2d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 13 Sep 2019 08:10:38 -0500 Subject: [PATCH 250/644] Tightening convergence --- .../regression_tests/test_chwirut_pounders_persistent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 39416371a..5ec11264a 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -50,8 +50,8 @@ 'initial_sample_size': 100, 'localopt_method': 'pounders', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'grtol': 1e-5, - 'gatol': 1e-5, + 'grtol': 1e-6, + 'gatol': 1e-6, 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'components': m, From 1a139d70577f388bdf3b3ba2164c1bcdf2f56213 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2019 14:59:39 -0500 Subject: [PATCH 251/644] Adds ulimit adjustment to travis (cherry picked from commit d935007644277c47c7c6acbe6675894f30dc5917) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b1cee2631..4cc24d54c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,8 @@ before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - source setbalsampath.sh + # Allow 10000 files to be open at once (critical for persistent_aposmm) + - ulimit -Sn 10000 # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; From 31f42d1df9f7b36dda7879b9405bee87b13dddbd Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 13 Sep 2019 10:28:55 -0500 Subject: [PATCH 252/644] Pounders may not find points with small Jacobians --- .../regression_tests/test_chwirut_pounders_persistent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 5ec11264a..4bb3bfea6 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -73,7 +73,7 @@ assert len(H) >= budget save_libE_output(H, persis_info, __file__, nworkers) - # Calculating the Jacobian at the best point (though this information was not used by pounders) - from libensemble.sim_funcs.chwirut1 import EvaluateJacobian - J = EvaluateJacobian(H['x'][np.argmin(H['f'])]) - assert np.linalg.norm(J) < 2000 + # # Calculating the Jacobian at the best point (though this information was not used by pounders) + # from libensemble.sim_funcs.chwirut1 import EvaluateJacobian + # J = EvaluateJacobian(H['x'][np.argmin(H['f'])]) + # assert np.linalg.norm(J) < 2000 From 063e33db9b5e29de2a050786ea4c81459f7ad037 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 11:35:09 -0500 Subject: [PATCH 253/644] modifies Balsam to not think self on Cooley if 'cc' in hostname --- conda/test_balsam_hworld.py | 21 +-------- libensemble/tests/regression_tests/common.py | 46 +++++++------------- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/conda/test_balsam_hworld.py b/conda/test_balsam_hworld.py index bb728b3d7..0c837d8f0 100644 --- a/conda/test_balsam_hworld.py +++ b/conda/test_balsam_hworld.py @@ -2,7 +2,7 @@ import os import time import libensemble -from libensemble.tests.regression_tests.common import modify_Balsam_worker, modify_Balsam_hostprint +from libensemble.tests.regression_tests.common import modify_Balsam_worker, modify_Balsam_JobEnv # TESTSUITE_COMMS: local # TESTSUITE_NPROCS: 3 @@ -98,7 +98,7 @@ def move_job_coverage(jobdir): basedb = os.environ['HOME'] + '/test-balsam/data/libe_test-balsam' modify_Balsam_worker() - modify_Balsam_hostprint() + modify_Balsam_JobEnv() run_Balsam_job() jobdir = wait_for_job_dir(basedb) @@ -107,20 +107,3 @@ def move_job_coverage(jobdir): move_job_coverage(jobdir) print('Test complete.') - - -# IN BALSAM LOG: - -# 11-Sep-2019 14:36:27|7301| ERROR|balsam:47] Uncaught Exception : Cooley WorkerGroup needs workers_file to setup -# Traceback (most recent call last): -# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 443, in -# main(args) -# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 422, in main -# launcher = Launcher(wf_filter, timelimit_min, gpus_per_node) -# File "/home/travis/build/Libensemble/balsam/balsam/launcher/launcher.py", line 104, in __init__ -# self.worker_group = worker.WorkerGroup() -# File "/home/travis/build/Libensemble/balsam/balsam/launcher/worker.py", line 50, in __init__ -# self.setup() -# File "/home/travis/build/Libensemble/balsam/balsam/launcher/worker.py", line 112, in setup_COOLEY -# raise ValueError("Cooley WorkerGroup needs workers_file to setup") -# ValueError: Cooley WorkerGroup needs workers_file to setup diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 82b2bc49e..aaa2f9784 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -227,41 +227,27 @@ def modify_Balsam_pyCoverage(): f.write(line) -def modify_Balsam_hostprint(): - # Also modify Balsam Worker & Worker Gropu to print Host type (for debugging - # purposes). Balsam test bug may be caused by setup_COOLEY() being called - # instead of setup_DEFAULT() within Balsam's worker.py +def modify_Balsam_JobEnv(): + # If Balsam detects that the system on which it is running contains the string + # 'cc' in it's hostname, then it thinks it's on Cooley! Travis hostnames are + # randomly generated and occasionally may contain that offending string. This + # modifies Balsam's JobEnvironment class to not check for 'cc'. import balsam - print_lines = {"host": " print('HOST TYPE: ', self.host_type)\n", - "COOLEY": " print('IN setup_COOLEY')\n", - "DEFAULT": " print('IN setup_DEFAULT')\n"} + bad_line = " 'COOLEY' : 'cooley cc'.split()\n" + new_line = " 'COOLEY' : 'cooley'.split()\n" - host_prior_lines = [" self.host_type = JobEnv.host_type\n", - " self.host_type = host_type\n"] + jobenv_file = 'JobEnvironment.py' + balsam_path = os.path.dirname(balsam.__file__) + '/service/schedulers' + balsam_jobenv_path = os.path.join(balsam_path, jobenv_file) - setup_prior_lines = [" def setup_COOLEY(self):\n", - " def setup_DEFAULT(self):\n"] - - workerfile = 'worker.py' - balsam_path = os.path.dirname(balsam.__file__) + '/launcher' - balsam_worker_path = os.path.join(balsam_path, workerfile) - - with open(balsam_worker_path, 'r') as f: + with open(balsam_jobenv_path, 'r') as f: lines = f.readlines() - newlines = [] - for line in lines: - if line in print_lines.values(): - continue - newlines.append(line) # Line of code from prior - if line in host_prior_lines: # - newlines.append(print_lines['host']) - elif line == setup_prior_lines[0]: - newlines.append(print_lines['COOLEY']) - elif line == setup_prior_lines[1]: - newlines.append(print_lines['DEFAULT']) + for i in range(len(lines)): + if lines[i] == bad_line: + lines[i] = new_line - with open(balsam_worker_path, 'w') as f: - for line in newlines: + with open(balsam_jobenv_path, 'w') as f: + for line in lines: f.write(line) From 85b9d564b451f03cd69a9960fb9a88997ff1fbe2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 13 Sep 2019 11:56:59 -0500 Subject: [PATCH 254/644] More coverage of persistent_aposmm --- libensemble/gen_funcs/persistent_aposmm.py | 43 ++++++++----------- .../test_6-hump_camel_persistent_aposmm_4.py | 5 ++- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 407694fa1..228f97f71 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -7,7 +7,6 @@ __all__ = ['initialize_APOSMM', 'decide_where_to_start_localopt', 'update_history_dist'] import sys -import traceback import numpy as np from scipy.spatial.distance import cdist from scipy import optimize as sp_opt @@ -280,11 +279,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): send_mgr_worker_msg(comm, local_H[new_inds_to_send_mgr + new_opt_inds_to_send_mgr][[i[0] for i in gen_specs['out']]]) - # for local_opter in local_opters: - # if local_opter.is_running: - # raise RuntimeError("[Parent]: Atleast one child process is still active, even after" - # " killing all the children.") - return local_H, persis_info, tag @@ -1012,25 +1006,24 @@ def clean_up_and_stop(local_H, local_opters, run_order): local_H[np.where(local_H['local_min'])]['x'], flush=True) for i, p in local_opters.items(): - if p.is_running: - p.destroy(local_H['x_on_cube'][run_order[i][-1]]) - - -def display_exception(e): - print(e.__doc__) - print(e.args) - _, _, tb = sys.exc_info() - traceback.print_tb(tb) # Fixed format - tb_info = traceback.extract_tb(tb) - filename, line, func, text = tb_info[-1] - print('An error occurred on line {} of function {} with statement {}'.format(line, func, text)) - - # PETSc/TAO errors are printed in the following manner: - if hasattr(e, '_traceback_'): - print('The error was:') - for i in e._traceback_: - print(i) - sys.stdout.flush() + p.destroy(local_H['x_on_cube'][run_order[i][-1]]) + + +# def display_exception(e): +# print(e.__doc__) +# print(e.args) +# _, _, tb = sys.exc_info() +# traceback.print_tb(tb) # Fixed format +# tb_info = traceback.extract_tb(tb) +# filename, line, func, text = tb_info[-1] +# print('An error occurred on line {} of function {} with statement {}'.format(line, func, text)) + +# # PETSc/TAO errors are printed in the following manner: +# if hasattr(e, '_traceback_'): +# print('The error was:') +# for i in e._traceback_: +# print(i) +# sys.stdout.flush() # if __name__ == "__main__": diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index e054031a5..995d266f3 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -19,7 +19,6 @@ # Import libEnsemble items for this test from libensemble.libE import libE -from math import gamma, pi, sqrt from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f @@ -50,9 +49,11 @@ 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), 'localopt_method': 'scipy_Nelder-Mead', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'fatol': 1e-5, 'xatol': 1e-5, + 'nu': 1e-8, + 'mu': 1e-8, + 'lhs_divisions': 5, 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, From d39e2e5bbfb832a9a293673e9b219c11340ce1fa Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 11:58:58 -0500 Subject: [PATCH 255/644] bump up exit criteria --- libensemble/tests/regression_tests/script_test_balsam_hworld.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 16772441b..ee6cc5609 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -56,7 +56,7 @@ def build_simfunc(): persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'elapsed_wallclock_time': 30} +exit_criteria = {'elapsed_wallclock_time': 35} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, From e935b830428c46a0ab21f4f502d23ebf449eda3b Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 12:29:25 -0500 Subject: [PATCH 256/644] tries modifying hostname within travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b1cee2631..be0832a1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,9 @@ matrix: services: - postgresql +addons: + hostname: libensemble-travis + # matrix: # allow_failures: # - env: MPI=openmpi From 34cd68148a078e3d6b24bbb7b5a3d2253bfc4610 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 13 Sep 2019 12:51:09 -0500 Subject: [PATCH 257/644] Adjusting tests --- .../test_6-hump_camel_persistent_aposmm_4.py | 5 ++--- .../regression_tests/test_chwirut_pounders_persistent.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 995d266f3..f6e9bf0b3 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -53,9 +53,8 @@ 'xatol': 1e-5, 'nu': 1e-8, 'mu': 1e-8, - 'lhs_divisions': 5, 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.5, + 'dist_to_bound_multiple': 0.01, 'max_active_runs': 6, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} @@ -64,7 +63,7 @@ persis_info = per_worker_stream({}, nworkers + 1) -exit_criteria = {'sim_max': 1000} +exit_criteria = {'sim_max': 2000} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 4bb3bfea6..ca5ea9b5f 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -54,6 +54,7 @@ 'gatol': 1e-6, 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, + 'lhs_divisions': 5, 'components': m, 'lb': (-2-np.pi/10)*np.ones(n), 'ub': 2*np.ones(n)} From fde0b038bb26bbf3f2acdeed3e024fbcf611f057 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 13:59:09 -0500 Subject: [PATCH 258/644] removing travis hostname. seems to break postgresql --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index be0832a1e..e5e1aaa35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,6 @@ matrix: services: - postgresql -addons: - hostname: libensemble-travis # matrix: # allow_failures: From 8000fb11e765fc4e1961f8df65df0db747c31f97 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 14:04:26 -0500 Subject: [PATCH 259/644] attempting to rearrange hostname --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index e5e1aaa35..ed989c370 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,9 @@ matrix: language: generic python: 3 +addons: + hostname: libensemble-travis + services: - postgresql From 5a14d5629cc00740f747f8e100a4a5848c52aa09 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 13 Sep 2019 14:11:26 -0500 Subject: [PATCH 260/644] actually removes hostname line --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed989c370..23e8340b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,6 @@ matrix: language: generic python: 3 -addons: - hostname: libensemble-travis services: - postgresql From 8c037797c0f4177b47b393bd6003d2461c94f47a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 16 Sep 2019 08:39:39 -0500 Subject: [PATCH 261/644] Removing blank lines --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23e8340b3..b1cee2631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,11 +24,9 @@ matrix: language: generic python: 3 - services: - postgresql - # matrix: # allow_failures: # - env: MPI=openmpi From e8d4470ca1f571851f3b6620a56df41a16ee1af1 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 16 Sep 2019 11:39:35 -0500 Subject: [PATCH 262/644] Removing c_flag or single_component from persistent aposmm. If wanted, this functionality should come from the alloc func, not the generator. --- libensemble/gen_funcs/persistent_aposmm.py | 62 ++++++---------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 228f97f71..9914ce761 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -61,7 +61,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): and optionally - ``'priority' [float]``: Value quantifying a point's desirability - - ``'f_i' [float]``: Value of ith objective component (if single_component) - ``'fvec' [m floats]``: All objective components (if calculated together) - ``'obj_component' [int]``: Index corresponding to value in ``'f_i``' - ``'pt_id' [int]``: Identify the point (useful when evaluating different @@ -105,8 +104,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): points must satisfy - ``'nu' [float]``: Distance from identified minima that all starting points must satisfy - - ``'single_component_at_a_time' [bool]``: True if single objective - components will be evaluated at a time - ``'rk_const' [float]``: Multiplier in front of the r_k value - ``'max_active_runs' [int]``: Bound on number of runs APOSMM is advancing @@ -156,9 +153,6 @@ def aposmm(H, persis_info, gen_specs, libE_info): Description of intermediate variables in aposmm_logic: n: domain dimension - c_flag: True if giving libEnsemble individual components of fvec - to evaluate. (Note if c_flag is True, APOSMM will use - only the component to store the function value f) n_s: the number of complete evaluations of sampled points updated_inds: indices of H that have been updated (and so all their information must be sent back to libE manager to update) @@ -189,7 +183,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - n, n_s, c_flag, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + n, n_s, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) # Initialize stuff for localopt children local_opters = {} @@ -209,7 +203,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): # Send our initial sample. We don't need to check that n_s is large enough: # the alloc_func only returns when the initial sample has function values. persis_info = add_k_sample_points_to_local_H(gen_specs['initial_sample_size'], gen_specs, - persis_info, n, c_flag, comm, local_H, + persis_info, n, comm, local_H, sim_id_to_child_indices) send_mgr_worker_msg(comm, local_H[:gen_specs['initial_sample_size']][[i[0] for i in gen_specs['out']]]) @@ -221,7 +215,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): clean_up_and_stop(local_H, local_opters, run_order) break - n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in) + n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, Work, calc_in) new_opt_inds_to_send_mgr = [] new_inds_to_send_mgr = [] @@ -236,7 +230,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): new_opt_inds_to_send_mgr.append(opt_ind) local_opters.pop(child_idx) else: - add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + add_to_local_H(local_H, x_new, gen_specs, local_flag=1, on_cube=True) new_inds_to_send_mgr.append(len(local_H)-1) run_order[child_idx].append(local_H[-1]['sim_id']) @@ -260,7 +254,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal - add_to_local_H(local_H, x_new, gen_specs, c_flag, local_flag=1, on_cube=True) + add_to_local_H(local_H, x_new, gen_specs, local_flag=1, on_cube=True) new_inds_to_send_mgr.append(len(local_H)-1) run_order[total_runs] = [ind, local_H[-1]['sim_id']] @@ -274,7 +268,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if len(new_inds_to_send_mgr) == 0: persis_info = add_k_sample_points_to_local_H(1, gen_specs, persis_info, n, - c_flag, comm, local_H, sim_id_to_child_indices) + comm, local_H, sim_id_to_child_indices) new_inds_to_send_mgr.append(len(local_H)-1) send_mgr_worker_msg(comm, local_H[new_inds_to_send_mgr + new_opt_inds_to_send_mgr][[i[0] for i in gen_specs['out']]]) @@ -620,7 +614,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read # }}} -def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, calc_in): +def update_local_H_after_receiving(local_H, n, n_s, gen_specs, Work, calc_in): for name in calc_in.dtype.names: local_H[name][Work['libE_info']['H_rows']] = calc_in[name] @@ -629,12 +623,12 @@ def update_local_H_after_receiving(local_H, n, n_s, gen_specs, c_flag, Work, cal n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) # dist -> distance - update_history_dist(local_H, n, gen_specs, c_flag) + update_history_dist(local_H, n, gen_specs) return n_s -def add_to_local_H(local_H, pts, gen_specs, c_flag, local_flag=0, sorted_run_inds=[], run=[], on_cube=True): +def add_to_local_H(local_H, pts, gen_specs, local_flag=0, sorted_run_inds=[], run=[], on_cube=True): """ Adds points to O, the numpy structured array to be sent back to the manager """ @@ -644,12 +638,6 @@ def add_to_local_H(local_H, pts, gen_specs, c_flag, local_flag=0, sorted_run_ind ub = gen_specs['ub'] lb = gen_specs['lb'] - if c_flag: - m = gen_specs['components'] - - assert len_local_H % m == 0, "Number of points in local_H not congruent to 0 mod 'components'" - pt_ids = np.sort(np.tile(np.arange((len_local_H)/m, (len_local_H)/m+len(pts)), (1, m))) - pts = np.tile(pts, (m, 1)) num_pts = len(pts) @@ -671,17 +659,13 @@ def add_to_local_H(local_H, pts, gen_specs, c_flag, local_flag=0, sorted_run_ind local_H['ind_of_better_l'][-num_pts:] = -1 local_H['ind_of_better_s'][-num_pts:] = -1 - if c_flag: - local_H['obj_component'][-num_pts:] = np.tile(range(0, m), (1, num_pts//m)) - local_H['pt_id'][-num_pts:] = pt_ids - if local_flag: local_H['num_active_runs'][-num_pts] += 1 else: local_H['priority'][-num_pts:] = 1 -def update_history_dist(H, n, gen_specs, c_flag): +def update_history_dist(H, n, gen_specs): """ Updates distances/indices after new points that have been evaluated. @@ -691,15 +675,7 @@ def update_history_dist(H, n, gen_specs, c_flag): new_inds = np.where(~H['known_to_aposmm'])[0] - if c_flag: - for v in np.unique(H['pt_id'][new_inds]): - inds = H['pt_id'] == v - H['f'][inds] = np.inf - H['f'][np.where(inds)[0][0]] = gen_specs['combine_component_func'](H['f_i'][inds]) - - p = np.logical_and.reduce((H['returned'], H['obj_component'] == 0, ~np.isnan(H['f']))) - else: - p = np.logical_and.reduce((H['returned'], ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['returned'], ~np.isnan(H['f']))) for new_ind in new_inds: # Loop over new returned points and update their distances @@ -924,13 +900,6 @@ def initialize_APOSMM(H, gen_specs, libE_info): n_s = 0 - if gen_specs.get('single_component_at_a_time'): - assert gen_specs['batch_mode'], ("Must be in batch mode when using " - "'single_component_at_a_time'") - c_flag = True - else: - c_flag = False - if 'rk_const' in gen_specs: rk_c = gen_specs['rk_const'] else: @@ -971,7 +940,6 @@ def initialize_APOSMM(H, gen_specs, libE_info): ('sim_id', int), ('paused', bool), ('returned', bool), - ('pt_id', int), # Identify the same point evaluated by different sim_f's or components ] if 'components' in gen_specs: @@ -979,22 +947,22 @@ def initialize_APOSMM(H, gen_specs, libE_info): local_H = np.empty(0, dtype=local_H_fields) - return n, n_s, c_flag, rk_c, ld, mu, nu, comm, local_H + return n, n_s, rk_c, ld, mu, nu, comm, local_H -def add_k_sample_points_to_local_H(k, gen_specs, persis_info, n, c_flag, comm, local_H, sim_id_to_child_indices): +def add_k_sample_points_to_local_H(k, gen_specs, persis_info, n, comm, local_H, sim_id_to_child_indices): if 'sample_points' in gen_specs: v = np.sum(~local_H['local_pt']) # Number of sample points so far sampled_points = gen_specs['sample_points'][v:v+k] on_cube = False # Assume points are on original domain, not unit cube if len(sampled_points): - add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=on_cube) + add_to_local_H(local_H, sampled_points, gen_specs, on_cube=on_cube) k = k-len(sampled_points) if k > 0: sampled_points = persis_info['rand_stream'].uniform(0, 1, (k, n)) - add_to_local_H(local_H, sampled_points, gen_specs, c_flag, on_cube=True) + add_to_local_H(local_H, sampled_points, gen_specs, on_cube=True) return persis_info From cbdc8c97982a3243f90a0fd3dc531b889d1fdd61 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 23 Sep 2019 16:14:06 -0500 Subject: [PATCH 263/644] Add OpenMP for CPU and GPU/accelerator to the forces test case --- .../tests/scaling_tests/forces/forces.c | 74 ++++++++++++++++--- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/forces.c b/libensemble/tests/scaling_tests/forces/forces.c index 1aebfa44b..7b99b74ca 100755 --- a/libensemble/tests/scaling_tests/forces/forces.c +++ b/libensemble/tests/scaling_tests/forces/forces.c @@ -8,6 +8,9 @@ Each rank computes forces for a subset of particles. Particle force arrays are allreduced across ranks. + Sept 2019: + Added OpenMP options for CPU and GPU. Toggle in forces_naive function. + Run executable on N procs: mpirun -np N ./forces.x @@ -18,8 +21,9 @@ #include #include #include -#include +#include #include +#include #define min(a,b) \ ({ __typeof__ (a) _a = (a); \ @@ -29,14 +33,38 @@ // Flags 0 or 1 #define PRINT_PARTICLE_DECOMP 0 #define PRINT_ALL_PARTICLES 0 +#define CHECK_THREADS 0 static FILE* stat_fp; +// Return elapsed wall clock time from start/end timevals +double elapsed(struct timeval *tv1, struct timeval *tv2) { + return (double)(tv2->tv_usec - tv1->tv_usec) / 1000000 + + (double)(tv2->tv_sec - tv1->tv_sec); +} + +// Print from each thread. +int check_threads(int rank) { + int tid, nthreads; + + #pragma omp parallel private(tid,nthreads) + { + #if defined(_OPENMP) + nthreads = omp_get_num_threads(); + tid = omp_get_thread_num(); + printf("Rank: %d: ThreadID: %d Num threads: %d\n",rank,tid,nthreads); + #else + printf("Rank: %d: OpenMP is disabled\n",rank); + #endif + } + return 0; +} + typedef struct particle { double p[3]; // Particle position double f[3]; // Particle force double q; // Particle charge -} particle; +}__attribute__((__packed__)) particle; // Seed RNG @@ -92,7 +120,18 @@ double forces_naive(int n, int lower, int upper, particle* parr) { double ret = 0.0; double dx, dy, dz, r, force; - for(i=lower; i Date: Mon, 23 Sep 2019 16:19:38 -0500 Subject: [PATCH 264/644] Add mini_forces non-MPI standalone code for experimentation --- .../forces/mini_forces/mini_forces.c | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c diff --git a/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c new file mode 100644 index 000000000..0c9c7729a --- /dev/null +++ b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c @@ -0,0 +1,179 @@ +/* -------------------------------------------------------------------- + Non-MPI, Single Step, Electostatics Code Example + + This is a complete working example to test threading/vectorization + without MPI or other non-trivial features. + + E.g: gcc build and run on 2 threads on CPU: + + gcc -O3 -fopenmp -o serial_forces.x serial_forces.c -lm + export OMP_NUM_THREADS=2 + ./mini_forces.x + + E.g: xlc build and run on GPU: + + # First toggle #omp pragma line to target in forces_naive function. + xlc_r -O3 -qsmp=omp -qoffload -o mini_forces_xlc_gpu.x mini_forces.c + ./mini_forces.x + + Functionality: + Particles position and charge are initiated by a random stream. + Computes forces for all particles. Note: This version uses + parallel arrays to store the data (SoA). + + OpenMP options for CPU and GPU. Toggle in forces_naive function. + + Author: S Hudson. +-------------------------------------------------------------------- */ +#include +#include +#include +#include +#include + +#define CHECK_THREADS 1 +#define NUM_PARTICLES 6000 + +// Seed RNG +int seed_rand(int seed) { + srand(seed); + return 0; +} + +// Return a random number from a persistent stream +double get_rand() { + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + return randnum; +} + + +// Particles start at random locations in 10x10x10 cube +int build_system(int n, double* x, double* y, double* z, double* fx, double* fy, double* fz, double* q) { + int q_range_low = -10; + int q_range_high = 10; + double extent = 10.0; + int i; + + for(i=0; i Date: Mon, 23 Sep 2019 21:08:26 -0500 Subject: [PATCH 265/644] Add build variations to build_forces.sh --- .../scaling_tests/forces/build_forces.sh | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/libensemble/tests/scaling_tests/forces/build_forces.sh b/libensemble/tests/scaling_tests/forces/build_forces.sh index d16443292..564e42c5a 100755 --- a/libensemble/tests/scaling_tests/forces/build_forces.sh +++ b/libensemble/tests/scaling_tests/forces/build_forces.sh @@ -1,10 +1,39 @@ #!/bin/bash +# Building flat MPI + # GCC mpicc -O3 -o forces.x forces.c -lm # Intel -# mpicc -O3 -o forces.x forces.c +# mpiicc -O3 -o forces.x forces.c # Cray # cc -O3 -o forces.x forces.c + +# ---------------------------------------------- + +# Building with OpenMP for CPU + +# GCC +# mpicc -O3 -fopenmp -o forces.x forces.c -lm + +# Intel +# mpiicc -O3 -qopenmp -o forces.x forces.c + +# Cray / Intel (for CCE OpenMP is recognized by default) +# cc -O3 -qopenmp -o forces.x forces.c + +# xl +# xlc_r -O3 -qsmp=omp -o forces.x forces.c + +# ---------------------------------------------- + +# Building with OpenMP for target device (e.g. GPU) +# Need to toggle to OpenMP target directive in forces.c. + +# xl +# xlc_r -O3 -qsmp=omp -qoffload -o forces.x forces.c + +# IRIS node (Intel Gen9 GPU) +# env MPICH_CC=icx mpigcc -g -fiopenmp -fopenmp-targets=spir64 -o forces.x forces.c From b67266d28efa2e9d0ab9554d6193278e48ef9be1 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 10:34:41 -0500 Subject: [PATCH 266/644] Trying updating SDK version --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4cc24d54c..0dd56154b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,8 +61,8 @@ before_install: install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.13.sdk.tar.xz; - mkdir ../sdk; tar xf MacOSX10.13.sdk.tar.xz -C ../sdk; + wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz; + mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; else From 516b6a0df055a0811427cbdacd992971bdc792ad Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 11:10:26 -0500 Subject: [PATCH 267/644] update conda_build_sysroot --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0dd56154b..f47236d21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ before_script: - ulimit -Sn 10000 # Set conda compilers to use new SDK instead of Travis default. - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh; + echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; source setenv.sh; fi From 314137920a5a56745cd23e6c1d7c8e1da078c5f0 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 11:19:35 -0500 Subject: [PATCH 268/644] set conda build sysroot definition earlier --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f47236d21..94bca1b6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,6 +65,8 @@ install: mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; + echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; + source setenv.sh; else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; From 788b5da2bf4fee303345d468cb8544bf6d202b25 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 12:00:13 -0500 Subject: [PATCH 269/644] Attempt to force clang 8.0.1 --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94bca1b6e..27032df52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,13 +65,12 @@ install: mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; - echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; - source setenv.sh; + conda install clang=8.0.1 else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; fi - - conda install $COMPILERS + - conda install --no-update-dependencies $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 From 84fef2995fd127ab18241d56613866df270e6075 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 12:10:20 -0500 Subject: [PATCH 270/644] fix clang 8.0.1 installation --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27032df52..2f9b7af15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,12 +65,12 @@ install: mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; - conda install clang=8.0.1 + conda install clang=8.0.1; else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; fi - - conda install --no-update-dependencies $COMPILERS + - conda install --freeze-installed $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 From 061a543464ea56163089a882846d663f93672c95 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 14:22:55 -0500 Subject: [PATCH 271/644] add brew update/upgrade, try installing sdk header files --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2f9b7af15..10b3b526e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,8 @@ cache: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh -O miniconda.sh; + brew update && brew upgrade; + sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target / else wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; fi From cd35b8df9a4c2ad63396b3e3f08149945381881f Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 14:28:26 -0500 Subject: [PATCH 272/644] semicolon --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 10b3b526e..67c98b44b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh -O miniconda.sh; brew update && brew upgrade; - sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target / + sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; else wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; fi From 59e8bf47092d1a44c52063bf5eef8575aac3a78f Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 14:44:30 -0500 Subject: [PATCH 273/644] update miniconda, use recent clang --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67c98b44b..7773de198 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,8 +45,7 @@ cache: # Setup Miniconda before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh -O miniconda.sh; - brew update && brew upgrade; + wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh -O miniconda.sh; sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; else wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; @@ -67,12 +66,12 @@ install: mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; - conda install clang=8.0.1; + #conda install clang=8.0.1; else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; fi - - conda install --freeze-installed $COMPILERS + - conda install $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 From a79821f601aa106e97470c57c59f337b19c42bc9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 14:47:18 -0500 Subject: [PATCH 274/644] trigger build? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7773de198..bc658a5f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ before_install: - hash -r - conda config --set always_yes yes --set changeps1 no #- conda update -q -y conda - - conda info -a # For debugging any issues with conda + - conda info -a # For debugging any issues with conda - conda config --add channels conda-forge - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv From 291a9fbdb435c76c03346579e57a3d8ec948443f Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 14:52:20 -0500 Subject: [PATCH 275/644] valid yaml --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc658a5f3..5c67d7321 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ before_install: - hash -r - conda config --set always_yes yes --set changeps1 no #- conda update -q -y conda - - conda info -a # For debugging any issues with conda + - conda info -a # For debugging any issues with conda - conda config --add channels conda-forge - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv @@ -66,7 +66,6 @@ install: mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; - #conda install clang=8.0.1; else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; From 3a5b940df494666413432bc5876c636228b26e00 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 15:05:39 -0500 Subject: [PATCH 276/644] try not using sdk --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c67d7321..79412fee7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,14 +62,14 @@ before_install: install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz; - mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; COMPILERS=clang_osx-64; MUMPS=mumps-mpi=5.1.2=haf446c3_1007; else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; fi + # wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz; + # mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; - conda install $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI @@ -93,10 +93,10 @@ before_script: # Allow 10000 files to be open at once (critical for persistent_aposmm) - ulimit -Sn 10000 # Set conda compilers to use new SDK instead of Travis default. - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; - source setenv.sh; - fi + # - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + # echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; + # source setenv.sh; + # fi # Run test (-z show output) script: From e4d545a166ed42038b5899e4cbf94ecf18c45fb3 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 15:17:07 -0500 Subject: [PATCH 277/644] test lower versions of xcode --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 79412fee7..7b6db7353 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,16 @@ matrix: env: MPI=mpich PY=3 language: generic python: 3 + - os: osx + osx_image: xcode10.3 + env: MPI=mpich PY=3 + language: generic + python: 3 + - os: osx + osx_image: xcode9.4 + env: MPI=mpich PY=3 + language: generic + python: 3 services: - postgresql From 8a4b86b23ab01aa9cb20165fd5631229203bdc99 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 15:26:48 -0500 Subject: [PATCH 278/644] dont install headers --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7b6db7353..f2aa048cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,6 @@ cache: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh -O miniconda.sh; - sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; else wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; fi @@ -69,6 +68,7 @@ before_install: - conda config --add channels conda-forge - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv + # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then From 902792a462c4c0a40c5ac67e20cfaffb73001e99 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 24 Sep 2019 15:53:36 -0500 Subject: [PATCH 279/644] Update mini_forces with AoS variation and build variants --- .../forces/mini_forces/build_forces.sh | 26 +++ .../forces/mini_forces/mini_forces.c | 2 +- .../forces/mini_forces/mini_forces_AoS.c | 180 ++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100755 libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh create mode 100644 libensemble/tests/scaling_tests/forces/mini_forces/mini_forces_AoS.c diff --git a/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh b/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh new file mode 100755 index 000000000..a3a9fb297 --- /dev/null +++ b/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Building with OpenMP for CPU + +# GCC for current host (generally not as good at vectorization as intel). +gcc -O3 -fopenmp -march=native -o mini_forces.x mini_forces.c -lm + +# Intel (fat binary covering vectorization for hswell/bdwell/skylake/knl) +# icc -O3 -qopenmp -axAVX,CORE-AVX2,CORE-AVX512,MIC-AVX512 -o mini_forces.x mini_forces.c + +# Intel with vectorization for current host (no cross-compilation). +# icc -O3 -qopenmp -xhost -o mini_forces.x mini_forces.c + +# xl +# xlc_r -O3 -qsimd=auto -qsmp=omp -o mini_forces.x mini_forces.c + +# ---------------------------------------------- + +# Building with OpenMP for target device (e.g. GPU) +# Need to toggle to OpenMP target directive in mini_forces.c. + +# xl +# xlc_r -O3 -qsmp=omp -qoffload -o mini_forces.x mini_forces.c + +# IRIS node (Intel Gen9 GPU) +# env MPICH_CC=icx mpigcc -g -fiopenmp -fopenmp-targets=spir64 -o mini_forces.x mini_forces.c diff --git a/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c index 0c9c7729a..625fac687 100644 --- a/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c +++ b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces.c @@ -6,7 +6,7 @@ E.g: gcc build and run on 2 threads on CPU: - gcc -O3 -fopenmp -o serial_forces.x serial_forces.c -lm + gcc -O3 -fopenmp -o mini_forces.x mini_forces.c -lm export OMP_NUM_THREADS=2 ./mini_forces.x diff --git a/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces_AoS.c b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces_AoS.c new file mode 100644 index 000000000..33d3b1cdb --- /dev/null +++ b/libensemble/tests/scaling_tests/forces/mini_forces/mini_forces_AoS.c @@ -0,0 +1,180 @@ +/* -------------------------------------------------------------------- + Non-MPI, Single Step, Electostatics Code Example + + This is a complete working example to test threading/vectorization + without MPI or other non-trivial features. + + E.g: gcc build and run on 2 threads on CPU: + + gcc -O3 -fopenmp -march=native -o mini_forces_AoS.x mini_forces_AoS.c -lm + export OMP_NUM_THREADS=2 + ./mini_forces_AoS.x + + E.g: xlc build and run on GPU: + + # First toggle #omp pragma line to target in forces_naive function. + xlc_r -O3 -qsmp=omp -qoffload -o mini_forces_AoS_xlc_gpu.x mini_forces_AoS.c + ./mini_forces_AoS.x + + Functionality: + Particles position and charge are initiated by a random stream. + Computes forces for all particles. Note: This version uses + an array of structures to store the data (AoS). The forces loop + should vectorize but will have overhead of gathers. + + OpenMP options for CPU and GPU. Toggle in forces_naive function. + + Author: S Hudson. +-------------------------------------------------------------------- */ +#include +#include +#include +#include +#include + +#define CHECK_THREADS 1 +#define NUM_PARTICLES 6000 + +typedef struct particle { + double p[3]; // Particle position + double f[3]; // Particle force + double q; // Particle charge +}__attribute__((__packed__)) particle; + +// Seed RNG +int seed_rand(int seed) { + srand(seed); + return 0; +} + +// Return a random number from a persistent stream +double get_rand() { + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + return randnum; +} + + +// Particles start at random locations in 10x10x10 cube +int build_system(int n, particle* parr) { + int q_range_low = -10; + int q_range_high = 10; + double extent = 10.0; + int i, dim; + + for(i=0; i Date: Tue, 24 Sep 2019 16:10:40 -0500 Subject: [PATCH 280/644] only activate xcode 9.4 for now --- .travis.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2aa048cf..ed23f77d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,16 +18,16 @@ env: matrix: include: - - os: osx - osx_image: xcode11 - env: MPI=mpich PY=3 - language: generic - python: 3 - - os: osx - osx_image: xcode10.3 - env: MPI=mpich PY=3 - language: generic - python: 3 + # - os: osx + # osx_image: xcode11 + # env: MPI=mpich PY=3 + # language: generic + # python: 3 + # - os: osx + # osx_image: xcode10.3 + # env: MPI=mpich PY=3 + # language: generic + # python: 3 - os: osx osx_image: xcode9.4 env: MPI=mpich PY=3 @@ -57,7 +57,7 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh -O miniconda.sh; else - wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O miniconda.sh; + wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh -O miniconda.sh; fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" @@ -68,7 +68,7 @@ before_install: - conda config --add channels conda-forge - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv - # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; + # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then From f501208495404554963fbce6e2790301421c6848 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 24 Sep 2019 16:32:33 -0500 Subject: [PATCH 281/644] narrow down working xcode versions --- .travis.yml | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed23f77d2..4799cc687 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,16 +18,31 @@ env: matrix: include: - # - os: osx - # osx_image: xcode11 - # env: MPI=mpich PY=3 - # language: generic - # python: 3 - # - os: osx - # osx_image: xcode10.3 - # env: MPI=mpich PY=3 - # language: generic - # python: 3 + - os: osx + osx_image: xcode11 + env: MPI=mpich PY=3 + language: generic + python: 3 + - os: osx + osx_image: xcode10.3 + env: MPI=mpich PY=3 + language: generic + python: 3 + - os: osx + osx_image: xcode10.2 + env: MPI=mpich PY=3 + language: generic + python: 3 + - os: osx + osx_image: xcode10.1 + env: MPI=mpich PY=3 + language: generic + python: 3 + - os: osx + osx_image: xcode10 + env: MPI=mpich PY=3 + language: generic + python: 3 - os: osx osx_image: xcode9.4 env: MPI=mpich PY=3 From 31c0fce0ab8a9db660d070355aa737d98bd771f4 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 24 Sep 2019 22:40:47 -0500 Subject: [PATCH 282/644] Fix mini_forces.c build line for icx --- .../tests/scaling_tests/forces/mini_forces/build_forces.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh b/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh index a3a9fb297..4c737ade0 100755 --- a/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh +++ b/libensemble/tests/scaling_tests/forces/mini_forces/build_forces.sh @@ -23,4 +23,4 @@ gcc -O3 -fopenmp -march=native -o mini_forces.x mini_forces.c -lm # xlc_r -O3 -qsmp=omp -qoffload -o mini_forces.x mini_forces.c # IRIS node (Intel Gen9 GPU) -# env MPICH_CC=icx mpigcc -g -fiopenmp -fopenmp-targets=spir64 -o mini_forces.x mini_forces.c +# icx -g -fiopenmp -fopenmp-targets=spir64 -o mini_forces.x mini_forces.c From 43a0e32d6600ae39f7ff2d756f487ec2d63633bd Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 25 Sep 2019 17:10:10 -0500 Subject: [PATCH 283/644] narrow down to most recent working version --- .travis.yml | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4799cc687..09c52672a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,36 +18,12 @@ env: matrix: include: - - os: osx - osx_image: xcode11 - env: MPI=mpich PY=3 - language: generic - python: 3 - - os: osx - osx_image: xcode10.3 - env: MPI=mpich PY=3 - language: generic - python: 3 - - os: osx - osx_image: xcode10.2 - env: MPI=mpich PY=3 - language: generic - python: 3 - os: osx osx_image: xcode10.1 env: MPI=mpich PY=3 language: generic python: 3 - - os: osx - osx_image: xcode10 - env: MPI=mpich PY=3 - language: generic - python: 3 - - os: osx - osx_image: xcode9.4 - env: MPI=mpich PY=3 - language: generic - python: 3 + services: - postgresql From f90b7a370444cfb6135fd66df6d31ce6d60e27a7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 26 Sep 2019 11:48:24 -0500 Subject: [PATCH 284/644] try omitting balsam_controller in non-balsam coverage --- libensemble/tests/regression_tests/.coveragerc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index eecb88a91..19178f1e8 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -9,7 +9,7 @@ omit = */branin_*/* branch = true - + data_file = .cov_reg_out parallel = true @@ -22,9 +22,11 @@ omit = */unit_tests/* */unit_tests_nompi/* */regression_tests/* + */balsam_controller.py + exclude_lines = if __name__ == .__main__.: -[html] +[html] directory = cov_reg From bf07e710a29cd23e59b24efdfd3d553344298fde Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 26 Sep 2019 12:08:16 -0500 Subject: [PATCH 285/644] adjust path --- libensemble/tests/regression_tests/.coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index 19178f1e8..525564f11 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -22,7 +22,7 @@ omit = */unit_tests/* */unit_tests_nompi/* */regression_tests/* - */balsam_controller.py + balsam_controller.py exclude_lines = From a7ab284e77ba48fd01243acf4f4c5dbadbdb43df Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 26 Sep 2019 12:53:05 -0500 Subject: [PATCH 286/644] Removing tabs --- .../release_platforms/rel_spack.rst | 2 +- .../job_submission_scripts/bebop_submit_slurm.sh | 2 +- .../bebop_submit_slurm_centralmode.sh | 2 +- libensemble/tests/regression_tests/.coveragerc | 2 +- libensemble/tests/scaling_tests/forces/readme.md | 14 +++++++------- libensemble/tests/unit_tests/.coveragerc | 2 +- libensemble/tests/unit_tests_logger/.coveragerc | 2 +- libensemble/tests/unit_tests_nompi/.coveragerc | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index c1b20b436..6a6c5c6c0 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -34,7 +34,7 @@ To set upstream repo:: (Optional) Prevent accidental pushes to upstream:: git remote set-url --push upstream no_push - git remote -v # Check for line: `upstream no_push (push)` + git remote -v # Check for line: `upstream no_push (push)` Now to update (the main develop branch) diff --git a/examples/job_submission_scripts/bebop_submit_slurm.sh b/examples/job_submission_scripts/bebop_submit_slurm.sh index d73de4fa5..49f58cf81 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm.sh @@ -43,7 +43,7 @@ fi; #--------------------------------------------------------------------------------------------- #Test echo -e "Slurm job ID: $SLURM_JOBID" - + #cd $PBS_O_WORKDIR cd $SLURM_SUBMIT_DIR diff --git a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh index d852b158f..48eb43026 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh @@ -31,7 +31,7 @@ export LIBE_WALLCLOCK=55 #--------------------------------------------------------------------------------------------- #Test echo -e "Slurm job ID: $SLURM_JOBID" - + #cd $PBS_O_WORKDIR cd $SLURM_SUBMIT_DIR diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index eecb88a91..c55ce25e9 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -9,7 +9,7 @@ omit = */branin_*/* branch = true - + data_file = .cov_reg_out parallel = true diff --git a/libensemble/tests/scaling_tests/forces/readme.md b/libensemble/tests/scaling_tests/forces/readme.md index 27ad70fcb..39490cbfd 100644 --- a/libensemble/tests/scaling_tests/forces/readme.md +++ b/libensemble/tests/scaling_tests/forces/readme.md @@ -66,28 +66,28 @@ scripts such as `theta_submit_balsam.sh`. The scripts are set up assuming a conda environment. To use script directly you will need to replace the following templated values: - in the COBALT -A directive with your project ID. + in the COBALT -A directive with your project ID. - is the name of your conda environment. + is the name of your conda environment. and in Balsam scripts: - The name of an initialized balsam database. - (with max_connections enough for number of workers) + The name of an initialized balsam database. + (with max_connections enough for number of workers) The included scripts are. * theta_submit_mproc.sh: - Example Theta submission script to run libEnsemble in central mode on the Theta launch (MOM) nodes with multiprocessing worker concurrency. + Example Theta submission script to run libEnsemble in central mode on the Theta launch (MOM) nodes with multiprocessing worker concurrency. * summit_submit_mproc.sh: - Example Summit submission script to run libEnsemble in central mode on the Summit launch nodes with multiprocessing worker concurrency. + Example Summit submission script to run libEnsemble in central mode on the Summit launch nodes with multiprocessing worker concurrency. * theta_submit_balsam.sh: - Example Theta submission script to run libEnsemble in central mode with MPI worker concurrency using Balsam. In this case libEnsemble manager and workers run on compute nodes and sumit jobs via Balsam. + Example Theta submission script to run libEnsemble in central mode with MPI worker concurrency using Balsam. In this case libEnsemble manager and workers run on compute nodes and sumit jobs via Balsam. #### Plotting Options diff --git a/libensemble/tests/unit_tests/.coveragerc b/libensemble/tests/unit_tests/.coveragerc index c520890fe..434d58279 100644 --- a/libensemble/tests/unit_tests/.coveragerc +++ b/libensemble/tests/unit_tests/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out #parallel = true diff --git a/libensemble/tests/unit_tests_logger/.coveragerc b/libensemble/tests/unit_tests_logger/.coveragerc index 50900a2c2..0a22a922f 100644 --- a/libensemble/tests/unit_tests_logger/.coveragerc +++ b/libensemble/tests/unit_tests_logger/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out3 #parallel = true diff --git a/libensemble/tests/unit_tests_nompi/.coveragerc b/libensemble/tests/unit_tests_nompi/.coveragerc index 95072f617..2c9a3a53d 100644 --- a/libensemble/tests/unit_tests_nompi/.coveragerc +++ b/libensemble/tests/unit_tests_nompi/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out2 #parallel = true From b90ebec7ed29eaf86b7a135bb348ea297d56a5f2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 26 Sep 2019 13:18:53 -0500 Subject: [PATCH 287/644] removing whitespace --- .flake8 | 10 +++--- .readthedocs.yml | 2 +- .style.yapf | 2 +- CONTRIBUTING.rst | 2 +- conda/run_travis_locally/readme.md | 31 +++++++++---------- docs/data_structures/alloc_specs.rst | 8 ++--- docs/data_structures/gen_specs.rst | 12 +++---- docs/data_structures/persis_info.rst | 2 +- docs/data_structures/sim_specs.rst | 14 ++++----- docs/data_structures/work_dict.rst | 26 ++++++++-------- docs/data_structures/worker_array.rst | 16 +++++----- .../dev_API/env_resources_module.rst | 2 +- docs/dev_guide/dev_API/history_module.rst | 2 +- docs/dev_guide/dev_API/manager_module.rst | 4 +-- docs/dev_guide/dev_API/resources_module.rst | 4 +-- docs/dev_guide/dev_API/worker_module.rst | 2 +- .../release_platforms/rel_pypi.rst | 2 +- .../release_platforms/rel_spack.rst | 28 ++++++++--------- docs/examples/alloc_funcs.rst | 4 +-- docs/examples/examples_index.rst | 2 +- docs/examples/gen_funcs.rst | 3 +- docs/examples/sim_funcs.rst | 2 +- docs/job_controller/balsam_controller.rst | 8 ++--- docs/job_controller/jc_index.rst | 2 +- docs/job_controller/job_controller.rst | 12 +++---- docs/job_controller/mpi_controller.rst | 6 ++-- docs/job_controller/overview.rst | 12 +++---- docs/user_funcs.rst | 2 +- .../bebop_submit_slurm.sh | 16 +++++----- .../bebop_submit_slurm_centralmode.sh | 16 +++++----- .../job_submission_scripts/blues_script.pbs | 8 ++--- .../theta_submit_balsam.sh | 2 +- libensemble/tests/.coveragerc | 2 +- .../bash_scripts/setup_balsam_tests.sh | 6 ++-- libensemble/tests/balsam_tests/readme.rst | 24 +++++++------- .../tests/regression_tests/.coveragerc | 4 +-- .../scaling_tests/forces/balsam_local.sh | 2 +- .../tests/scaling_tests/forces/clone.sh | 4 +-- .../tests/scaling_tests/forces/readme.md | 8 ++--- .../forces/summit_submit_mproc.sh | 6 ++-- .../forces/theta_submit_balsam.sh | 4 +-- .../forces/theta_submit_mproc.sh | 2 +- .../kill_test/log.autotest.txt | 17 +++++----- .../kill_test/log.burn_time_test_with_top.txt | 28 ++++++++--------- .../standalone_tests/kill_test/readme.txt | 2 +- .../mpi_launch_test/readme.txt | 2 +- libensemble/tests/unit_tests/.coveragerc | 6 ++-- .../tests/unit_tests_logger/.coveragerc | 6 ++-- .../tests/unit_tests_nompi/.coveragerc | 6 ++-- postproc_scripts/compare_npy.py | 4 +-- postproc_scripts/plot_libE_histogram.py | 6 ++-- postproc_scripts/readme.rst | 2 +- 52 files changed, 198 insertions(+), 207 deletions(-) diff --git a/.flake8 b/.flake8 index 5d7132daf..6f4334ab3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = +ignore = # Not worrying about line length for now E501, # Not worrying about spaces around arithmetic operations @@ -12,12 +12,12 @@ ignore = W504 # Lambda expressions are okay E731 -exclude = +exclude = # Don't need to check .git .git, # Largely autogenerated docs/conf.py - # Keeping bad python format to match PETSc source code + # Keeping bad python format to match PETSc source code libensemble/sim_funcs/chwirut1.py examples/sim_funcs/chwirut1.py # This is not implemented yet @@ -35,5 +35,5 @@ per-file-ignores = libensemble/libensemble/__init__.py:F401 # Need to turn of matching probes (before other imports) on some # systems/versions of MPI: - libensemble/tests/standalone_tests/mpi_launch_test/create_mpi_jobs.py:E402 - libensemble/libensemble/tests/standalone_tests/mpi_launch_test/create_mpi_jobs.py:E402 + libensemble/tests/standalone_tests/mpi_launch_test/create_mpi_jobs.py:E402 + libensemble/libensemble/tests/standalone_tests/mpi_launch_test/create_mpi_jobs.py:E402 diff --git a/.readthedocs.yml b/.readthedocs.yml index 2695aca1c..a8f801e04 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,6 +1,6 @@ conda: file: conda/environment.yml - + python: version: 3.6 pip_install: true diff --git a/.style.yapf b/.style.yapf index 4908524cd..31be33349 100644 --- a/.style.yapf +++ b/.style.yapf @@ -6,7 +6,7 @@ SPLIT_BEFORE_DOT = True COLUMN_LIMIT = 79 SPLIT_BEFORE_NAMED_ASSIGNS = False SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = False -ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS = False +ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS = False ALLOW_SPLIT_BEFORE_DICT_VALUE = False DEDENT_CLOSING_BRACKETS = False EACH_DICT_ENTRY_ON_SEPARATE_LINE = True diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 411f069ad..ad252cb8d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -13,7 +13,7 @@ given in the ".flake8" configuration file in the project directory. Issues can be raised at: https://github.com/Libensemble/libensemble/issues - + Issues may include reporting bugs or suggested features. Administrators will add issues, as appropriate, to the project board at: diff --git a/conda/run_travis_locally/readme.md b/conda/run_travis_locally/readme.md index fa8979c67..c0476c606 100644 --- a/conda/run_travis_locally/readme.md +++ b/conda/run_travis_locally/readme.md @@ -4,7 +4,7 @@ This explains how to replicate the Travis container environment locally. -This can save a lot of git pushes and trial-and-error when there is problems in Travis builds that are not +This can save a lot of git pushes and trial-and-error when there is problems in Travis builds that are not observed locally. #### References: @@ -32,8 +32,8 @@ Add user name to docker group so dont need sudo before docker: sudo usermod -aG docker ${USER} su - ${USER} id -nG # Confirm username added to docker users - - + + ### Install Travis Image for Python and create a container Name your new travis container: @@ -46,7 +46,7 @@ Install the Travis Docker image for Python and run the container (server): This might take a while if image is not downloaded. Once image is downloaded it will be run from cache. -Note: +Note: travisci/ci-garnet:packer-1512502276-986baf0 is the Travis Python image. The name in $CONTAINER is the name you are assigning to the new container made from the image. @@ -58,7 +58,7 @@ Alternative travisCI docker images can be found [here](https://hub.docker.com/r/ Then open a shell in running container: sudo docker exec -it $CONTAINER bash -l - + Prompt should say travis@ rather than root@: @@ -72,7 +72,7 @@ the installs against .travis.yml in the top level libEnsemble package directory. #### Copy build script from host system to the running container -Run the following **from your host machine environment**. This copies the given file into an existing +Run the following **from your host machine environment**. This copies the given file into an existing container named $CONTAINER. Where /home/travis is target location in container filesystem. @@ -82,19 +82,19 @@ On the docker side you may need to set ownership of the script: chown travis:travis /home/travis/build_mpich_libE.sh - + #### Run the libEnsemble build-and-run script Now, in the docker container, become user travis: su - travis - + Source the script, setting python version and libEnsemble branch if necessary (see script for defaults). **Always source** to maintain environment variables after running (inc. miniconda path). The following would run with Python 3.7 and test the libEnsemble branch hotfix/logging: . ./build_mpich_libE.sh -p 3.7 -b hotfix/logging - + Note: libEnsemble will be git cloned and checked out at the given branch. The script should build all dependencies and run tests. Having sourced the script, you should @@ -108,12 +108,12 @@ If the conda output is too verbose, remove the "set -x" line in the script. To exit the container session (client). exit # Travis user exit # Exit running session (but container server is still running. - + Re-enter while background container still running: sudo docker exec -it $CONTAINER bash -l su - travis - + Note that environment variables are not saved. To save the modified container as an image (with your changes to filesystem): @@ -123,7 +123,7 @@ Note: This will not by default save your environment variables First to check running containers (ie. running server - whether or not you are in a session): docker ps - + OR docker container ls @@ -144,13 +144,13 @@ Now it should show if you do: If it is saved you can stop the container (server) thats running and restart eg. docker stop $CONTAINER - + where $CONTAINER is the name you gave the container on the docker run command. You can restart from your new image with docker run and docker exec or to run server and run session in one: docker run -it / /bin/bash - + where / is as used above to save (or from first column in "docker images" output). @@ -164,6 +164,3 @@ container can be [restricted](https://docs.docker.com/config/containers/resource An alternative to this process is to log in to your Travis build and debug. For public repositories, this requires contacting Travis support for a token that can only be used by the given user. See [here](https://docs.travis-ci.com/user/running-build-in-debug-mode/). - - - diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index c1ab58c4f..a02c37f8d 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -8,13 +8,13 @@ Allocation function specifications to be set in user calling script and passed t alloc_specs: [dict, optional] : - Required keys : - + Required keys : + 'alloc_f' [func] : Default: give_sim_work_first - + Optional keys : - + 'out' [list of tuples] : Default: [('allocated',bool)] diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 97c87e8c4..2a937f492 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -7,17 +7,17 @@ Generation function specifications to be set in user calling script and passed t gen_specs: [dict]: - Required keys : - - 'gen_f' [func] : + Required keys : + + 'gen_f' [func] : generates inputs to sim_f - 'in' [list] : + 'in' [list] : field names (as strings) that will be given to gen_f 'out' [list of tuples (field name, data type, [size])] : gen_f outputs that will be stored in the libEnsemble history - + Optional keys : - + 'save_every_k' [int] : Save history array to file after every k generated points. diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 5bb8f6d25..3f5bfbbd2 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -13,7 +13,7 @@ is a randon number generator to be used in consecutive calls to a generator. If worker ``i`` sends back ``persis_info``, it is stored in ``persis_info[i]``. This functionality can be used to, for example, pass a random stream back to the manager to be included in future work -from the allocation function. +from the allocation function. :Examples: diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index 58afa8412..63a6396c1 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -8,18 +8,18 @@ Simulation function specifications to be set in user calling script and passed t sim_specs: [dict]: - Required keys : - - 'sim_f' [func] : + Required keys : + + 'sim_f' [func] : the simulation function being evaluated 'in' [list] : field names (as strings) that will be given to sim_f 'out' [list of tuples (field name, data type, [size])] : sim_f outputs that will be stored in the libEnsemble history - + Optional keys : - + 'save_every_k' [int] : Save history array to file after every k simulated points. 'sim_dir' [str] : @@ -32,7 +32,7 @@ Simulation function specifications to be set in user calling script and passed t Profile using cProfile. Default: False Additional entires in sim_specs will be given to sim_f - + :Notes: * The user may define other fields to be passed to the simulator function. @@ -49,7 +49,7 @@ From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py sim_specs = {'sim_f': six_hump_camel, # This is the function whose output is being minimized 'in': ['x'], # These keys will be given to the above function 'out': [('f',float)], # This is the output from the function being minimized - 'save_every_k': 400 + 'save_every_k': 400 } Note that the dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 1d78bf187..6e4bc7ec6 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -3,38 +3,38 @@ work dictionary =============== -Dictionary with integer keys ``i`` and dictionary values to be given to worker ``i``. +Dictionary with integer keys ``i`` and dictionary values to be given to worker ``i``. ``Work[i]`` has the following form:: Work[i]: [dict]: - Required keys : - 'persis_info' [dict]: Any persistent info to be sent to worker 'i' + Required keys : + 'persis_info' [dict]: Any persistent info to be sent to worker 'i' - 'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' + 'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' - 'tag' [int]: 'EVAL_SIM_TAG' (resp. 'EVAL_GEN_TAG') if worker 'i' is to call sim_func (resp. gen_func) + 'tag' [int]: 'EVAL_SIM_TAG' (resp. 'EVAL_GEN_TAG') if worker 'i' is to call sim_func (resp. gen_func) - 'libE_info' [dict]: This information is sent to and returned from the worker to help libEnsemble quickly update the 'H' and 'W'. + 'libE_info' [dict]: This information is sent to and returned from the worker to help libEnsemble quickly update the 'H' and 'W'. Available keys are: H_rows' [list of ints]: History rows to send to worker 'i' blocking' [list of ints]: Workers to be blocked by the calculation given to worker 'i' - persistent' [bool]: True if worker 'i' will enter persistent mode - - + persistent' [bool]: True if worker 'i' will enter persistent mode + + :Examples: .. How to link directly to the file? -| For allocation functions using persistent workers, see -| ``libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` -| or +| For allocation functions using persistent workers, see +| ``libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` +| or | ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` | -| For allocation functions giving work that blocks other workers, see +| For allocation functions giving work that blocks other workers, see | ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index a26112204..88420549e 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -10,14 +10,14 @@ convention: ========================================= ======= ============ ======= Worker state active persis_state blocked ========================================= ======= ============ ======= -idle worker 0 0 0 -active, nonpersistent sim 1 0 0 -active, nonpersistent gen 2 0 0 -active, persistent sim 1 1 0 -active, persistent gen 2 2 0 -waiting, persistent sim 0 1 0 -waiting, persistent gen 0 2 0 -worker blocked by some other calculation 1 0 1 +idle worker 0 0 0 +active, nonpersistent sim 1 0 0 +active, nonpersistent gen 2 0 0 +active, persistent sim 1 1 0 +active, persistent gen 2 2 0 +waiting, persistent sim 0 1 0 +waiting, persistent gen 0 2 0 +worker blocked by some other calculation 1 0 1 ========================================= ======= ============ ======= :Note: diff --git a/docs/dev_guide/dev_API/env_resources_module.rst b/docs/dev_guide/dev_API/env_resources_module.rst index 08747268a..3f4367034 100644 --- a/docs/dev_guide/dev_API/env_resources_module.rst +++ b/docs/dev_guide/dev_API/env_resources_module.rst @@ -6,5 +6,5 @@ Environment Resources Module .. autoclass:: EnvResources :member-order: bysource :members: - + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/history_module.rst b/docs/dev_guide/dev_API/history_module.rst index 57c97d821..0a5527380 100644 --- a/docs/dev_guide/dev_API/history_module.rst +++ b/docs/dev_guide/dev_API/history_module.rst @@ -6,5 +6,5 @@ History Module .. autoclass:: History :member-order: bysource :members: - + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/manager_module.rst b/docs/dev_guide/dev_API/manager_module.rst index c0917af21..ae4d97e9d 100644 --- a/docs/dev_guide/dev_API/manager_module.rst +++ b/docs/dev_guide/dev_API/manager_module.rst @@ -3,9 +3,9 @@ Manager Module .. automodule:: libE_manager :members: manager_main :undoc-members: - + .. autoclass:: Manager :member-order: bysource :members: - + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/resources_module.rst b/docs/dev_guide/dev_API/resources_module.rst index cc998cbf9..a05669f45 100644 --- a/docs/dev_guide/dev_API/resources_module.rst +++ b/docs/dev_guide/dev_API/resources_module.rst @@ -6,11 +6,11 @@ Resources Module .. autoclass:: Resources :member-order: bysource :members: - + .. automethod:: __init__ .. autoclass:: WorkerResources :member-order: bysource :members: - + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/worker_module.rst b/docs/dev_guide/dev_API/worker_module.rst index 855e39c4d..79a27b1f8 100644 --- a/docs/dev_guide/dev_API/worker_module.rst +++ b/docs/dev_guide/dev_API/worker_module.rst @@ -6,6 +6,6 @@ Worker Module .. autoclass:: Worker :member-order: bysource :members: - + .. automethod:: __init__ diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst index 906d250af..3cbdd32f3 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -3,7 +3,7 @@ PyPI release ============ -libEnsemble is released on the Python Package Index (commonly known as PyPI). This enable users to “pip install” the package. +libEnsemble is released on the Python Package Index (commonly known as PyPI). This enable users to “pip install” the package. The package is stored on PyPI in the form of a source distribution (commonly known as a tarball). The tarball could be obtained from github, though historically this has been created with a checkout of libensemble from git. diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index 6a6c5c6c0..db058de05 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -35,7 +35,7 @@ To set upstream repo:: git remote set-url --push upstream no_push git remote -v # Check for line: `upstream no_push (push)` - + Now to update (the main develop branch) --------------------------------------- @@ -66,12 +66,12 @@ Or to make your local machine identical to upstream repo (**WARNING** Any local git reset --hard upstream/develop - + (Optional) You may want to update your forked (origin) repo on github at this point. This may requires a forced push:: git push origin develop --force - + Making changes -------------- @@ -87,14 +87,14 @@ Quick example to update libensemble:: git branch update_libensemble git checkout update_libensemble - + This will open the libensemble package.py file in your editor (given by env variable EDITOR):: spack edit py-libensemble # SPACK_ROOT must be set (see above) (python packages use "py-" prefix) Or just open it manually: var/spack/repos/builtin/packages/py-libensemble/package.py - - + + Now get checksum for new lines: Get the tarball (see PyPI instructions), for the new release and use:: @@ -112,22 +112,22 @@ If OK add, commit and push to origin (forked repo):: git commit -am "Update libensemble" git push origin update_libensemble --force - + Once the branch is pushed to the forked repo - go to github and do a pull request from this branch on the fork to the develop branch on the upstream. - + Express summary: Make fork identical to upstream ------------------------------------------------ Quick summary for bringing develop branch on forked repo up to speed with upstream (YOU WILL LOSE ANY CHANGES):: - git remote add upstream https://github.com/spack/spack.git - git fetch upstream - git checkout develop - git reset --hard upstream/develop - git push origin develop --force + git remote add upstream https://github.com/spack/spack.git + git fetch upstream + git checkout develop + git reset --hard upstream/develop + git push origin develop --force Reference: - + diff --git a/docs/examples/alloc_funcs.rst b/docs/examples/alloc_funcs.rst index 75583bf7f..3ab9f1e8d 100644 --- a/docs/examples/alloc_funcs.rst +++ b/docs/examples/alloc_funcs.rst @@ -13,7 +13,7 @@ give_sim_work_first .. automodule:: give_sim_work_first :members: :undoc-members: - + fast_alloc ---------- .. automodule:: fast_alloc @@ -37,5 +37,3 @@ start_persistent_local_opt_gens .. automodule:: start_persistent_local_opt_gens :members: :undoc-members: - - diff --git a/docs/examples/examples_index.rst b/docs/examples/examples_index.rst index f5f29dd7a..d4a4dd89d 100644 --- a/docs/examples/examples_index.rst +++ b/docs/examples/examples_index.rst @@ -7,7 +7,7 @@ Example gen, sim and alloc functions for libEnsemble. .. toctree:: :maxdepth: 2 :caption: libEnsemble Example Functions: - + gen_funcs sim_funcs alloc_funcs diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 64250d174..110216e36 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -17,7 +17,7 @@ APOSMM .. automodule:: aposmm :members: :undoc-members: - + uniform_or_localopt ------------------- .. automodule:: uniform_or_localopt @@ -29,4 +29,3 @@ persistent_uniform_sampling .. automodule:: persistent_uniform_sampling :members: :undoc-members: - diff --git a/docs/examples/sim_funcs.rst b/docs/examples/sim_funcs.rst index 377daa6f5..78ccc1438 100644 --- a/docs/examples/sim_funcs.rst +++ b/docs/examples/sim_funcs.rst @@ -17,7 +17,7 @@ chwirut .. automodule:: chwirut1 :members: :undoc-members: - + job_control_hworld ------------------ .. automodule:: job_control_hworld diff --git a/docs/job_controller/balsam_controller.rst b/docs/job_controller/balsam_controller.rst index df3489a58..52fe72d46 100644 --- a/docs/job_controller/balsam_controller.rst +++ b/docs/job_controller/balsam_controller.rst @@ -8,20 +8,20 @@ To create a Balsam job controller, the calling script should contain:: jobctr = BalsamJobController() The Balsam job controller inherits from the MPI job controller. See the -:doc:`MPIJobController` for shared API. Any differences are +:doc:`MPIJobController` for shared API. Any differences are shown below. .. automodule:: balsam_controller :no-undoc-members: - + .. autoclass:: BalsamJobController :show-inheritance: .. :inherited-members: -.. :member-order: bysource +.. :member-order: bysource .. :members: __init__, launch, poll, manager_poll, kill, set_kill_mode .. autoclass:: BalsamJob :show-inheritance: :member-order: bysource -.. :members: workdir_exists, file_exists_in_workdir, read_file_in_workdir, stdout_exists, read_stdout +.. :members: workdir_exists, file_exists_in_workdir, read_file_in_workdir, stdout_exists, read_stdout .. :inherited-members: diff --git a/docs/job_controller/jc_index.rst b/docs/job_controller/jc_index.rst index 6318f06a6..16bed80a8 100644 --- a/docs/job_controller/jc_index.rst +++ b/docs/job_controller/jc_index.rst @@ -9,6 +9,6 @@ The job controller can be used within the simulator (and potentially generator) :maxdepth: 2 :titlesonly: :caption: libEnsemble Job Controller: - + overview job_controller diff --git a/docs/job_controller/job_controller.rst b/docs/job_controller/job_controller.rst index 84bd6a2f1..a2a312e9b 100644 --- a/docs/job_controller/job_controller.rst +++ b/docs/job_controller/job_controller.rst @@ -3,15 +3,15 @@ Job Controller Module .. automodule:: controller :no-undoc-members: - + See :doc:`example` for usage. -See the controller APIs for optional arguments. +See the controller APIs for optional arguments. .. toctree:: :maxdepth: 1 :caption: Job Controllers: - + mpi_controller balsam_controller @@ -32,10 +32,10 @@ polled/killed (or through other job or job controller functions). Job Attributes -------------- - + Following is a list of job status and configuration attributes that can be retrieved from a job. -:NOTE: These should not be set directly. Jobs are launched by the job controller and job information can be queired through the job attributes below and the query functions. +:NOTE: These should not be set directly. Jobs are launched by the job controller and job information can be queired through the job attributes below and the query functions. Job Status attributes include: @@ -53,7 +53,7 @@ Run configuration attributes - Some will be auto-generated: :job.workdir: (string) Work directory for the job :job.name: (string) Name of job - auto-generated :job.app: (app obj) Use application/executable, registered using jobctl.register_calc -:job.app_args: (string) Application arguments as a string +:job.app_args: (string) Application arguments as a string :job.stdout: (string) Name of file where the standard output of the job is written (in job.workdir) :job.stderr: (string) Name of file where the standard error of the job is written (in job.workdir) diff --git a/docs/job_controller/mpi_controller.rst b/docs/job_controller/mpi_controller.rst index f3e0915c4..117ea8293 100644 --- a/docs/job_controller/mpi_controller.rst +++ b/docs/job_controller/mpi_controller.rst @@ -5,8 +5,8 @@ To create an MPI job controller, the calling script should contain:: jobctl = MPIJobController() -See the controller API below for optional arguments. - +See the controller API below for optional arguments. + .. automodule:: mpi_controller :no-undoc-members: @@ -15,7 +15,7 @@ See the controller API below for optional arguments. :inherited-members: .. automethod:: __init__ - + .. :member-order: bysource .. :members: __init__, register_calc, launch, manager_poll diff --git a/docs/job_controller/overview.rst b/docs/job_controller/overview.rst index 9e34033e3..95b87bc53 100644 --- a/docs/job_controller/overview.rst +++ b/docs/job_controller/overview.rst @@ -13,16 +13,16 @@ cluster, or any other provision of compute resources. In order to remove the burden of system interaction from the user, and enable sim_f scripts that are portable between systems, a job_controller interface is provided by libEnsemble. The job_controller provides the key functions: ``launch()``, ``poll()`` and -``kill()``. libEnsemble auto-detects a number of system criteria, such as the MPI launcher, +``kill()``. libEnsemble auto-detects a number of system criteria, such as the MPI launcher, along with correct mechanisms for polling and killing jobs, on supported systems. It also contains built in resilience, such as re-launching jobs that fail due to system factors. User scripts that employ the job_controller interface will be portable between supported -systems. Job attributes can be queried to determine status after each poll. Functions are +systems. Job attributes can be queried to determine status after each poll. Functions are also provided to access and interrogate files in the job's working directory. The Job Controller module can be used to submit and manage jobs using a portable interface. Various back-end mechanisms may be -used to implement this interface on the system, including a proxy launcher and +used to implement this interface on the system, including a proxy launcher and job management system, such as Balsam. Currently, these job_controllers launch at the application level within an existing resource pool. However, submissions to a batch schedular may be supported in the future. @@ -62,7 +62,7 @@ In user sim func:: timeout_sec = 600 poll_delay_sec = 1 - + while(not job.finished): # Has manager sent a finish signal @@ -82,10 +82,10 @@ In user sim func:: else: time.sleep(poll_delay_sec) job.poll() - + print(job.state) # state may be finished/failed/killed -See the :doc:`job_controller` interface for API. +See the :doc:`job_controller` interface for API. For a more realistic example see: diff --git a/docs/user_funcs.rst b/docs/user_funcs.rst index 4cc352bc6..6c8d60211 100644 --- a/docs/user_funcs.rst +++ b/docs/user_funcs.rst @@ -10,6 +10,6 @@ must be provided. The required API and examples are given here. .. toctree:: :maxdepth: 2 :caption: libEnsemble User Functions: - + sim_gen_alloc_funcs examples/examples_index diff --git a/examples/job_submission_scripts/bebop_submit_slurm.sh b/examples/job_submission_scripts/bebop_submit_slurm.sh index 49f58cf81..fa2e07a18 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm.sh @@ -3,9 +3,9 @@ #SBATCH -N 4 #SBATCH -p knlall ##SBATCH -A -#SBATCH -o tlib.%j.%N.out -#SBATCH -e tlib.%j.%N.error -#SBATCH -t 01:00:00 +#SBATCH -o tlib.%j.%N.out +#SBATCH -e tlib.%j.%N.error +#SBATCH -t 01:00:00 #Launch script that evenly spreads workers and adds manager to the first node. #Requires even distribution - either multiple workers per node or nodes per worker @@ -43,7 +43,7 @@ fi; #--------------------------------------------------------------------------------------------- #Test echo -e "Slurm job ID: $SLURM_JOBID" - + #cd $PBS_O_WORKDIR cd $SLURM_SUBMIT_DIR @@ -56,7 +56,7 @@ srun hostname | sort -u > node_list #Generate list of nodes for workers if [[ $MANAGER_NODE = "true" ]]; then - tail -n +2 node_list > worker_list + tail -n +2 node_list > worker_list else cp node_list worker_list fi @@ -79,9 +79,9 @@ export SLURM_HOSTFILE=machinefile.$SLURM_JOBID # cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE" cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE $LIBE_WALLCLOCK" -echo The command is: $cmd -echo End PBS script information. -echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished +echo The command is: $cmd +echo End PBS script information. +echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished $cmd diff --git a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh index 48eb43026..700a8da06 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh @@ -3,12 +3,12 @@ #SBATCH -N 5 #SBATCH -p knlall ##SBATCH -A -#SBATCH -o tlib.%j.%N.out -#SBATCH -e tlib.%j.%N.error -#SBATCH -t 01:00:00 +#SBATCH -o tlib.%j.%N.out +#SBATCH -e tlib.%j.%N.error +#SBATCH -t 01:00:00 #Launch script for running in central mode. -#LibEnsemble will run on a dedicated node (or nodes). +#LibEnsemble will run on a dedicated node (or nodes). #The remaining nodes in the allocation will be dedicated to the jobs launched by the workers. #Requirements for running: @@ -31,7 +31,7 @@ export LIBE_WALLCLOCK=55 #--------------------------------------------------------------------------------------------- #Test echo -e "Slurm job ID: $SLURM_JOBID" - + #cd $PBS_O_WORKDIR cd $SLURM_SUBMIT_DIR @@ -42,9 +42,9 @@ echo -e "Directory is: $PWD" #This will work for the number of contexts that will fit on one node (eg. 320 on Bebop) - increase libE nodes for more. cmd="srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE $LIBE_WALLCLOCK" -echo The command is: $cmd -echo End PBS script information. -echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished +echo The command is: $cmd +echo End PBS script information. +echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished $cmd diff --git a/examples/job_submission_scripts/blues_script.pbs b/examples/job_submission_scripts/blues_script.pbs index f24c32d93..1afac76f1 100644 --- a/examples/job_submission_scripts/blues_script.pbs +++ b/examples/job_submission_scripts/blues_script.pbs @@ -24,7 +24,7 @@ # Specify CPU time needed #PBS -l walltime=0:01:00 -# Select queue +# Select queue ##PBS -q haswell ##PBS -q biggpu ##PBS -q ivy @@ -46,7 +46,7 @@ echo This job has allocated $NPROCS cores rm libE_machinefile # Parse the PBS_NODEFILE to a machinefile (to be used below) in order to put # ranks 1-4 on each of the 4 nodes, and put rank 0 (the manager) on the first -# node as well. +# node as well. head -n 1 $PBS_NODEFILE > libE_machinefile cat $PBS_NODEFILE | sort | uniq >> libE_machinefile @@ -57,11 +57,11 @@ echo Starting executation at: `date` pwd cmd="mpiexec -np 5 -machinefile libE_machinefile python2 libE_calling_script.py libE_machinefile" # This note that this command passes the libE_machinefile to both MPI and the -# libE_calling_script, in the latter script, it can be parsed and given to the +# libE_calling_script, in the latter script, it can be parsed and given to the # alloc_func echo The command is: $cmd -echo End PBS script information. +echo End PBS script information. echo All further output is from the process being run and not the pbs script.\n\n $cmd diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index 89a85a172..a06c7a2ae 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -29,7 +29,7 @@ export LIBE_WALLCLOCK=45 export WORKFLOW_NAME=libe_workflow #sh - todo - may currently be hardcoded to this in libE - allow user to specify #Tell libE manager to stop workers, dump timing.dat and exit after this time. Script must be set up to receive as argument. -export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3)) +export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3)) # export SCRIPT_ARGS='' #Default No args # Name of Conda environment (Need to have set up: https://balsam.alcf.anl.gov/quick/quickstart.html) diff --git a/libensemble/tests/.coveragerc b/libensemble/tests/.coveragerc index fa2110df3..543436d73 100644 --- a/libensemble/tests/.coveragerc +++ b/libensemble/tests/.coveragerc @@ -4,7 +4,7 @@ [run] data_file = .cov_merge_out -[html] +[html] directory = cov_merge #Report files can be controlled here diff --git a/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh b/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh index 54b5c2302..c24234f88 100755 --- a/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh +++ b/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh @@ -37,7 +37,7 @@ balsam app --name $SIM_APP_NAME --exec $SIM_DIR/$SIM_APP --desc "Run $SIM_APP_NA for LIBE_APP in $JOB_LIST do LIBE_APP_NAME=${LIBE_APP%.*} - balsam app --name $LIBE_APP_NAME --exec $WORK_DIR/$LIBE_APP --desc "Run $LIBE_APP_NAME" + balsam app --name $LIBE_APP_NAME --exec $WORK_DIR/$LIBE_APP --desc "Run $LIBE_APP_NAME" #Add jobs balsam job --name job_$LIBE_APP_NAME --workflow libe_workflow --application $LIBE_APP_NAME \ @@ -45,10 +45,10 @@ do --url-out="local:$WORK_DIR" \ --stage-out-files="${JOB_NAME}*" \ --yes - + #Add dependency so jobs run one at a time .... # *** Currently all jobs added will run simultaneously - kills may be an issue there - + done echo -e "\nListing apps:" diff --git a/libensemble/tests/balsam_tests/readme.rst b/libensemble/tests/balsam_tests/readme.rst index 4939e9e51..6a309984b 100644 --- a/libensemble/tests/balsam_tests/readme.rst +++ b/libensemble/tests/balsam_tests/readme.rst @@ -13,7 +13,7 @@ If the instructions are followed to set up a conda environment called balsam, th Quickstart - Balsam tests for libensemble ----------------------------------------- -Having set up balsam, the tests here can be run as follows. +Having set up balsam, the tests here can be run as follows. **i) Set up the tests** @@ -66,9 +66,9 @@ Register tests: .. code-block:: bash balsam app --name test_balsam_1__runjobs --exec test_balsam_1__runjobs.py --desc "Run balsam test 1" - + balsam app --name test_balsam_2__workerkill --exec test_balsam_2__workerkill.py --desc "Run balsam test 2" - + balsam app --name test_balsam_3__managerkill --exec test_balsam_3__managerkill.py --desc "Run balsam test 3" Register user application that will be called inside tests: @@ -121,13 +121,13 @@ In this case 4 ranks per node and 1 node are selected. This is for running on th Note that the user jobs (launched in a libensemble job) are registered from within the code. For staging out files, the output directory needs to somehow be accessible to the code. For the tests here, this is simply the directory of the test scripts (accessed via the __file__ variable in python). Search for dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) -To list jobs: +To list jobs: .. code-block:: bash balsam ls jobs -To clean: +To clean: .. code-block:: bash @@ -143,15 +143,15 @@ Launch all jobs: .. code-block:: bash - balsam launcher --consume-all - + balsam launcher --consume-all + For other launcher options: .. code-block:: bash balsam launcher -h - - + + 4 Reset jobs ------------ @@ -159,7 +159,7 @@ A script to reset the tests is available: reset_balsam_tests.py This script can be modified easily. However, to reset from the command line - without removing and re-adding jobs you can do the following. - + Note: After running tests the balsam job database will contain something like the following (job_ids abbreviated for space): .. code-block:: bash @@ -167,7 +167,7 @@ Note: After running tests the balsam job database will contain something like th $ balsam ls jobs :: - + job_id | name | workflow | application | latest update ------------------------------------------------------------------------------------------------------------- 29add031-8e7c-... | job_balsam1 | libe_workflow | test_balsam_1 | [01-30-2018 18:57:47 JOB_FINISHED] @@ -196,7 +196,7 @@ To remove only the generated jobs you can just use a sub-string of the job name job_id | name | workflow | application | latest update ----------------------------------------------------------------------------------------------------------------------- 29add031-8e7c-... | job_balsam1 | libe_workflow | test_balsam_1 | [01-30-2018 18:57:47 JOB_FINISHED] - + To run again - change status attribute to READY (you need to specify job_id - an abbreviation is ok) eg: diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index c55ce25e9..25dfd8d59 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -9,7 +9,7 @@ omit = */branin_*/* branch = true - + data_file = .cov_reg_out parallel = true @@ -26,5 +26,5 @@ omit = exclude_lines = if __name__ == .__main__.: -[html] +[html] directory = cov_reg diff --git a/libensemble/tests/scaling_tests/forces/balsam_local.sh b/libensemble/tests/scaling_tests/forces/balsam_local.sh index dd7828fea..9e447b743 100755 --- a/libensemble/tests/scaling_tests/forces/balsam_local.sh +++ b/libensemble/tests/scaling_tests/forces/balsam_local.sh @@ -12,7 +12,7 @@ export EXE=run_libe_forces.py export NUM_WORKERS=2 # Name of working directory where Balsam places running jobs/output (inside the database directory) -export WORKFLOW_NAME=libe_workflow +export WORKFLOW_NAME=libe_workflow export SCRIPT_ARGS=$NUM_WORKERS diff --git a/libensemble/tests/scaling_tests/forces/clone.sh b/libensemble/tests/scaling_tests/forces/clone.sh index 80f7bed07..a0f491179 100644 --- a/libensemble/tests/scaling_tests/forces/clone.sh +++ b/libensemble/tests/scaling_tests/forces/clone.sh @@ -3,11 +3,11 @@ then echo -e "Need output dir name: ./clone NEW_DIR" else NEWDIR=$1 - + INPUTS="sim/ forces.c forces_simf.py \ run_libe_forces.py forces.x \ *.sh" # Take you there if source script. - mkdir ../$NEWDIR && cp -rp $INPUTS ../$NEWDIR && cd ../$NEWDIR + mkdir ../$NEWDIR && cp -rp $INPUTS ../$NEWDIR && cd ../$NEWDIR fi diff --git a/libensemble/tests/scaling_tests/forces/readme.md b/libensemble/tests/scaling_tests/forces/readme.md index 39490cbfd..68cf8899f 100644 --- a/libensemble/tests/scaling_tests/forces/readme.md +++ b/libensemble/tests/scaling_tests/forces/readme.md @@ -11,17 +11,17 @@ code for a libEnsemble sim func. A system of charged particles is set up and simulated over a number of time-steps. Particles position and charge are initiated by a random stream. -Particles are replicated on all ranks. +Particles are replicated on all ranks. **Each rank** computes forces for a subset of particles (O(N^2)) Particle force arrays are allreduced across ranks. Particles are moved (replicated on each rank) Total energy is appended to file forces.stat To run forces as a standalone executable on N procs: - + mpirun -np N ./forces.x - + ### Running with libEnsemble. A random sample of seeds is taken and used as imput to the sim func (forces miniapp). @@ -61,7 +61,7 @@ While the key output files will be copied back to the run dir at completion. Als the log in `/log` if there are any issues. To run on batch systems, see the example scripts such as `theta_submit_balsam.sh`. - + ### Using batch scripts The scripts are set up assuming a conda environment. To use script directly you will need to replace the following templated values: diff --git a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh index f10d1d344..2279b50dd 100644 --- a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh @@ -21,7 +21,7 @@ export EXE=run_libe_forces.py # Wallclock for libE. Slightly smaller than job wallclock #export LIBE_WALLCLOCK=15 # Optional if pass to script -# Name of Conda environment +# Name of Conda environment export CONDA_ENV_NAME= export LIBE_PLOTS=true # Require plot scripts in $PLOT_DIR (see at end) @@ -38,13 +38,13 @@ export PYTHONNOUSERSITE=1 # hash -d python # Check pick up python in conda env hash -r # Check no commands hashed (pip/python...) -# Launch libE. +# Launch libE. #python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 #python $EXE $NUM_WORKERS > out.txt 2>&1 python $EXE > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py - python $PLOT_DIR/plot_libe_runs_util_v_time.py + python $PLOT_DIR/plot_libe_runs_util_v_time.py python $PLOT_DIR/plot_libE_histogram.py fi diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index b24a7010d..d51a5692f 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -25,7 +25,7 @@ export LIBE_WALLCLOCK=25 export WORKFLOW_NAME=libe_workflow #sh - todo - may currently be hardcoded to this in libE - allow user to specify #Tell libE manager to stop workers, dump timing.dat and exit after this time. Script must be set up to receive as argument. -export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) +export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) # export SCRIPT_ARGS='' #Default No args # Name of Conda environment (Need to have set up: https://balsam.alcf.anl.gov/quick/quickstart.html) @@ -95,7 +95,7 @@ if [[ $LIBE_PLOTS = "true" ]]; then if [[ $BALSAM_PLOTS = "true" ]]; then # export MPLBACKEND=TkAgg - python $PLOT_DIR/plot_util_v_time.py + python $PLOT_DIR/plot_util_v_time.py python $PLOT_DIR/plot_jobs_v_time.py python $PLOT_DIR/plot_waiting_v_time.py fi diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh index c62cc19f9..6454e4f7b 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh @@ -47,6 +47,6 @@ python $EXE > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py - python $PLOT_DIR/plot_libe_runs_util_v_time.py + python $PLOT_DIR/plot_libe_runs_util_v_time.py python $PLOT_DIR/plot_libE_histogram.py fi diff --git a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt index 9950cfc5c..82d4865ee 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt @@ -16,7 +16,7 @@ Build the C program: Either run on local node - or create an allocation of nodes and run:: python killtest.py - + where kill_type currently is 1 or 2. [1. is the original kill - 2. is using group ID approach] E.g Single node with 4 processes: @@ -65,8 +65,8 @@ Cooley (intelmpi):: Two nodes with 2 processes each: - kill 1: - kill 2: + kill 1: + kill 2: Theta (intelmpi):: @@ -79,18 +79,15 @@ Theta (intelmpi):: kill 1: Fails (Works with 'module unload xalt') kill 2: Launch does not work - + Kill type 2: Detaches subprocess from controlling terminal - but ALPS uses PID to keep track of things so cannot do this on Theta. - + Solution: Module unload xalt - + Now kill type 1 works (presumably supported by ALPS system). xalt creates a wrapper around aprun - and kill just kills the wrapping process. - + Alternative - may be to use apkill. This may work without unloading xalt - not tried. - - - diff --git a/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt b/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt index 5b55d0487..0ebc1535f 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt @@ -10,7 +10,7 @@ Instructions Either run on local node - or create an allocation of nodes and run:: python killtest.py - + where kill_type currently is 1 or 2. [1. is the original kill - 2. is using group ID approach] Then observe "top" on target nodes for burn_time.x processes. @@ -52,11 +52,11 @@ Cooley (intelmpi):: Theta (intelmpi):: - kill 1: - kill 2: - + kill 1: + kill 2: + + - 2018-07-02: Two nodes with 4 processes per node: @@ -72,17 +72,17 @@ Bebop (intelmpi):: Cooley (intelmpi):: - kill 1: - kill 2: + kill 1: + kill 2: Theta (intelmpi):: - kill 1: - kill 2: - - - -Example: + kill 1: + kill 2: + + + +Example: --------------------------------------------------------------------- Running on Cooley - for my dir setup (maybe should store in project space). @@ -97,7 +97,7 @@ Make sure got intel/mpi modules loaded. Then run: ./build.sh python killtest.py 1 1 4 #observe with top in another shell on that node (see below) - python killtest.py 2 1 4 #observe with top in another shell on that node (see below) + python killtest.py 2 1 4 #observe with top in another shell on that node (see below) If processes dont stop: diff --git a/libensemble/tests/standalone_tests/kill_test/readme.txt b/libensemble/tests/standalone_tests/kill_test/readme.txt index 3c1382272..a68623a49 100644 --- a/libensemble/tests/standalone_tests/kill_test/readme.txt +++ b/libensemble/tests/standalone_tests/kill_test/readme.txt @@ -22,7 +22,7 @@ Build the C program: Either run on local node - or create an allocation of nodes and run:: python killtest.py - + where kill_type currently is 1 or 2. [1. is the original kill - 2. is using group ID approach] E.g Single node with 4 processes: diff --git a/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt b/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt index 222c2d984..1a853b11d 100644 --- a/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt +++ b/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt @@ -5,7 +5,7 @@ You specify the size of the outer jobs and inner jobs as follows: mpirun -np 16 python create_mpi_jobs.py 4 runs a 16 way job, each launching a 4 processor python hello_world with a short sleep. - + This is a good test to run to ensure basic functioning on a system, including nested launching. The test should create an output file (job_N.out) for each outer rank. A hello_world line is printed in these for each inner rank. diff --git a/libensemble/tests/unit_tests/.coveragerc b/libensemble/tests/unit_tests/.coveragerc index 434d58279..856148509 100644 --- a/libensemble/tests/unit_tests/.coveragerc +++ b/libensemble/tests/unit_tests/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out #parallel = true @@ -21,9 +21,9 @@ omit = */unit_tests/* */regression_tests/* -exclude_lines = +exclude_lines = if __name__ == .__main__.: comm.Abort() -[html] +[html] directory = cov_unit diff --git a/libensemble/tests/unit_tests_logger/.coveragerc b/libensemble/tests/unit_tests_logger/.coveragerc index 0a22a922f..d09401055 100644 --- a/libensemble/tests/unit_tests_logger/.coveragerc +++ b/libensemble/tests/unit_tests_logger/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out3 #parallel = true @@ -22,9 +22,9 @@ omit = */unit_tests_nompi/* */regression_tests/* -exclude_lines = +exclude_lines = if __name__ == .__main__.: comm.Abort() -[html] +[html] directory = cov_unit diff --git a/libensemble/tests/unit_tests_nompi/.coveragerc b/libensemble/tests/unit_tests_nompi/.coveragerc index 2c9a3a53d..a919317be 100644 --- a/libensemble/tests/unit_tests_nompi/.coveragerc +++ b/libensemble/tests/unit_tests_nompi/.coveragerc @@ -8,7 +8,7 @@ omit = */regression_tests/* branch = true - + data_file = .cov_unit_out2 #parallel = true @@ -22,9 +22,9 @@ omit = */unit_tests_nompi/* */regression_tests/* -exclude_lines = +exclude_lines = if __name__ == .__main__.: comm.Abort() -[html] +[html] directory = cov_unit diff --git a/postproc_scripts/compare_npy.py b/postproc_scripts/compare_npy.py index 0489ff512..c408552ce 100755 --- a/postproc_scripts/compare_npy.py +++ b/postproc_scripts/compare_npy.py @@ -3,7 +3,7 @@ '''Script to compare libEnsemble history arrays in files. If two *.npy files are provided they are compared with each other. -If one *.npy file is provided if is compared with a hard-coded expected file +If one *.npy file is provided if is compared with a hard-coded expected file (by default located at ../expected.npy) Default NumPy tolerances are used for comparison (rtol=1e-05, atol=1e-08) and Nans compare as equal. Variable fields (such as those containing a time) @@ -19,7 +19,7 @@ elif len(sys.argv) > 1: results = np.load(sys.argv[1]) exp_results_file = "../expected.npy" - exp_results = np.load(exp_results_file) + exp_results = np.load(exp_results_file) else: print('You need to supply an .npy file - aborting') sys.exit() diff --git a/postproc_scripts/plot_libE_histogram.py b/postproc_scripts/plot_libE_histogram.py index dd71ca57a..020196523 100755 --- a/postproc_scripts/plot_libE_histogram.py +++ b/postproc_scripts/plot_libE_histogram.py @@ -67,7 +67,7 @@ def append_to_list(mylst, glob_list, found_time): append_to_list(in_times_kill,in_times,found_time) # Assumes Time comes first elif search_for_keyword(lst[i+1:len(lst)], run_exception): exceptions = True - append_to_list(in_times_exception,in_times,found_time) # Assumes Time comes first + append_to_list(in_times_exception,in_times,found_time) # Assumes Time comes first else: print('Error: Unkown status - rest of line: {}'.format(lst[i+1:len(lst)])) sys.exit() @@ -98,8 +98,8 @@ def append_to_list(mylst, glob_list, found_time): calc_type = 'sim' else: calc_type = 'calc' - -title = 'libEnsemble histogram of ' + calc_type + ' times' + ' (' + str(active_line_count) + ' user calcs)' + str(num_bins) + ' bins' + +title = 'libEnsemble histogram of ' + calc_type + ' times' + ' (' + str(active_line_count) + ' user calcs)' + str(num_bins) + ' bins' plt.title(title) plt.xlabel('Calc run-time (sec)') diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index 1bed83f25..ec861b81c 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -4,7 +4,7 @@ Timing analysis scripts Note that all plotting scripts produce a file rather than opening a plot interactively. -The following scripts must be run in the directory with the **libE_stats.txt** file. They extract and plot information from that file. +The following scripts must be run in the directory with the **libE_stats.txt** file. They extract and plot information from that file. * **plot_libe_calcs_util_v_time.py**: Extract worker utilization v time plot (with one second sampling). Shows number of workers running user sim or gen functions over time. From c166db5f2a55f2f90c902db68be2bcd8d7f404f8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 26 Sep 2019 13:24:00 -0500 Subject: [PATCH 288/644] Removing blank lines at the end --- conda/environment.yml | 1 - docs/conf.py | 3 --- docs/data_structures/persis_info.rst | 1 - docs/data_structures/work_dict.rst | 1 - docs/data_structures/worker_array.rst | 1 - docs/dev_guide/dev_API/worker_module.rst | 1 - .../release_management/release_platforms/rel_pypi.rst | 3 --- .../release_management/release_platforms/rel_spack.rst | 1 - docs/job_controller/job_controller.rst | 1 - docs/job_controller/mpi_controller.rst | 4 ---- docs/job_controller/overview.rst | 1 - docs/sim_gen_alloc_funcs.rst | 1 - .../tests/balsam_tests/bash_scripts/setup_balsam_tests.sh | 1 - libensemble/tests/balsam_tests/readme.balsam_tests.txt | 1 - libensemble/tests/balsam_tests/readme.rst | 2 -- libensemble/tests/controller_tests/readme.txt | 4 ---- libensemble/tests/scaling_tests/forces/clone.sh | 1 - libensemble/tests/standalone_tests/kill_test/build.sh | 1 - .../kill_test/log.burn_time_test_with_top.txt | 1 - libensemble/tests/standalone_tests/kill_test/readme.txt | 2 -- 20 files changed, 32 deletions(-) diff --git a/conda/environment.yml b/conda/environment.yml index f79b595ff..57e16c744 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,3 +1,2 @@ dependencies: - python>=3.6 - diff --git a/docs/conf.py b/docs/conf.py index 0becb45d4..d4fc3b2f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -209,6 +209,3 @@ def __getattr__(cls, name): author, 'libEnsemble', 'One line description of project.', 'Miscellaneous'), ] - - - diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 3f5bfbbd2..3f1a6e6ce 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -24,4 +24,3 @@ From: libEnsemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MAA.py:: for i in range(MPI.COMM_WORLD.Get_size()): persis_info[i] = {'rand_stream': np.random.RandomState(i)} # used as a random number stream for each worker - diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 6e4bc7ec6..4c1310b34 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -37,4 +37,3 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` | | For allocation functions giving work that blocks other workers, see | ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` - diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index 88420549e..e378caaf8 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -24,4 +24,3 @@ worker blocked by some other calculation 1 0 1 * libE only receives from workers with 'active' nonzero * libE only calls the alloc_f if some worker has 'active' zero - diff --git a/docs/dev_guide/dev_API/worker_module.rst b/docs/dev_guide/dev_API/worker_module.rst index 79a27b1f8..1b68af668 100644 --- a/docs/dev_guide/dev_API/worker_module.rst +++ b/docs/dev_guide/dev_API/worker_module.rst @@ -8,4 +8,3 @@ Worker Module :members: .. automethod:: __init__ - diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst index 3cbdd32f3..a299dca6a 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -32,6 +32,3 @@ It should also be visible here: https://pypi.org/project/libensemble/ For more details on creating PyPI packages see https://betterscientificsoftware.github.io/python-for-hpc/tutorials/python-pypi-packaging/ - - - diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index db058de05..7191abcfd 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -130,4 +130,3 @@ Quick summary for bringing develop branch on forked repo up to speed with upstre git push origin develop --force Reference: - diff --git a/docs/job_controller/job_controller.rst b/docs/job_controller/job_controller.rst index a2a312e9b..9b0a891b0 100644 --- a/docs/job_controller/job_controller.rst +++ b/docs/job_controller/job_controller.rst @@ -58,4 +58,3 @@ Run configuration attributes - Some will be auto-generated: :job.stderr: (string) Name of file where the standard error of the job is written (in job.workdir) A list of job_controller and job functions can be found under the Job Controller Module. - diff --git a/docs/job_controller/mpi_controller.rst b/docs/job_controller/mpi_controller.rst index 117ea8293..559a13a57 100644 --- a/docs/job_controller/mpi_controller.rst +++ b/docs/job_controller/mpi_controller.rst @@ -35,7 +35,3 @@ Example. To increase resilience against launch failures:: jobctrl.fail_time = 5 Note that an the re-try delay on launches starts at 5 seconds and increments by 5 seconds for each retry. So the 4th re-try will wait for 20 seconds before re-launching. - - - - diff --git a/docs/job_controller/overview.rst b/docs/job_controller/overview.rst index 95b87bc53..fa3c63748 100644 --- a/docs/job_controller/overview.rst +++ b/docs/job_controller/overview.rst @@ -92,4 +92,3 @@ For a more realistic example see: - libensemble/tests/scaling_tests/forces/ which launches the forces.x application as an MPI job. - diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index 6f253d24a..7e56753ba 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -124,4 +124,3 @@ Returns: **persis_info**: :obj:`dict` :doc:`(example)` - diff --git a/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh b/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh index c24234f88..c05e8f802 100755 --- a/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh +++ b/libensemble/tests/balsam_tests/bash_scripts/setup_balsam_tests.sh @@ -59,4 +59,3 @@ balsam ls jobs #Run launcher in either interactive session or via script echo -e "\nTo launch jobs run: balsam launcher --consume-all" - diff --git a/libensemble/tests/balsam_tests/readme.balsam_tests.txt b/libensemble/tests/balsam_tests/readme.balsam_tests.txt index 7e1e51970..fd19a9616 100644 --- a/libensemble/tests/balsam_tests/readme.balsam_tests.txt +++ b/libensemble/tests/balsam_tests/readme.balsam_tests.txt @@ -10,4 +10,3 @@ test_balsam_3__managerkill.py: Process 0 sends out a kill for jobs which include sim_id_1 in their name. This will kill any such jobs in the database at the time of the kill. Note that if a job has not yet been added to database at the time of the kill, it will still run. Note that test3 exploits the fact that balsam is built on a django database, and all django functionality for manipulating the database can easily be exposed. - diff --git a/libensemble/tests/balsam_tests/readme.rst b/libensemble/tests/balsam_tests/readme.rst index 6a309984b..8b4461347 100644 --- a/libensemble/tests/balsam_tests/readme.rst +++ b/libensemble/tests/balsam_tests/readme.rst @@ -229,5 +229,3 @@ To see time remaining: .. code-block:: bash qstat -fu - - diff --git a/libensemble/tests/controller_tests/readme.txt b/libensemble/tests/controller_tests/readme.txt index d7212edce..ef7459e8d 100644 --- a/libensemble/tests/controller_tests/readme.txt +++ b/libensemble/tests/controller_tests/readme.txt @@ -47,7 +47,3 @@ and: > balsam launcher --consume-all Note: On systems like Theta only one job can be launched per node. You will need at least 2 nodes. The first will run the parent script. The test test_jobcontroller_multi.py launches 3 sub-jobs which Balsam will launch as nodes become available. To run all 3 concurrently would require 4 nodes. - - - - diff --git a/libensemble/tests/scaling_tests/forces/clone.sh b/libensemble/tests/scaling_tests/forces/clone.sh index a0f491179..5ce6e0ba7 100644 --- a/libensemble/tests/scaling_tests/forces/clone.sh +++ b/libensemble/tests/scaling_tests/forces/clone.sh @@ -10,4 +10,3 @@ else # Take you there if source script. mkdir ../$NEWDIR && cp -rp $INPUTS ../$NEWDIR && cd ../$NEWDIR fi - diff --git a/libensemble/tests/standalone_tests/kill_test/build.sh b/libensemble/tests/standalone_tests/kill_test/build.sh index e638f0e00..8e47571e8 100755 --- a/libensemble/tests/standalone_tests/kill_test/build.sh +++ b/libensemble/tests/standalone_tests/kill_test/build.sh @@ -1,3 +1,2 @@ mpicc -g -o burn_time.x burn_time.c mpicc -g -o sleep_and_print.x sleep_and_print.c - diff --git a/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt b/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt index 0ebc1535f..8c16ea60a 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.burn_time_test_with_top.txt @@ -107,4 +107,3 @@ In second shell. Log in to whatever qsub node you got: top Four burn_time.x processes should appear, then go, then come, then go. - diff --git a/libensemble/tests/standalone_tests/kill_test/readme.txt b/libensemble/tests/standalone_tests/kill_test/readme.txt index a68623a49..1ab630b12 100644 --- a/libensemble/tests/standalone_tests/kill_test/readme.txt +++ b/libensemble/tests/standalone_tests/kill_test/readme.txt @@ -40,5 +40,3 @@ kill 2: python killtest.py 2 2 2 If the test fails, an assertion error will occur. Output files are produced out_0.txt and out_1.txt (one for each job). These can be deleted between runs. - - From e86709f99abd92a64f0848d15bd543c4236c0dac Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 26 Sep 2019 13:33:04 -0500 Subject: [PATCH 289/644] Why not do the same with f90 and c files --- .../tests/controller_tests/simdir/my_simjob.c | 18 +-- .../controller_tests/simdir/my_simjob.f90 | 20 ++-- .../tests/scaling_tests/forces/forces.c | 112 +++++++++--------- .../standalone_tests/kill_test/burn_time.c | 6 +- .../kill_test/sleep_and_print.c | 22 ++-- .../tests/unit_tests/simdir/my_simjob.c | 22 ++-- .../tests/unit_tests/simdir/my_simjob.f90 | 20 ++-- 7 files changed, 110 insertions(+), 110 deletions(-) diff --git a/libensemble/tests/controller_tests/simdir/my_simjob.c b/libensemble/tests/controller_tests/simdir/my_simjob.c index 86fa6225d..94acb2c39 100644 --- a/libensemble/tests/controller_tests/simdir/my_simjob.c +++ b/libensemble/tests/controller_tests/simdir/my_simjob.c @@ -4,13 +4,13 @@ #include #include -int main(int argc, char **argv) +int main(int argc, char **argv) { int ierr, num_procs, rank, delay, error; - + delay=3; error=0; - + if (argc >=3) { if (strcmp( argv[1],"sleep") == 0 ) { delay = atoi(argv[2]); @@ -20,23 +20,23 @@ int main(int argc, char **argv) if (strcmp( argv[3],"Error") == 0 ) { error=1; } - } + } ierr = MPI_Init(&argc, &argv); ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); ierr = MPI_Comm_size(MPI_COMM_WORLD, &num_procs); - + printf("Hello world sleeping for %d seconds on rank %d of %d\n",delay, rank,num_procs); sleep(delay); - //printf("Woken up on rank %d of %d\n",rank,num_procs); - + //printf("Woken up on rank %d of %d\n",rank,num_procs); + if (rank==0) { if (error==1) { printf("Oh Dear! An non-fatal Error seems to have occured on rank %d\n",rank); sleep(delay); } } - - ierr = MPI_Finalize(); + + ierr = MPI_Finalize(); return(0); } diff --git a/libensemble/tests/controller_tests/simdir/my_simjob.f90 b/libensemble/tests/controller_tests/simdir/my_simjob.f90 index 1f4d2c0c6..1a1a631ad 100644 --- a/libensemble/tests/controller_tests/simdir/my_simjob.f90 +++ b/libensemble/tests/controller_tests/simdir/my_simjob.f90 @@ -17,21 +17,21 @@ program hello_fortran use mpi use, intrinsic :: iso_c_binding, only: c_int use Fortran_Sleep - + implicit none - + integer (kind = 4) :: err integer (kind = 4) :: rank integer (kind = 4) :: nprocs character(len=32) :: arg - character(len=32) :: arglist(2) + character(len=32) :: arglist(2) integer (c_int) :: delay,dummy integer :: i - + delay=5 dummy = FortSleep(delay) - - + + !init in case get no args DO i = 1, 2 arglist(i) = "None" @@ -41,15 +41,15 @@ program hello_fortran CALL getarg(i, arg) arglist(i) = arg END DO - + call mpi_init(err) call mpi_comm_size (MPI_COMM_WORLD, nprocs, err) call mpi_comm_rank (MPI_COMM_WORLD, rank, err) - - + + write(*,100) rank, nprocs, trim(arglist(1)), trim(arglist(2)) - + call MPI_Finalize (err) 100 FORMAT('Hello World from proc: ',i3,' of',i3,' arg1: ',a,' arg2: ',a) diff --git a/libensemble/tests/scaling_tests/forces/forces.c b/libensemble/tests/scaling_tests/forces/forces.c index 1aebfa44b..827b7973d 100755 --- a/libensemble/tests/scaling_tests/forces/forces.c +++ b/libensemble/tests/scaling_tests/forces/forces.c @@ -2,16 +2,16 @@ Naive Electostatics Code Example This is designed only as an artificial, highly conifurable test code for a libEnsemble sim func. - + Particles position and charge are initiated by a random stream. - Particles are replicated on all ranks. + Particles are replicated on all ranks. Each rank computes forces for a subset of particles. Particle force arrays are allreduced across ranks. - + Run executable on N procs: - + mpirun -np N ./forces.x - + Author: S Hudson. -------------------------------------------------------------------- */ @@ -36,20 +36,20 @@ typedef struct particle { double p[3]; // Particle position double f[3]; // Particle force double q; // Particle charge -} particle; +} particle; // Seed RNG int seed_rand(int seed) { - srand(seed); + srand(seed); return 0; } // Return a random number from a persistent stream //TODO Use parallel RNG - As replicated data can currently do on master rank. double get_rand() { - double randnum; - randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) return randnum; } @@ -60,7 +60,7 @@ int build_system(int n, particle* parr) { int q_range_high = 10; double extent = 10.0; int i, dim; - + for(i=0; i=2) { num_particles = atoi(argv[1]); // no. of particles } - + if (argc >=3) { num_steps = atoi(argv[2]); // no. of timesteps } - + if (argc >=4) { rand_seed = atoi(argv[3]); // RNG seed seed_rand(rand_seed); } - + if (argc >=5) { kill_rate = atof(argv[4]); // Proportion of jobs to kill step_survival_rate = pow((1-kill_rate),(1.0/num_steps)); } - + particle* parr = malloc(num_particles * sizeof(particle)); build_system(num_particles, parr); //printf("\n"); - + ierr = MPI_Init(&argc, &argv); ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); ierr = MPI_Comm_size(MPI_COMM_WORLD, &num_procs); - + if (rank == 0) { printf("Particles: %d\n",num_particles); printf("Timesteps: %d\n",num_steps); printf("MPI Ranks: %d\n",num_procs); printf("Random seed: %d\n",rand_seed); } - + k = num_particles / num_procs; m = num_particles % num_procs; //Remainder = no. procs with extra particle p_lower = rank * k + min(rank, m); p_upper = (rank + 1) * k + min(rank + 1, m); local_n = p_upper - p_lower; - + if (PRINT_PARTICLE_DECOMP) { MPI_Barrier(MPI_COMM_WORLD); printf("Proc: %d has %d particles\n", rank, local_n); } MPI_Barrier(MPI_COMM_WORLD); - fflush(stdout); + fflush(stdout); if (rank == 0) { open_stat_file(); } - start = clock(); + start = clock(); for (step=0; step #include -int main(int argc, char **argv) +int main(int argc, char **argv) { int ierr, num_procs, rank, usec_delay, error; double fdelay; - + fdelay=3.0; error=0; - + if (argc >=3) { if (strcmp( argv[1],"sleep") == 0 ) { fdelay = atof(argv[2]); @@ -21,22 +21,22 @@ int main(int argc, char **argv) if (strcmp( argv[3],"Error") == 0 ) { error=1; } - } + } if (argc >=4) { if (strcmp( argv[3],"Fail") == 0 ) { return(1); } - } + } ierr = MPI_Init(&argc, &argv); ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); ierr = MPI_Comm_size(MPI_COMM_WORLD, &num_procs); - + printf("Hello world sleeping for %f seconds on rank %d of %d\n",fdelay, rank,num_procs); - + usec_delay = (int)(fdelay*1e6); usleep(usec_delay); - //printf("Woken up on rank %d of %d\n",rank,num_procs); - + //printf("Woken up on rank %d of %d\n",rank,num_procs); + if (rank==0) { if (error==1) { printf("Oh Dear! An non-fatal Error seems to have occured on rank %d\n",rank); @@ -44,7 +44,7 @@ int main(int argc, char **argv) usleep(usec_delay); } } - - ierr = MPI_Finalize(); + + ierr = MPI_Finalize(); return(0); } diff --git a/libensemble/tests/unit_tests/simdir/my_simjob.f90 b/libensemble/tests/unit_tests/simdir/my_simjob.f90 index 1f4d2c0c6..1a1a631ad 100644 --- a/libensemble/tests/unit_tests/simdir/my_simjob.f90 +++ b/libensemble/tests/unit_tests/simdir/my_simjob.f90 @@ -17,21 +17,21 @@ program hello_fortran use mpi use, intrinsic :: iso_c_binding, only: c_int use Fortran_Sleep - + implicit none - + integer (kind = 4) :: err integer (kind = 4) :: rank integer (kind = 4) :: nprocs character(len=32) :: arg - character(len=32) :: arglist(2) + character(len=32) :: arglist(2) integer (c_int) :: delay,dummy integer :: i - + delay=5 dummy = FortSleep(delay) - - + + !init in case get no args DO i = 1, 2 arglist(i) = "None" @@ -41,15 +41,15 @@ program hello_fortran CALL getarg(i, arg) arglist(i) = arg END DO - + call mpi_init(err) call mpi_comm_size (MPI_COMM_WORLD, nprocs, err) call mpi_comm_rank (MPI_COMM_WORLD, rank, err) - - + + write(*,100) rank, nprocs, trim(arglist(1)), trim(arglist(2)) - + call MPI_Finalize (err) 100 FORMAT('Hello World from proc: ',i3,' of',i3,' arg1: ',a,' arg2: ',a) From d961726a18643f07c6465a6bcf96aa23a9bc669f Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 26 Sep 2019 13:51:46 -0500 Subject: [PATCH 290/644] move omission from report to run --- libensemble/tests/regression_tests/.coveragerc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index 525564f11..090c7ad5d 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -7,6 +7,8 @@ omit = */unit_tests_nompi/* */regression_tests/* */branin_*/* + balsam_controller.py + branch = true @@ -22,7 +24,6 @@ omit = */unit_tests/* */unit_tests_nompi/* */regression_tests/* - balsam_controller.py exclude_lines = From e384df681b6a3792cff1336faf3d1b9460fd1541 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 26 Sep 2019 16:45:57 -0500 Subject: [PATCH 291/644] Try moving omission to /tests rc. Adds conditional omission on Balsam installation --- conda/install-balsam.py | 12 ++++++++++++ libensemble/tests/.coveragerc | 3 ++- libensemble/tests/regression_tests/.coveragerc | 3 --- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 853c84f8e..8d38e816e 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -26,8 +26,20 @@ def move_test_balsam(balsam_test): if not os.path.isfile(reg_dir_with_btest): os.rename('./conda/{}'.format(balsam_test), reg_dir_with_btest) +def configure_coverage(): + # Enables coverage of balsam_controller.py if running test + coveragerc = './libensemble/tests/.coveragerc' + with open(coveragerc, 'r') as f: + lines = f.readlines() + + newlines = [i for i in lines if i != ' */balsam_controller.py\n'] + + with open(coveragerc, 'w') as f: + for line in newlines: + f.write(line) if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ install_balsam() move_test_balsam('test_balsam_hworld.py') + configure_coverage() subprocess.run('./conda/configure-balsam-test.sh'.split()) diff --git a/libensemble/tests/.coveragerc b/libensemble/tests/.coveragerc index fa2110df3..0888f1435 100644 --- a/libensemble/tests/.coveragerc +++ b/libensemble/tests/.coveragerc @@ -4,7 +4,7 @@ [run] data_file = .cov_merge_out -[html] +[html] directory = cov_merge #Report files can be controlled here @@ -20,6 +20,7 @@ omit = */unit_tests_logger/* */regression_tests/* */sim_funcs/helloworld.py + */balsam_controller.py # */sim_funcs/* # */gen_funcs/* exclude_lines = diff --git a/libensemble/tests/regression_tests/.coveragerc b/libensemble/tests/regression_tests/.coveragerc index 090c7ad5d..25dfd8d59 100644 --- a/libensemble/tests/regression_tests/.coveragerc +++ b/libensemble/tests/regression_tests/.coveragerc @@ -7,8 +7,6 @@ omit = */unit_tests_nompi/* */regression_tests/* */branin_*/* - balsam_controller.py - branch = true @@ -25,7 +23,6 @@ omit = */unit_tests_nompi/* */regression_tests/* - exclude_lines = if __name__ == .__main__.: From 796fe2ca38d8019278432d54837dacf2ad9a0550 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 27 Sep 2019 08:35:55 -0500 Subject: [PATCH 292/644] Starting readthedocs editing branch --- CONTRIBUTING.rst | 4 ++-- README.rst | 2 +- .../release_platforms/rel_github.rst | 20 +++++++++++++------ .../release_platforms/rel_pypi.rst | 19 ++++++++++++------ .../release_platforms/rel_spack.rst | 8 ++++---- .../release_management/release_process.rst | 11 ++++++---- 6 files changed, 41 insertions(+), 23 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ad252cb8d..8145b0009 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,7 +1,7 @@ Contributing ============ -Contributions may be made via Github pull request to: +Contributions may be made via GitHub pull request to: https://github.com/Libensemble/libensemble @@ -20,7 +20,7 @@ will add issues, as appropriate, to the project board at: https://github.com/Libensemble/libensemble/projects By convention, user branch names should have a / format, where -example types are feature, bugfix, testing, docs and experimental. +example types are feature, bugfix, testing, docs, and experimental. Administrators may take a hotfix branch from the the master, which will be merged into master (as a patch) and develop. Administrators may also take a release branch off develop and merge into master and develop for a release. diff --git a/README.rst b/README.rst index cb73848dc..94fcca7d1 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,7 @@ libEnsemble is also available in the Spack_ distribution. It can be installed fr .. _Spack: https://spack.readthedocs.io/en/latest -The tests and examples can be accessed in the `github `_ repository. +The tests and examples can be accessed in the `GitHub `_ repository. A `tarball `_ of the most recent release is also available. diff --git a/docs/dev_guide/release_management/release_platforms/rel_github.rst b/docs/dev_guide/release_management/release_platforms/rel_github.rst index 8e59107f8..da345b173 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_github.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_github.rst @@ -1,18 +1,26 @@ .. _rel-github: -Github release +GitHub release ============== -The administrator should follow the github instructions to draft a new release. These can currently be found at: https://help.github.com/en/articles/creating-releases +The administrator should follow the GitHub instructions to draft a new release. +These can currently be found at: +https://help.github.com/en/articles/creating-releases Both the version and title will be of the form vX.Y.Z:: E.g. v0.5.0. -From version 1.0, these should follow semantic versioning where, where X/Y/Z are major, minor and patch revisions. +From version 1.0, these should follow semantic versioning where, where X/Y/Z +are major, minor and patch revisions. -Prior to version 1.0, the second number may include breaking API changes, and the third number may include minor additions. +Prior to version 1.0, the second number may include breaking API changes, and +the third number may include minor additions. -The release notes should be included in the description. These should already be in `docs/release_notes.rst`. The release nots should be copied just the current release, starting from the date. Hint: To see example of raw input click *edit* next to one of the previous releases. +The release notes should be included in the description. These should already +be in `docs/release_notes.rst`. The release nots should be copied just the +current release, starting from the date. Hint: To see example of raw input +click *edit* next to one of the previous releases. -Note, unlike some platforms (e.g. PyPI), github releases can be edited or deleted once created. +Note, unlike some platforms (e.g. PyPI), GitHub releases can be edited or +deleted once created. diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst index a299dca6a..e660f6c13 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -3,15 +3,20 @@ PyPI release ============ -libEnsemble is released on the Python Package Index (commonly known as PyPI). This enable users to “pip install” the package. +libEnsemble is released on the Python Package Index (commonly known as PyPI). +This enables users to "``pip install``" the package. -The package is stored on PyPI in the form of a source distribution (commonly known as a tarball). The tarball could be obtained from github, though historically this has been created with a checkout of libensemble from git. +The package is stored on PyPI in the form of a source distribution (commonly +known as a tarball). The tarball could be obtained from GitHub, though +historically this has been created with a checkout of libensemble from git. -You will need logon credentials for the libEnsemble PyPI. You will also need twine (which can be pip or conda installed). +You will need logon credentials for the libEnsemble PyPI. You will also need +twine (which can be pip or conda installed). -In the package directory on the master branch (the one containing setup.py) do the following: +In the package directory on the master branch (the one containing setup.py) do +the following: Create distribution: @@ -24,11 +29,13 @@ Upload (you will need username/password here): twine upload dist/* -If you now do "pip install libensemble" it should find the new version. +If you now do "``pip install libensemble``" it should find the new version. It should also be visible here: https://pypi.org/project/libensemble/ -For more details on creating PyPI packages see https://betterscientificsoftware.github.io/python-for-hpc/tutorials/python-pypi-packaging/ +For more details on creating PyPI packages see:: + +https://betterscientificsoftware.github.io/python-for-hpc/tutorials/python-pypi-packaging/ diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index 7191abcfd..851bc24f8 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -9,13 +9,13 @@ Introduction This assumes you have already: - Made a PyPI package for new version of libensemble - - Made a github fork of Spack and cloned it to your local system. + - Made a GitHub fork of Spack and cloned it to your local system. Details on how to create forks can be found at: https://help.github.com/articles/fork-a-repo You now have a configuration like shown in answer at: https://stackoverflow.com/questions/6286571/are-git-forks-actually-git-clones -Upstream, in this case, is the official Spack repository on github. Origin is your fork on github and Local Machine is your local clone (from your fork). +Upstream, in this case, is the official Spack repository on GitHub. Origin is your fork on GitHub and Local Machine is your local clone (from your fork). Make sure SPACK_ROOT is set and spack binary is in your path:: @@ -67,7 +67,7 @@ Or to make your local machine identical to upstream repo (**WARNING** Any local git reset --hard upstream/develop -(Optional) You may want to update your forked (origin) repo on github at this point. +(Optional) You may want to update your forked (origin) repo on GitHub at this point. This may requires a forced push:: git push origin develop --force @@ -113,7 +113,7 @@ If OK add, commit and push to origin (forked repo):: git commit -am "Update libensemble" git push origin update_libensemble --force -Once the branch is pushed to the forked repo - go to github and do a pull request from this +Once the branch is pushed to the forked repo - go to GitHub and do a pull request from this branch on the fork to the develop branch on the upstream. diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index 48475aa87..a26c36b7d 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -1,12 +1,15 @@ Release Process =============== -This document details the current release process for libEnsemble. A release can only be undertaken by a project administrator. A project administrator should have an administrator role on the libEnsemble github, PyPI and readthedocs pages. +This document details the current release process for libEnsemble. A release +can only be undertaken by a project administrator. A project administrator +should have an administrator role on the libEnsemble GitHub, PyPI, and +readthedocs pages. Before release -------------- -- A github issue is created with a checklist for the release. +- A GitHub issue is created with a checklist for the release. - A release branch should be taken off develop (or develop pulls controlled). @@ -43,7 +46,7 @@ An administrator will take the following steps. - Once CI tests have passed on master. - - A github release will be taken from the master (:ref:`github release`). + - A GitHub release will be taken from the master (:ref:`github release`). - A tarball (source distribution) will be uploaded to PyPI (:ref:`PyPI release`). @@ -56,4 +59,4 @@ An administrator will take the following steps. After release ------------- -- Ensure all relevant github issues are closed and moved to the *Done* column on the kanban project board (inc. the release checklist). +- Ensure all relevant GitHub issues are closed and moved to the *Done* column on the kanban project board (inc. the release checklist). From 9e5c3d948261d6c9e01ae3a806befad39cb679ab Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 27 Sep 2019 10:27:02 -0500 Subject: [PATCH 293/644] Clarify and update macOS Security Prompts --- docs/FAQ.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index a535a7655..7ece89094 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -142,18 +142,18 @@ error. macOS - Firewall prompts ------------------------ -**macOS - System constantly prompts Firewall Security Permission windows throughout execution** +**macOS - Constant Firewall Security permission windows throughout Job Controller task** -There are several ways to address this nuisance. One easy (but insecure) solution is -temporarily disabling the Firewall through System Preferences -> Security & Privacy --> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming -connections" rule can be tried for the offending Python installations, -but this may not prevent the prompts and only clear them shortly after appearing. -Finally, `Signing your Python installation with a self-signed certificate`_ may -be effective. +There are several ways to address this nuisance, but all involve trial and error. +One easy (but insecure) solution is temporarily disabling the Firewall through System Preferences +-> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming +connections" rule can be tried for the offending Job Controller executable. +Based on a suggestion from here here_, we've had the most success running +``sudo codesign --force --deep --sign - /path/to/application.app`` on our Job Controller executables, +then confirming the next alerts for the executable and ``mpiexec.hydra``. -.. _`Signing your Python installation with a self-signed certificate`: https://coderwall.com/p/5b_apq/stop-mac-os-x-firewall-from-prompting-with-python-in-virtualenv +.. _`here`: https://coderwall.com/p/5b_apq/stop-mac-os-x-firewall-from-prompting-with-python-in-virtualenv Running out of contexts when running libEnsemble in distributed mode on TMI fabric From eecc30ec6c3091436c2cf9df46784aee02aebbf2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 27 Sep 2019 10:42:31 -0500 Subject: [PATCH 294/644] extra here --- docs/FAQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 7ece89094..84ebef677 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -149,7 +149,7 @@ There are several ways to address this nuisance, but all involve trial and error One easy (but insecure) solution is temporarily disabling the Firewall through System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming connections" rule can be tried for the offending Job Controller executable. -Based on a suggestion from here here_, we've had the most success running +Based on a suggestion from here_, we've had the most success running ``sudo codesign --force --deep --sign - /path/to/application.app`` on our Job Controller executables, then confirming the next alerts for the executable and ``mpiexec.hydra``. From d3fd0611b5f326e62fe8f9113c444061c50951e4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 27 Sep 2019 10:56:34 -0500 Subject: [PATCH 295/644] Some edits --- .../release_platforms/rel_github.rst | 6 ++-- .../release_platforms/rel_pypi.rst | 8 ++--- .../release_platforms/rel_spack.rst | 13 ++++---- docs/quickstart.rst | 30 +++++++++++-------- libensemble/tests/balsam_tests/readme.rst | 8 ++--- libensemble/tests/run-tests.sh | 4 +-- .../tests/scaling_tests/forces/readme.md | 10 ++++++- .../standalone_tests/kill_test/readme.txt | 23 ++++++++++---- .../mpi_launch_test/readme.txt | 16 ++++++---- 9 files changed, 72 insertions(+), 46 deletions(-) diff --git a/docs/dev_guide/release_management/release_platforms/rel_github.rst b/docs/dev_guide/release_management/release_platforms/rel_github.rst index da345b173..f4173aac3 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_github.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_github.rst @@ -7,9 +7,7 @@ The administrator should follow the GitHub instructions to draft a new release. These can currently be found at: https://help.github.com/en/articles/creating-releases -Both the version and title will be of the form vX.Y.Z:: - - E.g. v0.5.0. +Both the version and title will be of the form vX.Y.Z, for example, v0.5.0. From version 1.0, these should follow semantic versioning where, where X/Y/Z are major, minor and patch revisions. @@ -18,7 +16,7 @@ Prior to version 1.0, the second number may include breaking API changes, and the third number may include minor additions. The release notes should be included in the description. These should already -be in `docs/release_notes.rst`. The release nots should be copied just the +be in ``docs/release_notes.rst``. The release nots should be copied just the current release, starting from the date. Hint: To see example of raw input click *edit* next to one of the previous releases. diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst index e660f6c13..98f1b6f84 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -8,7 +8,7 @@ This enables users to "``pip install``" the package. The package is stored on PyPI in the form of a source distribution (commonly known as a tarball). The tarball could be obtained from GitHub, though -historically this has been created with a checkout of libensemble from git. +historically this has been created with a checkout of libEnsemble from git. You will need logon credentials for the libEnsemble PyPI. You will also need @@ -19,12 +19,12 @@ In the package directory on the master branch (the one containing setup.py) do the following: -Create distribution: +Create distribution:: python setup.py sdist -Upload (you will need username/password here): +Upload (you will need username/password here):: twine upload dist/* @@ -36,6 +36,6 @@ It should also be visible here: https://pypi.org/project/libensemble/ -For more details on creating PyPI packages see:: +For more details on creating PyPI packages see: https://betterscientificsoftware.github.io/python-for-hpc/tutorials/python-pypi-packaging/ diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index 851bc24f8..da6c73f14 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -8,7 +8,7 @@ Introduction This assumes you have already: - - Made a PyPI package for new version of libensemble + - Made a PyPI package for new version of libEnsemble - Made a GitHub fork of Spack and cloned it to your local system. Details on how to create forks can be found at: https://help.github.com/articles/fork-a-repo @@ -18,7 +18,7 @@ You now have a configuration like shown in answer at: https://stackoverflow.com/ Upstream, in this case, is the official Spack repository on GitHub. Origin is your fork on GitHub and Local Machine is your local clone (from your fork). -Make sure SPACK_ROOT is set and spack binary is in your path:: +Make sure ``SPACK_ROOT`` is set and spack binary is in your path:: export SPACK_ROOT= export PATH=$SPACK_ROOT/bin:$PATH @@ -62,11 +62,10 @@ If you have local changes to go "on top" of latest code:: git rebase upstream/develop -Or to make your local machine identical to upstream repo (**WARNING** Any local changes WILL BE LOST):: +Or to make your local machine identical to upstream repo (**WARNING** Any local changes will be lost!):: git reset --hard upstream/develop - (Optional) You may want to update your forked (origin) repo on GitHub at this point. This may requires a forced push:: @@ -83,12 +82,12 @@ See the Spack [packaging](https://spack.readthedocs.io/en/latest/packaging_guide [contibution](https://spack.readthedocs.io/en/latest/contribution_guide.html) guides for more info. -Quick example to update libensemble:: +Quick example to update libEnsemble:: git branch update_libensemble git checkout update_libensemble -This will open the libensemble package.py file in your editor (given by env variable EDITOR):: +This will open the libEnsemble ``package.py`` file in your editor (given by env variable EDITOR):: spack edit py-libensemble # SPACK_ROOT must be set (see above) (python packages use "py-" prefix) @@ -101,7 +100,7 @@ Get the tarball (see PyPI instructions), for the new release and use:: sha256sum libensemble-*.tar.gz -Update the `package.py` file by pasting in the new checksum lines (and make sure url line points to latest version). +Update the ``package.py`` file by pasting in the new checksum lines (and make sure url line points to latest version). Also update any dependencies for the new version. Check package:: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index cb964522b..7292a27fe 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -86,20 +86,10 @@ The example sim and gen functions and tests require the following dependencies: * NLopt_ - Installed with `shared libraries enabled `_. PETSc and NLopt must be built with shared libraries enabled and present in -sys.path (e.g., via setting the PYTHONPATH environment variable). NLopt should +sys.path (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt should produce a file nlopt.py if Python is found on the system. NLopt may also require SWIG_ to be installed on certain systems. -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _Python: http://www.python.org -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _SciPy: http://www.scipy.org -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _Balsam: https://www.alcf.anl.gov/balsam -.. _SWIG: http://swig.org/ - Installation ------------ @@ -129,7 +119,7 @@ regularly on: * `Travis CI `_ -The test suite requires the mock, pytest, pytest-cov and pytest-timeout +The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ packages to be installed and can be run from the libensemble/tests directory of the source distribution by running:: @@ -169,7 +159,7 @@ Basic Usage The examples directory contains example libEnsemble calling scripts, sim functions, gen functions, alloc functions and job submission scripts. -The user will create a python script to call the libEnsemble :doc:`libE` function. +The user will create a python script to call the libEnsemble :doc:`libE ` function. This must supply the :ref:`sim_specs` and :ref:`gen_specs`, and optionally :ref:`libE_specs`, :ref:`alloc_specs` and :ref:`persis_info`. @@ -192,5 +182,19 @@ function used in the regression tests, which can be found in See the `user-guide `_ for more information. +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _Python: http://www.python.org +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org +.. _SciPy: http://www.scipy.org +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _SWIG: http://swig.org/ +.. _mock: https://pypi.org/project/mock +.. _pytest: https://pypi.org/project/pytest/ +.. _pytest-cov: https://pypi.org/project/pytest-cov/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ + .. include:: ../README.rst :start-after: docs-include-tag diff --git a/libensemble/tests/balsam_tests/readme.rst b/libensemble/tests/balsam_tests/readme.rst index 8b4461347..d2d1ac3e5 100644 --- a/libensemble/tests/balsam_tests/readme.rst +++ b/libensemble/tests/balsam_tests/readme.rst @@ -10,7 +10,7 @@ Theta: If the instructions are followed to set up a conda environment called balsam, then the script env_setup_theta.sh, under the balsam tests dir, can be sourced in future logins for theta. Or modified for other platforms. In some platforms you may need to only activate the balsam conda environment. ----------------------------------------- -Quickstart - Balsam tests for libensemble +Quickstart - Balsam tests for libEnsemble ----------------------------------------- Having set up balsam, the tests here can be run as follows. @@ -56,7 +56,7 @@ General Usage 1. Register applications ------------------------ -You must register with balsam the parent application (eg. libensemble) and any user application (eg. sim funcs/gen funcs). +You must register with balsam the parent application (eg. libEnsemble) and any user application (eg. sim funcs/gen funcs). This only need be done once - unless eg. name of application changes. Example (as used in tests) - run from within balsam_tests dir: @@ -116,9 +116,9 @@ A log will also be created when run under hpc-edge-service/log/ The standard output will go to file .out. So in above case this will be job_balsam1.out which will be staged out to $WORKDIR -In this case 4 ranks per node and 1 node are selected. This is for running on the parent application (eg. libensemble). This does not constrain the running of sub-apps (eg. helloworld), which will use the full allocation available. +In this case 4 ranks per node and 1 node are selected. This is for running on the parent application (e.g., libEnsemble). This does not constrain the running of sub-apps (eg. helloworld), which will use the full allocation available. -Note that the user jobs (launched in a libensemble job) are registered from within the code. For staging out files, the output directory needs to somehow be accessible to the code. For the tests here, this is simply the directory of the test scripts (accessed via the __file__ variable in python). Search for dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) +Note that the user jobs (launched in a libEnsemble job) are registered from within the code. For staging out files, the output directory needs to somehow be accessible to the code. For the tests here, this is simply the directory of the test scripts (accessed via the __file__ variable in python). Search for dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) To list jobs: diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index e55b3a64e..669ee5d5d 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -302,8 +302,8 @@ hint_colour=$(tput bold;tput setaf 4) #blue # Exit code 5: No tests were collected tput bold -#echo -e "\nRunning $RUN_PREFIX libensemble Test-suite .......\n" -echo -e "\n************** Running: Libensemble Test-Suite **************\n" +#echo -e "\nRunning $RUN_PREFIX libEnsemble Test-suite .......\n" +echo -e "\n************** Running: libEnsemble Test-Suite **************\n" tput sgr 0 echo -e "Selected:" [ $RUN_UNIT_TESTS = "true" ] && echo -e "Unit Tests" diff --git a/libensemble/tests/scaling_tests/forces/readme.md b/libensemble/tests/scaling_tests/forces/readme.md index 68cf8899f..2ab01d893 100644 --- a/libensemble/tests/scaling_tests/forces/readme.md +++ b/libensemble/tests/scaling_tests/forces/readme.md @@ -98,4 +98,12 @@ If either of the plotting options in the submission scripts are set to true, the #### Note on theta_submit_balsam.sh -Adjusting the node/core/worker count.: The NUM_WORKERS variable is only currently used if libEnsemble is running on one node, in which case it should be one less than the number of nodes in the job allocation (leaving one dedicated node to run libEnsemble). If more workers are used then the variables NUM_NODES and RANKS_PER_NODE need to be explicitly set (these are for libensemble which will require one task for the manager and the rest will be workers). The total node allocation (in the COBALT -n directive) will need to be the number of nodes for libEnsemble + number of nodes for each worker to launch jobs to. +Adjusting the node/core/worker count.: The NUM_WORKERS variable is only +currently used if libEnsemble is running on one node, in which case it should +be one less than the number of nodes in the job allocation (leaving one +dedicated node to run libEnsemble). If more workers are used then the variables +NUM_NODES and RANKS_PER_NODE need to be explicitly set (these are for +libEnsemble which will require one task for the manager and the rest will be +workers). The total node allocation (in the COBALT -n directive) will need to +be the number of nodes for libEnsemble + number of nodes for each worker to +launch jobs to. diff --git a/libensemble/tests/standalone_tests/kill_test/readme.txt b/libensemble/tests/standalone_tests/kill_test/readme.txt index 1ab630b12..0191fd0cc 100644 --- a/libensemble/tests/standalone_tests/kill_test/readme.txt +++ b/libensemble/tests/standalone_tests/kill_test/readme.txt @@ -1,15 +1,24 @@ Kill tests ========== -This is to test the killing of MPI jobs. There are two tests and different kill methods which can be selected on the command line. The aim is that kills in libensemble will be automatically configured to work correctly based on system detection. However, this test will show what type of kill works for a given system, and this may be configurable in libensemble in the future. +This is to test the killing of MPI jobs. There are two tests and different kill +methods which can be selected on the command line. The aim is that kills in +libEnsemble will be automatically configured to work correctly based on system +detection. However, this test will show what type of kill works for a given +system, and this may be configurable in libEnsemble in the future. Test: sleep_and_print -The default test (sleep_and_print), should automatically test if all processes of an MPI job are correctly killed and also that a second job can be launched and killed. +The default test (sleep_and_print), should automatically test if all processes +of an MPI job are correctly killed and also that a second job can be launched +and killed. -Launching MPI jobs which write from each MPI task, at regular intervals, to an output file (see sleep_and_print.c). +Launching MPI jobs which write from each MPI task, at regular intervals, to an +output file (see sleep_and_print.c). -This test launches a job, then kills after a few seconds, and then monitors output file to see if output continues. If the first job is succesfully killed, a second is launched and the test repeated. +This test launches a job, then kills after a few seconds, and then monitors +output file to see if output continues. If the first job is succesfully killed, +a second is launched and the test repeated. Instructions ------------ @@ -23,7 +32,8 @@ Either run on local node - or create an allocation of nodes and run:: python killtest.py -where kill_type currently is 1 or 2. [1. is the original kill - 2. is using group ID approach] +where kill_type currently is 1 or 2. [1. is the original kill - 2. is using +group ID approach] E.g Single node with 4 processes: -------------------------------- @@ -39,4 +49,5 @@ kill 2: python killtest.py 2 2 2 If the test fails, an assertion error will occur. -Output files are produced out_0.txt and out_1.txt (one for each job). These can be deleted between runs. +Output files are produced out_0.txt and out_1.txt (one for each job). These can +be deleted between runs. diff --git a/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt b/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt index 1a853b11d..6dc355d0b 100644 --- a/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt +++ b/libensemble/tests/standalone_tests/mpi_launch_test/readme.txt @@ -1,13 +1,19 @@ -This test creates an mpi4py job where each processor launches MPI jobs on a node. This is an essential capability for libensemble with mpi4py. +This test creates an mpi4py job where each processor launches MPI jobs on a +node. This is an essential capability for libEnsemble with mpi4py. You specify the size of the outer jobs and inner jobs as follows: mpirun -np 16 python create_mpi_jobs.py 4 -runs a 16 way job, each launching a 4 processor python hello_world with a short sleep. +runs a 16 way job, each launching a 4 processor python hello_world with a short +sleep. -This is a good test to run to ensure basic functioning on a system, including nested launching. +This is a good test to run to ensure basic functioning on a system, including +nested launching. -The test should create an output file (job_N.out) for each outer rank. A hello_world line is printed in these for each inner rank. +The test should create an output file (job_N.out) for each outer rank. A +hello_world line is printed in these for each inner rank. -Additionally it will print the total number of processes being launched (including outer and inner). This can help test, for example, if oversubscription is supported. +Additionally it will print the total number of processes being launched +(including outer and inner). This can help test, for example, if +oversubscription is supported. From a82d454866cb2ec64718f83a8841ffaa4cc0c8dc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 27 Sep 2019 10:59:06 -0500 Subject: [PATCH 296/644] Adding a possible formatting guidelines --- docs/rst_formatting_guidelines | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/rst_formatting_guidelines diff --git a/docs/rst_formatting_guidelines b/docs/rst_formatting_guidelines new file mode 100644 index 000000000..83329a529 --- /dev/null +++ b/docs/rst_formatting_guidelines @@ -0,0 +1,21 @@ + +Put variables (e.g., gen_f or H) in between double ticks ``stuff here`` (no quotes) + +Put short code in between double ticks ``stuff here`` (no quotes) + +Put referenced file names in between double ticks ``stuff here`` (no quotes) + +Put environment variables in between double ticks ``stuff here`` (no quotes) + +Put warnings line: **WARNING:** + +Put breakout code snippets into a box using two colons :: + +No italics on text of links + +Use italics for emphasis + + +* Unfortunately, the Sphinx crossrefs (e.g., :doc:`ref`) are different then +* hrefs, and they are italicized in the pdf. I assume there is a way to change +* this formatting, but I couldn't find it. From 8dcca8550782c364950831935e07f5bdaf46ded4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 27 Sep 2019 12:26:50 -0500 Subject: [PATCH 297/644] More edits --- CONTRIBUTING.rst | 2 +- docs/FAQ.rst | 24 ++++++------- docs/data_structures/data_structures.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 46 +++++++++++++----------- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8145b0009..46b5dcba6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,7 @@ Contributions may be made via GitHub pull request to: libEnsemble uses the Gitflow model. Contributors should branch from, and make pull requests to, the develop branch. The master branch is used only for releases. Code should pass flake8 tests, allowing for the exceptions -given in the ".flake8" configuration file in the project directory. +given in the ``.flake8`` configuration file in the project directory. Issues can be raised at: diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 84ebef677..deae12a49 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -20,7 +20,7 @@ need to install xQuartz_. .. _xQuartz: https://www.xquartz.org/ -If running in ``'local'`` comms mode try using one of the ``ForkablePdb`` +If running in ``local`` comms mode try using one of the ``ForkablePdb`` routines in ``libensemble/util/forkpdb.py`` to set breakpoints. How well these work may depend on the system. Usage:: @@ -31,7 +31,7 @@ work may depend on the system. Usage:: AssertionError - Idle workers ----------------------------- -**"AssertionError: Should not wait for workers when all workers are idle."** +**AssertionError: Should not wait for workers when all workers are idle.** with ``mpiexec -np 1 python [calling script].py`` @@ -46,7 +46,7 @@ The generator will occupy the one worker, leaving none to run simulation functio Not enough processors per worker to honour arguments ---------------------------------------------------- -**"libensemble.resources.ResourcesException: Not enough processors per worker to honour arguments."** +**libensemble.resources.ResourcesException: Not enough processors per worker to honour arguments.** This is likely when using the job_controller, when there are not enough cores/nodes available to launch jobs. This can be disabled if you want @@ -63,13 +63,13 @@ hyperthreads/SMT threads available. FileExistsError --------------- -**"FileExistsError: [Errno 17] File exists: './sim_worker1'"** +**FileExistsError: [Errno 17] File exists: './sim_worker1'** This can happen when libEnsemble tries to create sim directories that already exist. If the directory does not already exist, a possible cause is that you are trying -to run using ``mpiexec``, when the ``libE_specs['comms']`` option is set to ``'local'``. +to run using ``mpiexec``, when the ``libE_specs['comms']`` option is set to ``local``. Note that to run with differently named sub-directories you can use the -``'sim_dir_suffix'`` option to :ref:`sim_specs`. +``sim_dir_suffix`` option to :ref:`sim_specs`. libEnsemble hangs when using mpi4py @@ -81,7 +81,7 @@ observed with Intels Truescale (TMI) fabric at time of writing. This can be solv either by switch fabric or turning off matching probes before the MPI module is first imported. -Add these two lines BEFORE 'from mpi4py import MPI':: +Add these two lines BEFORE ``from mpi4py import MPI``:: import mpi4py mpi4py.rc.recv_mprobe = False @@ -94,7 +94,7 @@ Messages are not received correctly when using mpi4py This may manifest itself with the following error: -**"_pickle.UnpicklingError: invalid load key, '\x00'."** +**_pickle.UnpicklingError: invalid load key, '\x00'.** or some similar variation. This has been observed with the OFA fabric. The solution is to either switch fabric or turn off matching probes. @@ -115,7 +115,7 @@ PETSc and MPI errors with ``python [test with PETSc].py --comms local --nworkers 4`` This error occurs on some platforms, including Travis, when using PETSc with libEnsemble -in 'local' (multiprocessing) mode. We believe this is due to PETSc initializing MPI +in ``local`` (multiprocessing) mode. We believe this is due to PETSc initializing MPI before libEnsemble forks processes using multiprocessing. The recommended solution is running libEnsemble in MPI mode. An alternative solution may be using a serial build of PETSc. @@ -127,7 +127,7 @@ handles an existing MPI communicator in a particular platform. Fatal error in MPI_Init_thread ------------------------------ -**"Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"** +**Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed** This error may be macOS specific. MPI uses TCP to initiate connections, @@ -161,8 +161,8 @@ Running out of contexts when running libEnsemble in distributed mode on TMI fabr The error message may be similar to below: -**"can't open hfi unit: -1 (err=23)"** -**"[13] MPI startup(): tmi fabric is not available and fallback fabric is not enabled"** +**can't open hfi unit: -1 (err=23)** +**[13] MPI startup(): tmi fabric is not available and fallback fabric is not enabled** This may occur on TMI when libEnsemble Python processes have been launched to a node and these, in turn, launch jobs on the node; creating too many processes for the available contexts. Note that diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index 9f0a7b52a..dc55d7ae6 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -1,7 +1,7 @@ Data Structures =============== -Users can check the formatting and consistency of ``exit_criteria`` and each ``'specs'`` +Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` dictionary with the ``check_inputs()`` function within the ``libE module``. Provide any combination of these data structures as keyword arguments. For example:: diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 1f0b46d62..0e23fb40a 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -7,10 +7,10 @@ calculations in parallel using :doc:`libEnsemble<../quickstart>` with Python's M The foundation of writing libEnsemble routines is accounting for four components: - 1. The *Generator Function* :ref:`gen_f`, which produces values for simulations. - 2. The *Simulator Function* :ref:`sim_f`, which performs simulations based on values from ``gen_f``. - 3. The *Allocation Function* :ref:`alloc_f`, which decides which of the previous two functions should be called, when. - 4. The *Calling Script*, which defines parameters and information about these functions and the libEnsemble task, then begins execution. + 1. The generator function :ref:`gen_f`, which produces values for simulations. + 2. The simulator function :ref:`sim_f`, which performs simulations based on values from ``gen_f``. + 3. The allocation function :ref:`alloc_f`, which decides which of the previous two functions should be called, when. + 4. The calling script, which defines parameters and information about these functions and the libEnsemble task, then begins execution. libEnsemble initializes a *manager* process and as many *worker* processes as the user requests. The manager coordinates data-transfer between workers and assigns each @@ -66,15 +66,22 @@ function. An available libEnsemble worker will call this generator function with the following parameters: -* :ref:`H`: The History array. Updated by the workers with ``gen_f`` and ``sim_f`` inputs and outputs, then returned to the user. libEnsemble passes ``H`` to the generator function in case the user wants to generate new values based on previous data. +* :ref:`H`: The History array. Updated by the workers + with ``gen_f`` and ``sim_f`` inputs and outputs, then returned to the user. + libEnsemble passes ``H`` to the generator function in case the user wants to + generate new values based on previous data. -* :ref:`persis_info`: Dictionary with worker-specific information. In our case this dictionary contains mechanisms called random streams for generating random numbers. +* :ref:`persis_info`: Dictionary with worker-specific + information. In our case this dictionary contains mechanisms called random + streams for generating random numbers. -* :ref:`gen_specs`: Dictionary with ``gen_f`` specifications like simulation IDs, inputs and outputs, data-types, and other fields. +* :ref:`gen_specs`: Dictionary with ``gen_f`` + specifications like simulation IDs, inputs and outputs, data-types, and other + fields. Later on, we'll populate ``gen_specs`` and ``persis_info`` in our calling script. -For now, create a new Python file named 'generator.py'. Write the following: +For now, create a new Python file named ``generator.py``. Write the following: .. code-block:: python :linenos: @@ -117,7 +124,7 @@ functions perform calculations based on values from the generator function. The only new parameter here is :ref:`sim_specs`, which serves a similar purpose to ``gen_specs``. -Create a new Python file named 'simulator.py'. Write the following: +Create a new Python file named ``simulator.py``. Write the following: .. code-block:: python :linenos: @@ -148,7 +155,7 @@ Calling Script Now we can write the calling script that configures our generator and simulator functions and calls libEnsemble. -Create an empty Python file named 'calling_script.py'. +Create an empty Python file named ``calling_script.py``. In this file, we'll start by importing NumPy, libEnsemble, and the generator and simulator functions we just created. @@ -159,7 +166,6 @@ use. Our communication method, 'local', refers to Python's Multiprocessing. .. code-block:: python :linenos: - import numpy as np from libensemble.libE import libE from generator import gen_random_sample from simulator import sim_find_sine @@ -213,7 +219,7 @@ This :ref:`H` is the final version of the History arra :linenos: H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs) + libE_specs=libE_specs) print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array print(H) @@ -302,7 +308,7 @@ Modifying the Calling Script """""""""""""""""""""""""""" Only a few changes are necessary to make our code MPI-compatible. Modify the top -of the Calling Script as follows: +of the calling script as follows: .. code-block:: python :linenos: @@ -321,14 +327,14 @@ of the Calling Script as follows: is_master = (MPI.COMM_WORLD.Get_rank() == 0) # master process has MPI rank 0 So that only one process executes the graphing and printing portion of our code, -modify the bottom of the Calling Script like this: +modify the bottom of the calling script like this: .. code-block:: python :linenos: :emphasize-lines: 4 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs) + libE_specs=libE_specs) if is_master: @@ -339,15 +345,15 @@ modify the bottom of the Calling Script like this: colors = ['b', 'g', 'r', 'y', 'm', 'c', 'k', 'w'] for i in range(1, nworkers + 1): - worker_xy = np.extract(H['sim_worker'] == i, H) - x = [entry.tolist()[0] for entry in worker_xy['x']] - y = [entry for entry in worker_xy['y']] - plt.scatter(x, y, label='Worker {}'.format(i), c=colors[i-1]) + worker_xy = np.extract(H['sim_worker'] == i, H) + x = [entry.tolist()[0] for entry in worker_xy['x']] + y = [entry for entry in worker_xy['y']] + plt.scatter(x, y, label='Worker {}'.format(i), c=colors[i-1]) plt.title('Sine calculations for a uniformly sampled random distribution') plt.xlabel('x') plt.ylabel('sine(x)') - plt.legend(loc = 'lower right') + plt.legend(loc='lower right') plt.show() With these changes in place, our libEnsemble code can be run with MPI by: From 874aa5dc5dda1cbcd645df10434e34d93a52153a Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 1 Oct 2019 12:19:08 -0500 Subject: [PATCH 298/644] Modify test launch line --- libensemble/mpi_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index 31991bbf1..32e790f12 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -228,8 +228,7 @@ def launch(self, calc_type, num_procs=None, num_nodes=None, runline.extend(job.app_args.split()) if test: - logger.info('Test selected: Not launching job') - logger.info('runline args are {}'.format(runline)) + logger.info('Test (No launch) Runline: {}'.format(' '.join(runline))) else: subgroup_launch = True if self.mpi_launch_type in ['aprun']: From cbab9f1c369a2ae247aa607386b8785b3fa65132 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 1 Oct 2019 13:39:56 -0500 Subject: [PATCH 299/644] Clean up forces.c --- .../tests/scaling_tests/forces/forces.c | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/forces.c b/libensemble/tests/scaling_tests/forces/forces.c index 7b99b74ca..829bc4c11 100755 --- a/libensemble/tests/scaling_tests/forces/forces.c +++ b/libensemble/tests/scaling_tests/forces/forces.c @@ -2,19 +2,19 @@ Naive Electostatics Code Example This is designed only as an artificial, highly conifurable test code for a libEnsemble sim func. - + Particles position and charge are initiated by a random stream. Particles are replicated on all ranks. Each rank computes forces for a subset of particles. Particle force arrays are allreduced across ranks. - + Sept 2019: Added OpenMP options for CPU and GPU. Toggle in forces_naive function. - + Run executable on N procs: - + mpirun -np N ./forces.x - + Author: S Hudson. -------------------------------------------------------------------- */ @@ -39,8 +39,8 @@ static FILE* stat_fp; // Return elapsed wall clock time from start/end timevals double elapsed(struct timeval *tv1, struct timeval *tv2) { - return (double)(tv2->tv_usec - tv1->tv_usec) / 1000000 - + (double)(tv2->tv_sec - tv1->tv_sec); + return (double)(tv2->tv_usec - tv1->tv_usec) / 1000000 + + (double)(tv2->tv_sec - tv1->tv_sec); } // Print from each thread. @@ -64,7 +64,7 @@ typedef struct particle { double p[3]; // Particle position double f[3]; // Particle force double q; // Particle charge -}__attribute__((__packed__)) particle; +}__attribute__((__packed__)) particle; // Seed RNG @@ -125,12 +125,12 @@ double forces_naive(int n, int lower, int upper, particle* parr) { #pragma omp target teams distribute parallel for \ map(to: lower,upper,n) map(tofrom: parr[0:n]) \ reduction(+: ret) //thread_limit(128) //*/ - + // For CPU //* #pragma omp parallel for default(none) shared(n,lower,upper,parr) \ private(i,j,dx,dy,dz,r,force) \ - reduction(+:ret) //*/ + reduction(+:ret) //*/ for(i=lower; i=2) { - num_particles = atoi(argv[1]); // no. of particles + num_particles = atoi(argv[1]); // No. of particles } - + if (argc >=3) { - num_steps = atoi(argv[2]); // no. of timesteps + num_steps = atoi(argv[2]); // No. of timesteps } - + if (argc >=4) { rand_seed = atoi(argv[3]); // RNG seed seed_rand(rand_seed); } - + if (argc >=5) { kill_rate = atof(argv[4]); // Proportion of jobs to kill step_survival_rate = pow((1-kill_rate),(1.0/num_steps)); } - + particle* parr = malloc(num_particles * sizeof(particle)); build_system(num_particles, parr); //printf("\n"); - + ierr = MPI_Init(&argc, &argv); ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); ierr = MPI_Comm_size(MPI_COMM_WORLD, &num_procs); - + if (rank == 0) { printf("Particles: %d\n",num_particles); printf("Timesteps: %d\n",num_steps); printf("MPI Ranks: %d\n",num_procs); printf("Random seed: %d\n",rand_seed); } - + if (CHECK_THREADS) { check_threads(rank); } k = num_particles / num_procs; - m = num_particles % num_procs; //Remainder = no. procs with extra particle + m = num_particles % num_procs; // Remainder = no. procs with extra particle p_lower = rank * k + min(rank, m); p_upper = (rank + 1) * k + min(rank + 1, m); local_n = p_upper - p_lower; - + if (PRINT_PARTICLE_DECOMP) { MPI_Barrier(MPI_COMM_WORLD); printf("Proc: %d has %d particles\n", rank, local_n); @@ -388,39 +387,39 @@ int main(int argc, char **argv) { gettimeofday(&tstart, NULL); for (step=0; step Date: Tue, 1 Oct 2019 13:54:36 -0500 Subject: [PATCH 300/644] Space cleanup forces and mini_forces files --- .../tests/scaling_tests/forces/forces.c | 48 +++++++++---------- .../forces/mini_forces/mini_forces.c | 38 +++++++-------- .../forces/mini_forces/mini_forces_AoS.c | 38 +++++++-------- 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/forces.c b/libensemble/tests/scaling_tests/forces/forces.c index 829bc4c11..6526b939b 100755 --- a/libensemble/tests/scaling_tests/forces/forces.c +++ b/libensemble/tests/scaling_tests/forces/forces.c @@ -4,11 +4,11 @@ code for a libEnsemble sim func. Particles position and charge are initiated by a random stream. - Particles are replicated on all ranks. + Particles are replicated on all ranks. Each rank computes forces for a subset of particles. Particle force arrays are allreduced across ranks. - Sept 2019: + Sept 2019: Added OpenMP options for CPU and GPU. Toggle in forces_naive function. Run executable on N procs: @@ -69,15 +69,15 @@ typedef struct particle { // Seed RNG int seed_rand(int seed) { - srand(seed); + srand(seed); return 0; } // Return a random number from a persistent stream //TODO Use parallel RNG - As replicated data can currently do on master rank. double get_rand() { - double randnum; - randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) return randnum; } @@ -88,7 +88,7 @@ int build_system(int n, particle* parr) { int q_range_high = 10; double extent = 10.0; int i, dim; - + for(i=0; i @@ -36,14 +36,14 @@ // Seed RNG int seed_rand(int seed) { - srand(seed); + srand(seed); return 0; } // Return a random number from a persistent stream double get_rand() { - double randnum; - randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) return randnum; } @@ -54,7 +54,7 @@ int build_system(int n, double* x, double* y, double* z, double* fx, double* fy, int q_range_high = 10; double extent = 10.0; int i; - + for(i=0; i @@ -43,14 +43,14 @@ typedef struct particle { // Seed RNG int seed_rand(int seed) { - srand(seed); + srand(seed); return 0; } // Return a random number from a persistent stream double get_rand() { - double randnum; - randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) + double randnum; + randnum = (double)rand()/(double)(RAND_MAX + 1.0); //[0,1) return randnum; } @@ -61,7 +61,7 @@ int build_system(int n, particle* parr) { int q_range_high = 10; double extent = 10.0; int i, dim; - + for(i=0; i Date: Wed, 2 Oct 2019 11:31:29 -0500 Subject: [PATCH 301/644] Trying a more-modern sphinx version --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index d4fc3b2f3..25373417b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ def __getattr__(cls, name): # If your documentation needs a minimal Sphinx version, state it here. # -# needs_sphinx = '1.0' +needs_sphinx = '2.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom From cc9d658772e4b19960e8762f74768b4315b789c7 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 2 Oct 2019 13:45:39 -0500 Subject: [PATCH 302/644] Change srun to kill type 1 --- libensemble/mpi_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index c5e5f2ec8..5f4412df3 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -234,7 +234,7 @@ def launch(self, calc_type, num_procs=None, num_nodes=None, logger.info('runline args are {}'.format(runline)) else: subgroup_launch = True - if self.mpi_launch_type in ['aprun']: + if self.mpi_launch_type in ['aprun', 'srun']: subgroup_launch = False retry_count = 0 From 8bccb5e765ded86de37863a73b4b31919dcae558 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 2 Oct 2019 13:47:59 -0500 Subject: [PATCH 303/644] Update kill test and info for Cori --- .../kill_test/log.autotest.txt | 19 ++++++++++++++++++- .../standalone_tests/kill_test/readme.txt | 2 +- .../kill_test/sleep_and_print.c | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt index 9950cfc5c..fcb690ba3 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt @@ -10,7 +10,7 @@ Instructions Build the C program: mpicc -g -o sleep_and_print.x sleep_and_print.c - OR (e.g. on Theta): + OR (e.g. on Theta/Cori): cc -g -o sleep_and_print.x sleep_and_print.c Either run on local node - or create an allocation of nodes and run:: @@ -55,6 +55,8 @@ Bebop (intelmpi):: kill 1: Fails kill 2: Works + + [*Update 2019: kill 1 seems to also work if use srun instead of mpirun] Cooley (intelmpi):: @@ -93,4 +95,19 @@ Theta (intelmpi):: Alternative - may be to use apkill. This may work without unloading xalt - not tried. +Cori (intelmpi - launched with srun):: + + Single node on 4 processes: + + kill 1: Works + kill 2: Works + + Two nodes with 2 processes each: + + kill 1: Works + kill 2: Works + + *Note: Launching of sruns from python subprocess is slow. + + diff --git a/libensemble/tests/standalone_tests/kill_test/readme.txt b/libensemble/tests/standalone_tests/kill_test/readme.txt index 3c1382272..d0c00e3d1 100644 --- a/libensemble/tests/standalone_tests/kill_test/readme.txt +++ b/libensemble/tests/standalone_tests/kill_test/readme.txt @@ -16,7 +16,7 @@ Instructions Build the C program: mpicc -g -o sleep_and_print.x sleep_and_print.c - OR (e.g. on Theta): + OR (e.g. on Theta/Cori): cc -g -o sleep_and_print.x sleep_and_print.c Either run on local node - or create an allocation of nodes and run:: diff --git a/libensemble/tests/standalone_tests/kill_test/sleep_and_print.c b/libensemble/tests/standalone_tests/kill_test/sleep_and_print.c index 1eb88301d..569616c4e 100644 --- a/libensemble/tests/standalone_tests/kill_test/sleep_and_print.c +++ b/libensemble/tests/standalone_tests/kill_test/sleep_and_print.c @@ -11,7 +11,7 @@ int main(int argc, char **argv) double time_secs; delay=1; - num_sleeps = 12; + num_sleeps = 16; ierr = MPI_Init(&argc, &argv); ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); From 3c71d7af49e1d484ff8a1cbac1f57ddf8f83a63f Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 2 Oct 2019 16:10:53 -0500 Subject: [PATCH 304/644] trigger unique build with small change --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 09c52672a..7b6344214 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,6 @@ before_script: - source setbalsampath.sh # Allow 10000 files to be open at once (critical for persistent_aposmm) - ulimit -Sn 10000 - # Set conda compilers to use new SDK instead of Travis default. # - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; # source setenv.sh; From bf70f8517819cf92d75b254c592fc61ad4bd8456 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 2 Oct 2019 17:25:01 -0500 Subject: [PATCH 305/644] Add node count when using hostlist on srun This is required as the hostlist option on srun is not exclusive. --- libensemble/mpi_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index 63bda3764..1623c983e 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -89,7 +89,8 @@ def __init__(self, auto_resources=True, central_mode=False, '-N {ranks_per_node}'], 'jsrun': ['jsrun', '--np {num_procs}'], # Need to add more 'srun': ['srun', '-w {hostlist}', '-n {num_procs}', - '--ntasks-per-node {ranks_per_node}'] # Need to add more + '--nodes {num_nodes}', + '--ntasks-per-node {ranks_per_node}'] } self.mpi_launch_type = MPIResources.get_MPI_variant() self.mpi_command = mpi_commands[self.mpi_launch_type] From 2f265c88586f0d72265697c205800dcb62872736 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 2 Oct 2019 20:28:27 -0500 Subject: [PATCH 306/644] Use hostlist if one worker per node --- libensemble/mpi_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py index 1623c983e..739f26a0d 100644 --- a/libensemble/mpi_controller.py +++ b/libensemble/mpi_controller.py @@ -126,8 +126,8 @@ def _get_mpi_specs(self, num_procs, num_nodes, ranks_per_node, ranks_per_node=ranks_per_node, hyperthreads=hyperthreads) - # Use hostlist if multiple nodes, otherwise machinefile - if num_nodes > 1: + # Use hostlist if full nodes, otherwise machinefile + if self.resources.worker_resources.workers_per_node == 1: hostlist = self.resources.get_hostlist() else: machinefile = "machinefile_autogen" From e316abc606e7298a41968d4dc2bb63396a59df30 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 3 Oct 2019 10:43:35 -0500 Subject: [PATCH 307/644] attempt installing libopenblas earlier --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7b6344214..574b3d4da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,7 @@ install: # mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; - conda install $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI + - conda install libopenblas - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 - pip install flake8 From 35f09ff5c872c754f31bda3e0e3802f32b4a01b7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 3 Oct 2019 15:57:31 -0500 Subject: [PATCH 308/644] user guide reformatting --- docs/user_guide.rst | 96 ++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 0be2d95d8..135e6b959 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -5,24 +5,24 @@ libEnsemble overview -------------------- libEnsemble is a software library to coordinate the concurrent evaluation of ensembles of calculations. libEnsemble uses a manager to allocate work to -various workers. (A libEnsemble worker is the smallest indivisible unit to -perform some calculation.) The work performed by libEnsemble is governed by +various workers. A libEnsemble worker is the smallest indivisible unit to +perform some calculation. The work performed by libEnsemble is governed by three routines: -* gen_f: Generates inputs to sim_f. -* sim_f: Evaluates a simulation or other evaluation at output from gen_f. -* alloc_f: Decides whether sim_f or gen_f should be called (and with what input/resources) as workers become available. +* :ref:`gen_f`: Generates inputs to ``sim_f``. +* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. +* :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. -Example sim_f, gen_f, and alloc_f routines can be found in the +Example ``sim_f``, ``gen_f``, and ``alloc_f`` routines can be found in the ``examples/sim_funcs/``, ``examples/gen_funcs/``, and ``examples/alloc_funcs/`` directories, respectively. Examples of scripts used for calling libEnsemble can be found in ``examples/calling_scripts/``. To enable portability, a :doc:`job_controller` -interface is supplied for users to launch and monitor jobs in their user-provided sim_f and -gen_f routines. +interface is supplied for users to launch and monitor jobs in their user-provided ``sim_f`` and +``gen_f`` routines. -The default alloc_f tells each available worker to call sim_f with the highest -priority unit of work from gen_f. If a worker is idle and there is no gen_f -output to give, the worker is told to call gen_f. +The default ``alloc_f`` tells each available worker to call ``sim_f`` with the highest +priority unit of work from ``gen_f``. If a worker is idle and there is no ``gen_f`` +output to give, the worker is told to call ``gen_f``. Expected use cases @@ -34,36 +34,36 @@ to support) and plan to have examples of: * A user is looking to optimize a simulation calculation. The simulation may already be using parallel resources, but not a large fraction of some computer. libEnsemble can coordinate the concurrent evaluation of the - simulation sim_f at various parameter values and gen_f would return candidate - parameter values (possibly after each sim_f output). + simulation ``sim_f`` at various parameter values and ``gen_f`` would return candidate + parameter values (possibly after each ``sim_f`` output). -* A user has a gen_f that produces different meshes to be used within a - sim_f. Given the sim_f output, gen_f will refine a mesh or produce a new +* A user has a ``gen_f`` that produces different meshes to be used within a + ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or produce a new mesh. libEnsemble can ensure that the calculated meshes can be used by multiple simulations without requiring movement of data. -* A user is attempting to sample a simulation sim_f at some parameter values, +* A user is attempting to sample a simulation ``sim_f`` at some parameter values, many of which will cause the simulation to fail. libEnsemble can stop unresponsive evaluations, and recover computational resources for future - evaluations. gen_f can possibly update the sampling after discovering regions - where evaluations of sim_f fail. + evaluations. ``gen_f`` can possibly update the sampling after discovering regions + where evaluations of ``sim_f`` fail. -* A user has a simulation sim_f that requires calculating multiple expensive - quantities, some of which depend on other quantities. sim_f can observe +* A user has a simulation ``sim_f`` that requires calculating multiple expensive + quantities, some of which depend on other quantities. ``sim_f`` can observe intermediate quantities in order to stop related calculations and preempt future calculations associated with poor parameter values. -* A user has a sim_f with multiple fidelities, with the +* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity evaluations requiring more computational resources, and a - gen_f/alloc_f that decides which parameters should be evaluated and at what + ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and at what fidelity level. libEnsemble can coordinate these evaluations without requiring the user know parallel programming. -* A user wishes to identify multiple local optima for a sim_f. Furthermore, +* A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, sensitivity analysis is desired at each identified optimum. libEnsemble can - use the points from the APOSMM gen_f to identify optima; and after a point is - ruled to be an optimum, a different gen_f can produce a collection of - parameters necessary for sensitivity analysis of sim_f. + use the points from the APOSMM ``gen_f`` to identify optima; and after a point is + ruled to be an optimum, a different ``gen_f`` can produce a collection of + parameters necessary for sensitivity analysis of ``sim_f``. Naturally, combinations of these use cases are supported as well. An example of @@ -74,51 +74,51 @@ relies on simulations that fail frequently. The libEnsemble History Array ----------------------------- -libEnsemble uses a numpy structured array H to store output from gen_f and -corresponding sim_f output. Similarly, gen_f and sim_f are expected to return -output in numpy structured arrays. The names of the fields to be given as input -to gen_f and sim_f must be an output from gen_f or sim_f. In addition to the -fields output from sim_f and gen_f, the final history returned from libEnsemble +libEnsemble uses a NumPy structured array :ref:`H` to store output from ``gen_f`` and +corresponding ``sim_f`` output. Similarly, ``gen_f`` and ``sim_f`` are expected to return +output in NumPy structured arrays. The names of the fields to be given as input +to ``gen_f`` and ``sim_f`` must be an output from ``gen_f`` or ``sim_f``. In addition to the +fields output from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will include the following fields: -* sim_id' [int]: Each unit of work output from gen_f must have an associated - sim_id. The generator can assign this, but users must be careful to ensure - points are added in order. For example, if alloc_f allows for two gen_f - instances to be running simultaneously, alloc_f should ensure that both don’t - generate points with the same sim_id. +* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an associated + ``sim_id``. The generator can assign this, but users must be careful to ensure + points are added in order. For example, ``if alloc_f`` allows for two ``gen_f`` + instances to be running simultaneously, ``alloc_f`` should ensure that both don’t + generate points with the same ``sim_id``. -* given' [bool]: Has this gen_f output been given to a libEnsemble worker to be +* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble worker to be evaluated yet? -* given_time' [float]: At what time (since the epoch) was this gen_f output +* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` output given to a worker? -* sim_worker' [int]: libEnsemble worker that it was given to be evaluated. +* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. -* gen_worker' [int]: libEnsemble worker that generated this sim_id +* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` -* gen_time' [float]: At what time (since the epoch) was this entry (or - collection of entries) put into H by the manager +* ``gen_time`` [float]: At what time (since the epoch) was this entry (or + collection of entries) put into ``H`` by the manager -* returned' [bool]: Has this worker completed the evaluation of this unit of +* ``returned`` [bool]: Has this worker completed the evaluation of this unit of work? libEnsemble Output ------------------ -The history array and persis_info dictionary are returned to the user by libEnsemble. -In the case that libEnsemble aborts on an exception, these are dumped to files. The +The history array and :ref:`persis_info` dictionary are returned to the user by libEnsemble. +If libEnsemble aborts on an exception, these structures are dumped to files. The existing history array is dumped to a file named ``libE_history_at_abort_.npy``, -and the persis_info to ``libE_history_at_abort_.pickle``, where sim_count is +and ``persis_info`` to ``libE_history_at_abort_.pickle``, where ``sim_count`` is the number of points evaluated. Other libEnsemble files produced by default are: -**libE_stats.txt**: This contains a one-line summary of all user calculations. Each +``libE_stats.txt``: This contains a one-line summary of all user calculations. Each calculation summary is sent by workers to the manager and printed as the run progresses. -**ensemble.log**: This is the logging output from libEnsemble. The default logging +``ensemble.log``: This is the logging output from libEnsemble. The default logging is at INFO level. To gain additional diagnostics logging level can be set to DEBUG. If this file is not removed, multiple runs will append output. Messages at or above level MANAGER_WARNING are also copied to stderr to alert the user promptly. From 19a0f9b2cab6decc9d6247bfef9f48262cc45534 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 3 Oct 2019 15:59:41 -0500 Subject: [PATCH 309/644] all-around reformatting and clarification --- docs/FAQ.rst | 61 ++++++++++++++++++++++----------------------- docs/logging.rst | 4 +-- docs/quickstart.rst | 55 +++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 59 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index deae12a49..d0c44346f 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -13,6 +13,7 @@ Parallel Debugging **How can I perform parallel debugging on libEnsemble, or debug specific processes?** +This is most easily addressed when running with MPI. Try the following: ``mpiexec -np [num processes] xterm -e 'python [calling script].py'`` This will launch an xterm terminal window specific to each process. Mac users will @@ -28,18 +29,18 @@ work may depend on the system. Usage:: ForkablePdb().set_trace() -AssertionError - Idle workers +AssertionError - idle workers ----------------------------- **AssertionError: Should not wait for workers when all workers are idle.** with ``mpiexec -np 1 python [calling script].py`` -This error occurs when the manager is waiting, although no workers are busy. +This error occurs when the manager is waiting although no workers are busy. In the above case, this occurs because an MPI libEnsemble run was initiated with only one process, resulting in one manager but no workers. -Note: this may also occur with two processes if you are using a persistent generator. +This may also occur with two processes if you are using a persistent generator. The generator will occupy the one worker, leaving none to run simulation functions. @@ -48,16 +49,15 @@ Not enough processors per worker to honour arguments **libensemble.resources.ResourcesException: Not enough processors per worker to honour arguments.** -This is likely when using the job_controller, when there are not enough -cores/nodes available to launch jobs. This can be disabled if you want -to oversubscribe (often if testing on a local machine). Set up the -job_controller with ``auto_resources=False``. E.g.:: +This error often occurs when there aren't enough cores/nodes available to launch +jobs with the job controller. Automatic partitioning of resources can be disabled +if you want to oversubscribe (often if testing on a local machine) by configuring +the job controller with ``auto_resources=False``. E.g.:: jobctrl = MPIJobController(auto_resources=False) -Also, note that the job_controller launch command has the argument -hyperthreads, which is set to True, will attempt to use all -hyperthreads/SMT threads available. +Note that the job_controller ``.launch()`` method has a parameter``hyperthreads`` +which will attempt to use all hyperthreads/SMT threads available if set to ``True`` FileExistsError @@ -65,21 +65,22 @@ FileExistsError **FileExistsError: [Errno 17] File exists: './sim_worker1'** -This can happen when libEnsemble tries to create sim directories that already exist. If -the directory does not already exist, a possible cause is that you are trying -to run using ``mpiexec``, when the ``libE_specs['comms']`` option is set to ``local``. -Note that to run with differently named sub-directories you can use the -``sim_dir_suffix`` option to :ref:`sim_specs`. +This can happen when libEnsemble tries to create sim directories that already exist. +If these directories do not already exist, another possibility is that you are trying +to run libEnsemble using ``mpiexec`` when the ``libE_specs['comms']`` option is +set to ``local``. + +To create differently named sim directories, you can use the ``sim_dir_suffix`` +option in :ref:`sim_specs`. libEnsemble hangs when using mpi4py ----------------------------------- -One cause of this could be that the communications fabric does not support matching -probes (part of the MPI 3.0 standard), which mpi4py uses by default. This has been -observed with Intels Truescale (TMI) fabric at time of writing. This can be solved -either by switch fabric or turning off matching probes before the MPI module is first -imported. +This may occur if matching probes, which mpi4py uses by default, are not supported +by the communications fabric. This has been observed with Intels Truescale (TMI) +fabric at time of writing. This can be solved either by switching fabrics or disabling +matching probes before the MPI module is first imported. Add these two lines BEFORE ``from mpi4py import MPI``:: @@ -89,10 +90,10 @@ Add these two lines BEFORE ``from mpi4py import MPI``:: Also see https://software.intel.com/en-us/articles/python-mpi4py-on-intel-true-scale-and-omni-path-clusters -Messages are not received correctly when using mpi4py ------------------------------------------------------- +Messages not received correctly when using mpi4py +------------------------------------------------- -This may manifest itself with the following error: +This may manifest with the following error: **_pickle.UnpicklingError: invalid load key, '\x00'.** @@ -146,14 +147,12 @@ macOS - Firewall prompts There are several ways to address this nuisance, but all involve trial and error. -One easy (but insecure) solution is temporarily disabling the Firewall through System Preferences --> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming -connections" rule can be tried for the offending Job Controller executable. -Based on a suggestion from here_, we've had the most success running -``sudo codesign --force --deep --sign - /path/to/application.app`` on our Job Controller executables, -then confirming the next alerts for the executable and ``mpiexec.hydra``. - -.. _`here`: https://coderwall.com/p/5b_apq/stop-mac-os-x-firewall-from-prompting-with-python-in-virtualenv +An easy (but insecure) solution is temporarily disabling the Firewall through +System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, +adding a Firewall "Allow incoming connections" rule can be attempted for the offending +Job Controller executable. We've had limited success running ``sudo codesign --force --deep --sign - /path/to/application.app`` +on our Job Controller executables, then confirming the next alerts for the executable +and ``mpiexec.hydra``. Running out of contexts when running libEnsemble in distributed mode on TMI fabric diff --git a/docs/logging.rst b/docs/logging.rst index d8ed600dc..a5bdd53b6 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -9,7 +9,7 @@ and when jobs are killed. To gain additional diagnostics, the logging level can to DEBUG. libEnsemble produces logging to the file ensemble.log by default. A log file name can also be supplied. -E.g. To change the logging level to DEBUG, provide the following in your the calling scripts:: +To change the logging level to DEBUG, provide the following in your the calling scripts:: from libensemble import libE_logger libE_logger.set_level('DEBUG') @@ -22,7 +22,7 @@ This boundary can be adjusted as follows:: # Only display messages with level >= ERROR libE_logger.set_stderr_level('ERROR') -This feature can be effectively disabled by setting the stderr level to CRITICAL. +stderr displaying can be effectively disabled by setting the stderr level to CRITICAL. Logging API diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 7292a27fe..d74f4061c 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,9 +22,9 @@ What is libEnsemble? ==================== -libEnsemble is a Python library to coordinate the concurrent evaluation of ensembles of computations. -Designed with flexibility in mind, libEnsemble can utilize massively parallel resources to accelerate -the solution of design, decision, and inference problems. +libEnsemble is a Python library to coordinate the concurrent evaluation of ensembles +of computations. Designed with flexibility in mind, libEnsemble can utilize massively +parallel resources to accelerate the solution of design, decision, and inference problems. A visual overview is given in the libEnsemble poster_. @@ -38,18 +38,18 @@ libEnsemble aims for: • Portability and flexibility • Exploitation of persistent data/control flow. -The user selects or supplies a generation function that produces simulation input as well as -a simulation function that performs and monitors the simulations. The generation function -may contain, for example, an optimization method to generate new simulation parameters -on-the-fly and based on the results of previous simulations. Examples and templates of these -functions are included in the library. +The user selects or supplies a generation function that produces simulation input +as well as a simulation function that performs and monitors the simulations. The +generation function may contain, for example, an optimization method to generate +new simulation parameters on-the-fly and based on the results of previous simulations. +Examples and templates of these functions are included in the library. -libEnsemble employs a manager-worker scheme that can run on various communication media -(including MPI, multiprocessing, and TCP). Each worker can control and monitor any type -of job from small sub-node jobs to huge many-node simulations. A job controller -interface is provided to ensure scripts are portable, resilient and flexible; it also -enables automatic detection of the nodes and cores in a system and can split up -jobs automatically if nodes/cores are not supplied. +libEnsemble employs a manager-worker scheme that can run on various communication +media (including MPI, multiprocessing, and TCP). Each worker can control and monitor +any level of work from small sub-node jobs to huge many-node simulations. A job +controller interface is provided to ensure scripts are portable, resilient and +flexible; it also enables automatic detection of the nodes and cores in a system +and can split up jobs automatically if resource data isn't supplied. Dependencies @@ -75,8 +75,9 @@ Optional dependency: From v0.2.0, libEnsemble has the option of using the Balsam job manager. This is required for running libEnsemble on the compute nodes of some supercomputing -platforms (e.g., Cray XC40); platforms that do not support launching jobs from compute nodes. -Note that as of v0.5.0, libEnsemble can also be run on the launch nodes using multiprocessing. +platforms (e.g., Cray XC40); platforms that do not support launching jobs from +compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the launch +nodes using multiprocessing. The example sim and gen functions and tests require the following dependencies: @@ -104,7 +105,8 @@ libEnsemble is also available in the Spack_ distribution. It can be installed fr .. _Spack: https://spack.readthedocs.io/en/latest -The tests and examples can be accessed in the `GitHub `_ repository. If necessary, you may install all optional dependencies (listed above) at once with:: +The tests and examples can be accessed in the `GitHub `_ repository. +If necessary, you may install all optional dependencies (listed above) at once with:: pip install libensemble[extras] @@ -143,14 +145,15 @@ in the top-level directory containing the setup script. Coverage reports are produced separately for unit tests and regression tests under the relevant directories. For parallel tests, the union of all processors is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after running the tests via the html file -libensemble/tests/cov_merge/index.html. The Travis CI coverage results are -given online at +which can be viewed after running the tests via the HTML file +``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are +available online at `Coveralls `_. Note: The job_controller tests can be run using the direct-launch or -Balsam job controllers. However, currently only the direct-launch versions can -be run on Travis CI, which reduces the test coverage results. +Balsam job controllers. Although only the direct-launch versions can +be run on Travis CI, Balsam integration with libEnsemble is now tested via +``test_balsam_hworld.py``. Basic Usage @@ -159,7 +162,7 @@ Basic Usage The examples directory contains example libEnsemble calling scripts, sim functions, gen functions, alloc functions and job submission scripts. -The user will create a python script to call the libEnsemble :doc:`libE ` function. +The user creates a python script to call the libEnsemble :doc:`libE ` function. This must supply the :ref:`sim_specs` and :ref:`gen_specs`, and optionally :ref:`libE_specs`, :ref:`alloc_specs` and :ref:`persis_info`. @@ -192,9 +195,9 @@ See the `user-guide Date: Thu, 3 Oct 2019 18:02:39 -0500 Subject: [PATCH 310/644] Adds env_resources capability to parse different partitions in same allocation --- libensemble/env_resources.py | 36 +++++++++++++------ .../tests/unit_tests/test_env_resources.py | 8 +++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 2225bf795..9c406ebc8 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -105,23 +105,39 @@ def _range_split(s): b = b + 1 return a, b, nnum_len + @staticmethod + def _noderange_split(prefix, nidstr, nidlst): + for nidgroup in nidstr.split(','): + a, b, nnum_len = EnvResources._range_split(nidgroup) + for nid in range(a, b): + nidlst.append(prefix + str(nid).zfill(nnum_len)) + return nidlst + @staticmethod def get_slurm_nodelist(node_list_env): """Get global libEnsemble nodelist from the Slurm environment""" - nidlst = [] fullstr = os.environ[node_list_env] if not fullstr: return [] - splitstr = fullstr.split('[', 1) + splitstr = fullstr.split('],') if len(splitstr) == 1: - return splitstr - prefix = splitstr[0] - nidstr = splitstr[1].strip("]") - for nidgroup in nidstr.split(','): - a, b, nnum_len = EnvResources._range_split(nidgroup) - for nid in range(a, b): - nidlst.append(prefix + str(nid).zfill(nnum_len)) - return sorted(nidlst) + splitstr = fullstr.split('[', 1) + if len(splitstr) == 1: + return splitstr + prefix = splitstr[0] + nidstr = splitstr[1].strip("]") + nidlst = EnvResources._noderange_split(prefix, nidstr, []) + return sorted(nidlst) + else: + splitgroups = [str.split('[', 1) for str in splitstr] + prefixgroups = [group[0] for group in splitgroups] + nodegroups = [group[1].strip(']') for group in splitgroups] + nidlst = [] + for i in range(len(prefixgroups)): + prefix = prefixgroups[i] + nidstr = nodegroups[i] + nidlst.extend(EnvResources._noderange_split(prefix, nidstr, nidlst)) + return sorted(set(nidlst)) @staticmethod def get_cobalt_nodelist(node_list_env): diff --git a/libensemble/tests/unit_tests/test_env_resources.py b/libensemble/tests/unit_tests/test_env_resources.py index b63bbd0fa..3ed5be38e 100644 --- a/libensemble/tests/unit_tests/test_env_resources.py +++ b/libensemble/tests/unit_tests/test_env_resources.py @@ -66,6 +66,13 @@ def test_slurm_nodelist_groups(): assert nodelist == exp_out, "Nodelist returned does not match expected" +def test_slurm_nodelist_groups_partitions(): + os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "bdw-[0254,0384,0565-0568],bdwd-[0004,0009]" + exp_out = ['bdw-0254', 'bdw-0384', 'bdw-0565', 'bdw-0566', 'bdw-0567', 'bdw-0568', 'bdwd-0004', 'bdwd-0009'] + nodelist = EnvResources.get_slurm_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST") + assert nodelist == exp_out, "Nodelist returned does not match expected" + + def test_slurm_nodelist_groups_nodash(): os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "nid0[0020-0022,0137-0139,1234]" exp_out = ['nid00020', 'nid00021', 'nid00022', 'nid00137', 'nid00138', 'nid00139', 'nid01234'] @@ -209,6 +216,7 @@ def test_abbrev_nodenames_cobalt(): test_slurm_nodelist_knl_seq() test_slurm_nodelist_bdw_seq() test_slurm_nodelist_groups() + test_slurm_nodelist_groups_partitions() test_slurm_nodelist_groups_nodash() test_slurm_nodelist_groups_longprefix() test_slurm_nodelist_reverse_grp() From 593597bcc0d9886d1f1084e4e4ca13dff8e6e2d7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 3 Oct 2019 18:03:29 -0500 Subject: [PATCH 311/644] misc flake8 (that wasn't caught previously?) --- conda/install-balsam.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 8d38e816e..0c358c575 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -26,6 +26,7 @@ def move_test_balsam(balsam_test): if not os.path.isfile(reg_dir_with_btest): os.rename('./conda/{}'.format(balsam_test), reg_dir_with_btest) + def configure_coverage(): # Enables coverage of balsam_controller.py if running test coveragerc = './libensemble/tests/.coveragerc' @@ -38,6 +39,7 @@ def configure_coverage(): for line in newlines: f.write(line) + if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ install_balsam() move_test_balsam('test_balsam_hworld.py') From 5d9ceb493f8a1976244994ff0d6d0e7c9f2a3b38 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 08:07:37 -0500 Subject: [PATCH 312/644] Removing .svg errors --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 25373417b..58fe4632a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'numpydoc'] #extensions = ['sphinx.ext.autodoc', 'breathe'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } From b65d1b05acd011c695b0b92b032e3854f4b15e39 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 08:13:20 -0500 Subject: [PATCH 313/644] Wrapping/harmonizing README and quickstart --- README.rst | 69 +++++++++++++++++++++++++++++---------------- docs/quickstart.rst | 58 +++++++++++++++++++------------------ 2 files changed, 76 insertions(+), 51 deletions(-) diff --git a/README.rst b/README.rst index 94fcca7d1..7105a700a 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ .. image:: docs/images/libE_logo.png - :alt: libEnsemble + :alt: libEnsemble | @@ -22,15 +22,16 @@ What is libEnsemble? ==================== -libEnsemble is a Python library to coordinate the concurrent evaluation of ensembles of computations. -Designed with flexibility in mind, libEnsemble can utilize massively parallel resources to accelerate -the solution of design, decision, and inference problems. +libEnsemble is a Python library to coordinate the concurrent evaluation of +ensembles of computations. Designed with flexibility in mind, libEnsemble can +utilize massively parallel resources to accelerate the solution of design, +decision, and inference problems. libEnsemble aims for: • Extreme scaling -• Fault tolerance -• Monitoring/killing jobs (recovers resources) +• Resilience/fault tolerance +• Monitoring/killing jobs (and recovering resources) • Portability and flexibility • Exploitation of persistent data/control flow. @@ -66,8 +67,9 @@ Optional dependency: From v0.2.0, libEnsemble has the option of using the Balsam job manager. This is required for running libEnsemble on the compute nodes of some supercomputing -platforms (eg. Cray XC40); platforms that do not support launching jobs from compute nodes. -Note that as of v0.5.0, libEnsemble can also be run on the launch nodes using multiprocessing. +platforms (e.g., Cray XC40); platforms that do not support launching jobs from +compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the +launch nodes using multiprocessing. The example sim and gen functions and tests require the following dependencies: @@ -77,17 +79,9 @@ The example sim and gen functions and tests require the following dependencies: * NLopt_ - Installed with `shared libraries enabled `_. PETSc and NLopt must be built with shared libraries enabled and present in -sys.path (eg. via setting the PYTHONPATH environment variable). NLopt should -produce a file nlopt.py if Python is found on the system. +``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt +should produce a file nlopt.py if Python is found on the system. -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _Python: http://www.python.org -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _SciPy: http://www.scipy.org -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _Balsam: https://www.alcf.anl.gov/balsam Installation @@ -104,6 +98,10 @@ libEnsemble is also available in the Spack_ distribution. It can be installed fr .. _Spack: https://spack.readthedocs.io/en/latest The tests and examples can be accessed in the `GitHub `_ repository. +If necessary, you may install all optional dependencies (listed above) at once with:: + + pip install libensemble[extras] + A `tarball `_ of the most recent release is also available. @@ -115,7 +113,7 @@ regularly on: * `Travis CI `_ -The test suite requires the mock, pytest, pytest-cov and pytest-timeout +The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ packages to be installed and can be run from the libensemble/tests directory of the source distribution by running:: @@ -129,17 +127,25 @@ Further options are available. To see a complete list of options run:: ./run-tests.sh -h +If you have the source distribution, you can download (but not install) the testing +prerequisites and run the tests with:: + + python setup.py test + +in the top-level directory containing the setup script. + Coverage reports are produced separately for unit tests and regression tests under the relevant directories. For parallel tests, the union of all processors is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after running the tests via the html file -libensemble/tests/cov_merge/index.html. The Travis CI coverage results are -given online at +which can be viewed after running the tests via the HTML file +``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are +available online at `Coveralls `_. Note: The job_controller tests can be run using the direct-launch or -Balsam job controllers. However, currently only the direct-launch versions can -be run on Travis CI, which reduces the test coverage results. +Balsam job controllers. Although only the direct-launch versions can +be run on Travis CI, Balsam integration with libEnsemble is now tested via +``test_balsam_hworld.py``. Basic Usage @@ -186,3 +192,18 @@ or email questions to: or communicate (and establish a private channel, if desired) at: * https://libensemble.slack.com + +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _Python: http://www.python.org +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org +.. _SciPy: http://www.scipy.org +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _SWIG: http://swig.org/ +.. _mock: https://pypi.org/project/mock +.. _pytest: https://pypi.org/project/pytest/ +.. _pytest-cov: https://pypi.org/project/pytest-cov/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ + diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d74f4061c..f46cb3995 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,5 +1,5 @@ .. image:: images/libE_logo.png - :alt: libEnsemble + :alt: libEnsemble | @@ -22,34 +22,37 @@ What is libEnsemble? ==================== -libEnsemble is a Python library to coordinate the concurrent evaluation of ensembles -of computations. Designed with flexibility in mind, libEnsemble can utilize massively -parallel resources to accelerate the solution of design, decision, and inference problems. - -A visual overview is given in the libEnsemble poster_. - -.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 +libEnsemble is a Python library to coordinate the concurrent evaluation of +ensembles of computations. Designed with flexibility in mind, libEnsemble can +utilize massively parallel resources to accelerate the solution of design, +decision, and inference problems. libEnsemble aims for: • Extreme scaling -• Resilience/Fault tolerance +• Resilience/fault tolerance • Monitoring/killing jobs (and recovering resources) • Portability and flexibility • Exploitation of persistent data/control flow. -The user selects or supplies a generation function that produces simulation input -as well as a simulation function that performs and monitors the simulations. The -generation function may contain, for example, an optimization method to generate -new simulation parameters on-the-fly and based on the results of previous simulations. -Examples and templates of these functions are included in the library. +The user selects or supplies a generation function that produces simulation +input as well as a simulation function that performs and monitors the +simulations. The generation function may contain, for example, an optimization +method to generate new simulation parameters on-the-fly and based on the +results of previous simulations. Examples and templates of these functions are +included in the library. + +libEnsemble employs a manager-worker scheme that can run on various +communication media (including MPI, multiprocessing, and TCP). Each worker can +control and monitor any level of work from small sub-node jobs to huge +many-node simulations. A job controller interface is provided to ensure scripts +are portable, resilient and flexible; it also enables automatic detection of +the nodes and cores in a system and can split up jobs automatically if resource +data isn't supplied. + +A visual overview is given in the libEnsemble poster_. -libEnsemble employs a manager-worker scheme that can run on various communication -media (including MPI, multiprocessing, and TCP). Each worker can control and monitor -any level of work from small sub-node jobs to huge many-node simulations. A job -controller interface is provided to ensure scripts are portable, resilient and -flexible; it also enables automatic detection of the nodes and cores in a system -and can split up jobs automatically if resource data isn't supplied. +.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 Dependencies @@ -76,8 +79,8 @@ Optional dependency: From v0.2.0, libEnsemble has the option of using the Balsam job manager. This is required for running libEnsemble on the compute nodes of some supercomputing platforms (e.g., Cray XC40); platforms that do not support launching jobs from -compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the launch -nodes using multiprocessing. +compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the +launch nodes using multiprocessing. The example sim and gen functions and tests require the following dependencies: @@ -87,9 +90,9 @@ The example sim and gen functions and tests require the following dependencies: * NLopt_ - Installed with `shared libraries enabled `_. PETSc and NLopt must be built with shared libraries enabled and present in -sys.path (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt should -produce a file nlopt.py if Python is found on the system. NLopt may also require SWIG_ -to be installed on certain systems. +``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt +should produce a file nlopt.py if Python is found on the system. NLopt may also +require SWIG_ to be installed on certain systems. Installation @@ -185,6 +188,9 @@ function used in the regression tests, which can be found in See the `user-guide `_ for more information. +.. include:: ../README.rst + :start-after: docs-include-tag + .. _PETSc: http://www.mcs.anl.gov/petsc .. _Python: http://www.python.org .. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt @@ -199,5 +205,3 @@ See the `user-guide Date: Fri, 4 Oct 2019 08:36:41 -0500 Subject: [PATCH 314/644] Wrapping/editing user_guide. --- docs/user_guide.rst | 121 +++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 135e6b959..cbdbe20fb 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -13,16 +13,15 @@ three routines: * :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. * :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. -Example ``sim_f``, ``gen_f``, and ``alloc_f`` routines can be found in the -``examples/sim_funcs/``, ``examples/gen_funcs/``, and ``examples/alloc_funcs/`` directories, -respectively. Examples of scripts used for calling libEnsemble can be found in -``examples/calling_scripts/``. To enable portability, a :doc:`job_controller` -interface is supplied for users to launch and monitor jobs in their user-provided ``sim_f`` and -``gen_f`` routines. +Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts +be found in the ``examples/`` directory. To enable portability, a +:doc:`job_controller` +interface is supplied for users to launch and monitor jobs in their +user-provided ``sim_f`` and ``gen_f`` routines. -The default ``alloc_f`` tells each available worker to call ``sim_f`` with the highest -priority unit of work from ``gen_f``. If a worker is idle and there is no ``gen_f`` -output to give, the worker is told to call ``gen_f``. +The default ``alloc_f`` tells each available worker to call ``sim_f`` with the +highest priority unit of work from ``gen_f``. If a worker is idle and there is +no ``gen_f`` output to give, the worker is told to call ``gen_f``. Expected use cases @@ -34,36 +33,36 @@ to support) and plan to have examples of: * A user is looking to optimize a simulation calculation. The simulation may already be using parallel resources, but not a large fraction of some computer. libEnsemble can coordinate the concurrent evaluation of the - simulation ``sim_f`` at various parameter values and ``gen_f`` would return candidate - parameter values (possibly after each ``sim_f`` output). + simulation ``sim_f`` at various parameter values and ``gen_f`` would return + candidate parameter values (possibly after each ``sim_f`` output). * A user has a ``gen_f`` that produces different meshes to be used within a - ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or produce a new - mesh. libEnsemble can ensure that the calculated meshes can be used by - multiple simulations without requiring movement of data. + ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or + produce a new mesh. libEnsemble can ensure that the calculated meshes can be + used by multiple simulations without requiring movement of data. -* A user is attempting to sample a simulation ``sim_f`` at some parameter values, - many of which will cause the simulation to fail. libEnsemble can stop +* A user is attempting to sample a simulation ``sim_f`` at some parameter + values, many of which will cause the simulation to fail. libEnsemble can stop unresponsive evaluations, and recover computational resources for future - evaluations. ``gen_f`` can possibly update the sampling after discovering regions - where evaluations of ``sim_f`` fail. - -* A user has a simulation ``sim_f`` that requires calculating multiple expensive - quantities, some of which depend on other quantities. ``sim_f`` can observe - intermediate quantities in order to stop related calculations and preempt - future calculations associated with poor parameter values. - -* A user has a ``sim_f`` with multiple fidelities, with the - higher-fidelity evaluations requiring more computational resources, and a - ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and at what - fidelity level. libEnsemble can coordinate these evaluations without + evaluations. ``gen_f`` can possibly update the sampling after discovering + regions where evaluations of ``sim_f`` fail. + +* A user has a simulation ``sim_f`` that requires calculating multiple + expensive quantities, some of which depend on other quantities. ``sim_f`` can + observe intermediate quantities in order to stop related calculations and + preempt future calculations associated with poor parameter values. + +* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity + evaluations requiring more computational resources, and a + ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and + at what fidelity level. libEnsemble can coordinate these evaluations without requiring the user know parallel programming. * A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, sensitivity analysis is desired at each identified optimum. libEnsemble can - use the points from the APOSMM ``gen_f`` to identify optima; and after a point is - ruled to be an optimum, a different ``gen_f`` can produce a collection of - parameters necessary for sensitivity analysis of ``sim_f``. + use the points from the APOSMM ``gen_f`` to identify optima; and after a + point is ruled to be an optimum, a different ``gen_f`` can produce a + collection of parameters necessary for sensitivity analysis of ``sim_f``. Naturally, combinations of these use cases are supported as well. An example of @@ -74,24 +73,25 @@ relies on simulations that fail frequently. The libEnsemble History Array ----------------------------- -libEnsemble uses a NumPy structured array :ref:`H` to store output from ``gen_f`` and -corresponding ``sim_f`` output. Similarly, ``gen_f`` and ``sim_f`` are expected to return -output in NumPy structured arrays. The names of the fields to be given as input -to ``gen_f`` and ``sim_f`` must be an output from ``gen_f`` or ``sim_f``. In addition to the -fields output from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble -will include the following fields: +libEnsemble uses a NumPy structured array :ref:`H` to +store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, +``gen_f`` and ``sim_f`` are expected to return output in NumPy structured +arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` +must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output +from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will +include the following fields: -* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an associated - ``sim_id``. The generator can assign this, but users must be careful to ensure - points are added in order. For example, ``if alloc_f`` allows for two ``gen_f`` - instances to be running simultaneously, ``alloc_f`` should ensure that both don’t - generate points with the same ``sim_id``. +* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an + associated ``sim_id``. The generator can assign this, but users must be + careful to ensure points are added in order. For example, ``if alloc_f`` + allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` + should ensure that both don’t generate points with the same ``sim_id``. -* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble worker to be - evaluated yet? +* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble + worker to be evaluated yet? -* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` output - given to a worker? +* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` + output given to a worker? * ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. @@ -107,19 +107,24 @@ will include the following fields: libEnsemble Output ------------------ -The history array and :ref:`persis_info` dictionary are returned to the user by libEnsemble. -If libEnsemble aborts on an exception, these structures are dumped to files. The -existing history array is dumped to a file named ``libE_history_at_abort_.npy``, -and ``persis_info`` to ``libE_history_at_abort_.pickle``, where ``sim_count`` is -the number of points evaluated. +The history array :ref:`H` and +:ref:`persis_info` dictionary are returned to the user +by libEnsemble. If libEnsemble aborts on an exception, these structures are +dumped to the respective files, + +* ``libE_history_at_abort_.npy`` +* ``libE_history_at_abort_.pickle`` + +where ``sim_count`` is the number of points evaluated. Other libEnsemble files produced by default are: -``libE_stats.txt``: This contains a one-line summary of all user calculations. Each -calculation summary is sent by workers to the manager and printed as the run progresses. +* ``libE_stats.txt``: This contains a one-line summary of all user + calculations. Each calculation summary is sent by workers to the manager and + printed as the run progresses. -``ensemble.log``: This is the logging output from libEnsemble. The default logging -is at INFO level. To gain additional diagnostics logging level can be set to DEBUG. -If this file is not removed, multiple runs will append output. Messages at or above -level MANAGER_WARNING are also copied to stderr to alert the user promptly. -For more info, see :doc:`Logging`. +* ``ensemble.log``: This is the logging output from libEnsemble. The default + logging is at INFO level. To gain additional diagnostics logging level can be + set to DEBUG. If this file is not removed, multiple runs will append output. + Messages at or above level MANAGER_WARNING are also copied to stderr to alert + the user promptly. For more info, see :doc:`Logging`. From 1bc48def78cca713d783117c999ae1ae80a17e1d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 08:58:43 -0500 Subject: [PATCH 315/644] Removing badges from PDF (only HTML) --- docs/index.rst | 24 +++++++++++++----------- docs/quickstart.rst | 27 ++++++++++++++------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 85870d4e4..6b81824f4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,22 +5,24 @@ .. image:: images/libE_logo.png :alt: libEnsemble -| -.. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue - :target: https://pypi.org/project/libensemble +.. only::html + | -.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master - :target: https://travis-ci.org/Libensemble/libensemble + .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue + :target: https://pypi.org/project/libensemble -.. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master - :target: https://coveralls.io/github/Libensemble/libensemble?branch=master + .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master + :target: https://travis-ci.org/Libensemble/libensemble -.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 - :target: https://libensemble.readthedocs.org/en/latest/ - :alt: Documentation Status + .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master + :target: https://coveralls.io/github/Libensemble/libensemble?branch=master -| + .. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 + :target: https://libensemble.readthedocs.org/en/latest/ + :alt: Documentation Status + + | ======================================= Welcome to libEnsemble's documentation! diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f46cb3995..cd02cc87a 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,22 +1,23 @@ -.. image:: images/libE_logo.png - :alt: libEnsemble +.. only::html + .. image:: images/libE_logo.png + :alt: libEnsemble -| + | -.. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue - :target: https://pypi.org/project/libensemble + .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue + :target: https://pypi.org/project/libensemble -.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master - :target: https://travis-ci.org/Libensemble/libensemble + .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master + :target: https://travis-ci.org/Libensemble/libensemble -.. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master - :target: https://coveralls.io/github/Libensemble/libensemble?branch=master + .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master + :target: https://coveralls.io/github/Libensemble/libensemble?branch=master -.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 - :target: https://libensemble.readthedocs.org/en/latest/ - :alt: Documentation Status + .. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 + :target: https://libensemble.readthedocs.org/en/latest/ + :alt: Documentation Status -| + | ==================== What is libEnsemble? From 11dc2495d89c6ee3b19479708c341b851be039b5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 09:05:54 -0500 Subject: [PATCH 316/644] early install other blas --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 574b3d4da..7c9bd938a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ install: # mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; - conda install $COMPILERS #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - - conda install libopenblas + - conda install libblas libopenblas # Attempt to prevent 'File exists' error - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI # pip install these as the conda installs downgrade pytest on python3.4 - pip install flake8 From 8e8560eff9c3d0fb20e2e71ec47207f777a6c3cd Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 10:24:45 -0500 Subject: [PATCH 317/644] refactoring --- libensemble/env_resources.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 9c406ebc8..25390d020 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -106,7 +106,8 @@ def _range_split(s): return a, b, nnum_len @staticmethod - def _noderange_split(prefix, nidstr, nidlst): + def _noderange_split(prefix, nidstr): + nidlst = [] for nidgroup in nidstr.split(','): a, b, nnum_len = EnvResources._range_split(nidgroup) for nid in range(a, b): @@ -126,8 +127,7 @@ def get_slurm_nodelist(node_list_env): return splitstr prefix = splitstr[0] nidstr = splitstr[1].strip("]") - nidlst = EnvResources._noderange_split(prefix, nidstr, []) - return sorted(nidlst) + nidlst = EnvResources._noderange_split(prefix, nidstr) else: splitgroups = [str.split('[', 1) for str in splitstr] prefixgroups = [group[0] for group in splitgroups] @@ -136,8 +136,9 @@ def get_slurm_nodelist(node_list_env): for i in range(len(prefixgroups)): prefix = prefixgroups[i] nidstr = nodegroups[i] - nidlst.extend(EnvResources._noderange_split(prefix, nidstr, nidlst)) - return sorted(set(nidlst)) + nidlst.extend(EnvResources._noderange_split(prefix, nidstr)) + + return sorted(nidlst) @staticmethod def get_cobalt_nodelist(node_list_env): From 4ed85170572f645c315379b94440094bed856e93 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 10:38:38 -0500 Subject: [PATCH 318/644] comments, formatting --- libensemble/env_resources.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 25390d020..9ce951e04 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -33,6 +33,7 @@ class EnvResources: default_nodelist_env_lsf = 'LSB_HOSTS' default_nodelist_env_lsf_shortform = 'LSB_MCPU_HOSTS' + def __init__(self, nodelist_env_slurm=None, nodelist_env_cobalt=None, @@ -76,6 +77,7 @@ def __init__(self, self.ndlist_funcs['LSF'] = EnvResources.get_lsf_nodelist self.ndlist_funcs['LSF_shortform'] = EnvResources.get_lsf_nodelist_frm_shortform + def get_nodelist(self): """Return nodelist from environment or an empty list""" for env, env_var in self.nodelists.items(): @@ -87,12 +89,14 @@ def get_nodelist(self): return global_nodelist return [] + def abbrev_nodenames(self, node_list): """Return nodelist with entries in abbreviated form""" if self.schedular == 'Cobalt': return EnvResources.cobalt_abbrev_nodenames(node_list) return node_list + @staticmethod def _range_split(s): """Split ID range string.""" @@ -105,8 +109,10 @@ def _range_split(s): b = b + 1 return a, b, nnum_len + @staticmethod - def _noderange_split(prefix, nidstr): + def _noderange_append(prefix, nidstr): + """Format and append nodes to overall nodelist""" nidlst = [] for nidgroup in nidstr.split(','): a, b, nnum_len = EnvResources._range_split(nidgroup) @@ -114,21 +120,22 @@ def _noderange_split(prefix, nidstr): nidlst.append(prefix + str(nid).zfill(nnum_len)) return nidlst + @staticmethod def get_slurm_nodelist(node_list_env): """Get global libEnsemble nodelist from the Slurm environment""" fullstr = os.environ[node_list_env] if not fullstr: return [] - splitstr = fullstr.split('],') - if len(splitstr) == 1: + part_splitstr = fullstr.split('],') + if len(part_splitstr) == 1: # Single Partition splitstr = fullstr.split('[', 1) - if len(splitstr) == 1: + if len(splitstr) == 1: # Single Node return splitstr prefix = splitstr[0] nidstr = splitstr[1].strip("]") - nidlst = EnvResources._noderange_split(prefix, nidstr) - else: + nidlst = EnvResources._noderange_append(prefix, nidstr) + else: # Multiple Partitions splitgroups = [str.split('[', 1) for str in splitstr] prefixgroups = [group[0] for group in splitgroups] nodegroups = [group[1].strip(']') for group in splitgroups] @@ -136,10 +143,11 @@ def get_slurm_nodelist(node_list_env): for i in range(len(prefixgroups)): prefix = prefixgroups[i] nidstr = nodegroups[i] - nidlst.extend(EnvResources._noderange_split(prefix, nidstr)) + nidlst.extend(EnvResources._noderange_append(prefix, nidstr)) return sorted(nidlst) + @staticmethod def get_cobalt_nodelist(node_list_env): """Get global libEnsemble nodelist from the Cobalt environment""" @@ -153,6 +161,7 @@ def get_cobalt_nodelist(node_list_env): nidlst.append(str(nid)) return sorted(nidlst, key=int) + @staticmethod def cobalt_abbrev_nodenames(node_list, prefix='nid'): """Return nodelist with prefix and leading zeros stripped""" @@ -160,6 +169,7 @@ def cobalt_abbrev_nodenames(node_list, prefix='nid'): newlist = [s.lstrip('0') for s in newlist] return newlist + @staticmethod def get_lsf_nodelist(node_list_env): """Get global libEnsemble nodelist from the LSF environment""" @@ -170,6 +180,7 @@ def get_lsf_nodelist(node_list_env): nodes = [n for n in unique_entries if 'batch' not in n] return nodes + @staticmethod def get_lsf_nodelist_frm_shortform(node_list_env): """Get global libEnsemble nodelist from the LSF environment from short-form version""" From 155ba0c0517046e8ec305e76862bc6f0a44fafc7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 10:40:02 -0500 Subject: [PATCH 319/644] rename var --- libensemble/env_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index 9ce951e04..bcccee3ec 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -136,7 +136,7 @@ def get_slurm_nodelist(node_list_env): nidstr = splitstr[1].strip("]") nidlst = EnvResources._noderange_append(prefix, nidstr) else: # Multiple Partitions - splitgroups = [str.split('[', 1) for str in splitstr] + splitgroups = [str.split('[', 1) for str in part_splitstr] prefixgroups = [group[0] for group in splitgroups] nodegroups = [group[1].strip(']') for group in splitgroups] nidlst = [] From 6205eeef6aeec5d7809f1c0e964be09c8f9b50f5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 10:53:47 -0500 Subject: [PATCH 320/644] flake8 --- libensemble/env_resources.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libensemble/env_resources.py b/libensemble/env_resources.py index bcccee3ec..b818780cb 100644 --- a/libensemble/env_resources.py +++ b/libensemble/env_resources.py @@ -33,7 +33,6 @@ class EnvResources: default_nodelist_env_lsf = 'LSB_HOSTS' default_nodelist_env_lsf_shortform = 'LSB_MCPU_HOSTS' - def __init__(self, nodelist_env_slurm=None, nodelist_env_cobalt=None, @@ -77,7 +76,6 @@ def __init__(self, self.ndlist_funcs['LSF'] = EnvResources.get_lsf_nodelist self.ndlist_funcs['LSF_shortform'] = EnvResources.get_lsf_nodelist_frm_shortform - def get_nodelist(self): """Return nodelist from environment or an empty list""" for env, env_var in self.nodelists.items(): @@ -89,14 +87,12 @@ def get_nodelist(self): return global_nodelist return [] - def abbrev_nodenames(self, node_list): """Return nodelist with entries in abbreviated form""" if self.schedular == 'Cobalt': return EnvResources.cobalt_abbrev_nodenames(node_list) return node_list - @staticmethod def _range_split(s): """Split ID range string.""" @@ -109,7 +105,6 @@ def _range_split(s): b = b + 1 return a, b, nnum_len - @staticmethod def _noderange_append(prefix, nidstr): """Format and append nodes to overall nodelist""" @@ -120,7 +115,6 @@ def _noderange_append(prefix, nidstr): nidlst.append(prefix + str(nid).zfill(nnum_len)) return nidlst - @staticmethod def get_slurm_nodelist(node_list_env): """Get global libEnsemble nodelist from the Slurm environment""" @@ -147,7 +141,6 @@ def get_slurm_nodelist(node_list_env): return sorted(nidlst) - @staticmethod def get_cobalt_nodelist(node_list_env): """Get global libEnsemble nodelist from the Cobalt environment""" @@ -161,7 +154,6 @@ def get_cobalt_nodelist(node_list_env): nidlst.append(str(nid)) return sorted(nidlst, key=int) - @staticmethod def cobalt_abbrev_nodenames(node_list, prefix='nid'): """Return nodelist with prefix and leading zeros stripped""" @@ -169,7 +161,6 @@ def cobalt_abbrev_nodenames(node_list, prefix='nid'): newlist = [s.lstrip('0') for s in newlist] return newlist - @staticmethod def get_lsf_nodelist(node_list_env): """Get global libEnsemble nodelist from the LSF environment""" @@ -180,7 +171,6 @@ def get_lsf_nodelist(node_list_env): nodes = [n for n in unique_entries if 'batch' not in n] return nodes - @staticmethod def get_lsf_nodelist_frm_shortform(node_list_env): """Get global libEnsemble nodelist from the LSF environment from short-form version""" From 713319765c87c808ead1ebf69b280c42290f86eb Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 11:38:13 -0500 Subject: [PATCH 321/644] Adding better 'basic usage' section to README --- README.rst | 54 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 7105a700a..c57e58f62 100644 --- a/README.rst +++ b/README.rst @@ -35,9 +35,20 @@ libEnsemble aims for: • Portability and flexibility • Exploitation of persistent data/control flow. -A more detailed overview can be found in the docs_. - -.. _docs: https://libensemble.readthedocs.org/en/latest/ +The user selects or supplies a generation function that produces simulation +input as well as a simulation function that performs and monitors the +simulations. The generation function may contain, for example, an optimization +method to generate new simulation parameters on-the-fly and based on the +results of previous simulations. Examples and templates of these functions are +included in the library. + +libEnsemble employs a manager-worker scheme that can run on various +communication media (including MPI, multiprocessing, and TCP). Each worker can +control and monitor any level of work from small sub-node jobs to huge +many-node simulations. A job controller interface is provided to ensure scripts +are portable, resilient and flexible; it also enables automatic detection of +the nodes and cores in a system and can split up jobs automatically if resource +data isn't supplied. A visual overview is given in the libEnsemble poster_. @@ -80,8 +91,8 @@ The example sim and gen functions and tests require the following dependencies: PETSc and NLopt must be built with shared libraries enabled and present in ``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file nlopt.py if Python is found on the system. - +should produce a file nlopt.py if Python is found on the system. NLopt may also +require SWIG_ to be installed on certain systems. Installation @@ -154,9 +165,38 @@ Basic Usage The examples directory contains example libEnsemble calling scripts, sim functions, gen functions, alloc functions and job submission scripts. -See the `user-guide `_ for more information. +The user creates a python script to call the libEnsemble :doc:`libE +` function. This must supply the +:ref:`sim_specs` and +:ref:`gen_specs`, and optionally +:ref:`libE_specs`, +:ref:`alloc_specs` and +:ref:`persis_info`. + +The default manager/worker communications mode is MPI. The user script is +launched as:: + + mpiexec -np N python myscript.py + +where ``N`` is the number of processors. This will launch one manager and +``N-1`` workers. + +If running in local mode, which uses Python's multiprocessing module, the +'local' comms option and the number of workers must be specified in +:ref:`libE_specs`. The script can then be run as a +regular python script:: + + python myscript.py + +When specifying these options via command line options, one may use the +``parse_args`` function used in the regression tests, which can be found in +``libensemble/tests/regression_tests/common.py`` + + +See the +`user-guide`_ for +more information. -.. docs-include-tag Documentation ------------- From 0c9263ae3e8fe6c0a110b8912fed0ae2b340928c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 11:47:19 -0500 Subject: [PATCH 322/644] Using README inside quickstart --- README.rst | 10 ++- docs/quickstart.rst | 186 +------------------------------------------- 2 files changed, 7 insertions(+), 189 deletions(-) diff --git a/README.rst b/README.rst index c57e58f62..f45f2dcef 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,7 @@ :alt: Documentation Status | +.. after_badges_tag ==================== What is libEnsemble? @@ -193,9 +194,9 @@ When specifying these options via command line options, one may use the ``libensemble/tests/regression_tests/common.py`` -See the -`user-guide`_ for -more information. +See the +`user-guide `_ +for more information. Documentation @@ -210,7 +211,8 @@ Please use the following to cite libEnsemble in a publication: .. code-block:: bibtex @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and David Bindel and John-Luke Navarro}, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, title = {{libEnsemble} Users Manual}, institution = {Argonne National Laboratory}, number = {Revision 0.5.2}, diff --git a/docs/quickstart.rst b/docs/quickstart.rst index cd02cc87a..755fd6fbd 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -19,190 +19,6 @@ | -==================== -What is libEnsemble? -==================== - -libEnsemble is a Python library to coordinate the concurrent evaluation of -ensembles of computations. Designed with flexibility in mind, libEnsemble can -utilize massively parallel resources to accelerate the solution of design, -decision, and inference problems. - -libEnsemble aims for: - -• Extreme scaling -• Resilience/fault tolerance -• Monitoring/killing jobs (and recovering resources) -• Portability and flexibility -• Exploitation of persistent data/control flow. - -The user selects or supplies a generation function that produces simulation -input as well as a simulation function that performs and monitors the -simulations. The generation function may contain, for example, an optimization -method to generate new simulation parameters on-the-fly and based on the -results of previous simulations. Examples and templates of these functions are -included in the library. - -libEnsemble employs a manager-worker scheme that can run on various -communication media (including MPI, multiprocessing, and TCP). Each worker can -control and monitor any level of work from small sub-node jobs to huge -many-node simulations. A job controller interface is provided to ensure scripts -are portable, resilient and flexible; it also enables automatic detection of -the nodes and cores in a system and can split up jobs automatically if resource -data isn't supplied. - -A visual overview is given in the libEnsemble poster_. - -.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 - - -Dependencies ------------- - -Required dependencies: - -* Python_ 3.5 or above. - -* NumPy_ - -For libEnsemble running with the mpi4py parallelism: - -* A functional MPI 1.x/2.x/3.x implementation such as `MPICH - `_ built with shared/dynamic libraries. - -* mpi4py_ v2.0.0 or above - - -Optional dependency: - -* Balsam_ - -From v0.2.0, libEnsemble has the option of using the Balsam job manager. This -is required for running libEnsemble on the compute nodes of some supercomputing -platforms (e.g., Cray XC40); platforms that do not support launching jobs from -compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the -launch nodes using multiprocessing. - -The example sim and gen functions and tests require the following dependencies: - -* SciPy_ -* petsc4py_ -* PETSc_ - This can optionally be installed by pip along with petsc4py -* NLopt_ - Installed with `shared libraries enabled `_. - -PETSc and NLopt must be built with shared libraries enabled and present in -``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file nlopt.py if Python is found on the system. NLopt may also -require SWIG_ to be installed on certain systems. - - -Installation ------------- - -Use pip to install libEnsemble and its dependencies:: - - pip install libensemble - -libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: - - spack install py-libensemble - -.. _Spack: https://spack.readthedocs.io/en/latest - -The tests and examples can be accessed in the `GitHub `_ repository. -If necessary, you may install all optional dependencies (listed above) at once with:: - - pip install libensemble[extras] - -A `tarball `_ of the most recent release is also available. - - -Testing ---------- - -The provided test suite includes both unit and regression tests and is run -regularly on: - -* `Travis CI `_ - -The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ -packages to be installed and can be run from the libensemble/tests directory of -the source distribution by running:: - - ./run-tests.sh - -To clean the test repositories run:: - - ./run-tests.sh -c - -Further options are available. To see a complete list of options run:: - - ./run-tests.sh -h - -If you have the source distribution, you can download (but not install) the testing -prerequisites and run the tests with:: - - python setup.py test - -in the top-level directory containing the setup script. - -Coverage reports are produced separately for unit tests and regression tests -under the relevant directories. For parallel tests, the union of all processors -is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after running the tests via the HTML file -``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are -available online at -`Coveralls `_. - -Note: The job_controller tests can be run using the direct-launch or -Balsam job controllers. Although only the direct-launch versions can -be run on Travis CI, Balsam integration with libEnsemble is now tested via -``test_balsam_hworld.py``. - - -Basic Usage ------------ - -The examples directory contains example libEnsemble calling scripts, sim -functions, gen functions, alloc functions and job submission scripts. - -The user creates a python script to call the libEnsemble :doc:`libE ` function. -This must supply the :ref:`sim_specs` and :ref:`gen_specs`, -and optionally :ref:`libE_specs`, :ref:`alloc_specs` and :ref:`persis_info`. - -The default manager/worker communications mode is MPI. The user script is launched as:: - - mpiexec -np N python myscript.py - -where ``N`` is the number of processors. This will launch one manager and ``N-1`` workers. - -If running in local mode, which uses Python's multiprocessing module, the 'local' -comms option and the number of workers must be specified in :ref:`libE_specs`. -The script can then be run as a regular python script:: - - python myscript.py - -When specifying these options via command line options, one may use the ``parse_args`` -function used in the regression tests, which can be found in -``libensemble/tests/regression_tests/common.py`` - - -See the `user-guide `_ for more information. - .. include:: ../README.rst - :start-after: docs-include-tag - -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _Python: http://www.python.org -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _SciPy: http://www.scipy.org -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _Balsam: https://www.alcf.anl.gov/balsam -.. _SWIG: http://swig.org/ -.. _mock: https://pypi.org/project/mock -.. _pytest: https://pypi.org/project/pytest/ -.. _pytest-cov: https://pypi.org/project/pytest-cov/ -.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ + :start-after: after_badges_tag From bbe1bab0bfe6806d9b4668b772e16f0981133431 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 11:58:43 -0500 Subject: [PATCH 323/644] Better logo quality --- docs/conf.py | 1 + docs/images/libE_logo.png | Bin 10227 -> 54153 bytes 2 files changed, 1 insertion(+) mode change 100644 => 100755 docs/images/libE_logo.png diff --git a/docs/conf.py b/docs/conf.py index 58fe4632a..0a0e70d4b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -188,6 +188,7 @@ def __getattr__(cls, name): 'Stephen Hudson, Jeffrey Larson, Stefan M. Wild, \\\\ \\hfill David Bindel, John-Luke Navarro', 'manual'), ] +latex_logo = 'images/libEnsemble_Logo.png' # -- Options for manual page output --------------------------------------- diff --git a/docs/images/libE_logo.png b/docs/images/libE_logo.png old mode 100644 new mode 100755 index 81d1e2589e340395778ed9251dfd4b84abf032c9..17f051faab46f91a58f98c4dc617304e97707fcf GIT binary patch literal 54153 zcmeFY^;?wP8$C)Wr3ebb5EAMOQqo-tQpyY>-7O%Ee!l1X7n~o?To>0c%=7Ge*1hg^ue}L>3{xT}qbI||!y{Kwe((ejk1Prg zk3g4%2>j)ViFP4)BXw5Rdya>9_tyCjzE_T{2Ob^^p2~y!nqDctC)FGobm}%|;}n02 z=$QI(y@p@ZuEj6HClIdugxA(n{4~KjMURNq)LS|5UN1wwGS{ny7YW+;>@J=<4=8<( z7z@ZQ)Uedr^2Teed5$pusI2~bFLncqPUJNq0A2riweM2I^58z;wUm=8;{W|b1mim& z{eLe$=NIa@-|+&2sG|PY$mJAM>c3x^&RjtJ{eVZv4!QXE6GW`_&fgC$E&qR4xW)MY z7ZA4+c>mvr#RQVcG|-W&(3LlMQz*DRuF26Tlu{;)vXo|iG(PMqZYBo_=}o?eb8~oZ z%IfjNtidOkD(ox4dJ@)@Fv>n~X-?PQn5q2-a&b9ONC9&vn9>5pPS>|A5sF|9ra~+I zjm6eGEkxI~=<;4Mg;A!}c+oJ=IgtGQ*3<`tl1N5=PoB$>HOcHEz0(OaN$hCU;tFms z;|hFaFlwGFib$|1kSH%%A^ggApj{CdS=Bq{>IuUmAC%E33v_2tE<5f$Irx%=91WdygUtrN~$t)=;}A zR{UwHo-OkERqp=&eY(%)_^F%^-duP0qP35~%+nj1Ekl5EDF-6`C*B{7rr zyOB$pDbAQ@@t61rx6a8_Dgp09_Da@U9nCK%F!L>W)Qjs-{v~maUW*M&tG7{njx=cc)h%dlVDA1V{AS5*g zR(eQmh#P#&7hVlAe*nlbe1(;t)X;HiGn$z+KZq1OT=@ zV(0v>HsbVCbe9=;9HEq@cK+D7EY9Q!UfgvVx4?7p79!A`ke; zUMo$SVxEG#YmOl)!Ux7+l7H?s#XG-Ms>J)9wl-?I3W6-9FV6WNQtkod={cy;Ho9x= zV2iVHUj*V)MiFdE|1UE^@c!=kCpJvN-qUNy_LM z=eXxraJ}F$-2U%D(&YuF8i8wF_ESlvfu*7mC%nVFCiU=Ls)MLbTE$?@%G|{l^6NXmvVVv}DAa_#E= z3KLW4>8u>#0)WLFb-$-{W3kJ{)1@mZI2ho)egi^aO$=5vm%h7vWX#=J5qBk}izS+E zCzZ0zpJa9qxqR4&J>m$ad^uT!ac|OxA!|;?aER!iP%~-rb)m!8f2_G^eEC$4L=Eoj zH_v%=1Ff(_5B<3%?);vW=@cR-7HLwn_`~ADpOLp9Kv?K&YFb5Ioc27X@6vUXEg4{c ziLzuOL?OylDZg*&f-r$;_pZcu;e=FR*tIBv%3@;Fav4~KO2Q?S529PDG%42Z#BZ!S&oybAC;JeorGct<3n0`x&*Im~ND{4iomNVw`v4 z&6$JR^~k^Rct8(QP7=UZ!R{g|84&$%ekVtv4_&JVbryNMhAYS_pke52iGB$6y}()q}?r=T`z`3@0&%Pyz*E-swMYsQhzV8u( z2})$=_lS{VyrTu{lvme~3R!Cq=ub6ylC3eB6_jnW9#3A^4Y9xEC-U2x0`FcPpdA5z zYU$m_)xRpOL6U-lv6m#Ed#|M678s8KLqW@&q2VFTI2osnYF=JhNq3uJ#6ft34=`D1 z>xFKQ#k46nRbE?e*V=P^_vS>lZ|l}q!qQ{ecG?GBpAp~y-9iKO^W}EySPMyWdpbF< zgq0aum${{9;?mfjvyHLhPFODqKRVLX}r8TE8DTT z!hH>Z9L@C5JyLbL~J5Vn?d7AP^TO4K~)MNRMlf>$x_ z5FOfuyEr2cAa(NN(#Pc#HoFNY*DqN$PlS$gXU+ac;v-K$d+YgfI47$P@;O9F@!%yfKL`E@M$75GOGvHB8a^F-6*3oEH>BWh= z0uv0bSOWUeI@XNqm+|ltqPtcqOe;$`5P+gO^O4W|v3cL}{s$-!Zbxf9^TSDOabh6WW1gq$Mm$bt z9CpJAzd)XIWg99TD_;Yi43afCl8Alxna~I3r%f759EuLxC%9qTXQoRp<4x;mF2YF6 z*iRmLo^9BiaZUTyLXb?7hZXBb+eqQ&*f4Q&Rw*av5*PsA;4^Ut#%D2XJ6?C|$*;Ho zSR#|s{w9PIkSME-)#I@oBNO#I*7*!%L$4H)YbW-)%B=XyeWz#G=7jpEmEsPa0Q|H2 z@CWFRc0xbeC~tA$Mq~i+U>}mj=_KD@@3L&%zB?tpR=knxm0z3SFZ3aRgtfum3pEA$ ze<|kYQI#Sic83Z2Q;9WrUKNSfZ~gl`+F2kMiP%eCPuEku^5D39l3)!qhdS3vWT6yn zcwu|bjz8_glOw+_x)-R|Y(uBkplQ{(M@aM6@H3P1)N=?2X~$->p?0I)LQV@exKJfN zGR#`Ce5GMW;q(lMOIccamAh zuZPFjY5no7A!%n^TgTC+)XVYIuhypz$JVg7-mjzUe*4XPtSe=wJIrAV9I49S0=rWb7|G@Lqt9vw-q^!U$&wcXaJIwiwjAYBCN~w+R|LNt9hSK!%Ow zV38hASV4ftWGVDvBhP)?&jn&J%#`|9bn*Z=r@p?4g1lnd^Np|>o^ME1?ec!pG(`jI zCEd;fL|){UeM#$YBF5Z1wQIuL;>Xqcu;J^}I2`x8yihl_0JC_6KXqS?prQJA*;e&f zZQs`WvH=V#vkT!P?Op2sib`G5`LEn_15u`1gb%%TlHd-5oK$v>!}D7^L6{udHt^mM zi)Rl1xNKIVhARdG-2vT~aDh4^;z2ACefU&WAK$jR%QHCR(2Aqf5l2UYH4p>OzAwG$ zwWY|7j8MQa*8ykio&F?1_vqWG#L{ig61#s+rji_&?9A^Ri}+vHI^>7mkRG2olUQy_eS3 z>cloeNz%szULGN1+y(L!{7jq_hmHWSx_ThDC*DLg!KfNlJ}wQOlwIkH@*`Jt>c}{7 zKYpm8qz2S2Eusq&DYMez?fbnRD8y&tClietxPfFmx`4QM3lx*N;aNx@*MG`1Y`oJ5 zP{W@QZpEE74dm2$Qh>N6;KrN`+J z3<4iTt;A=xVtTrW;`3H~_6CL>zmGd;_{ucKq7^}Ksc0yPUW)+@rBau9D5SjbNnnd0 z`@61Q@BS^$Bk(ES%ApXL!H(;bv>cSuYC$%n}<{hQupH8mI4vO8YJTnq`g zm~|tK>Bwkh>@@FZjp!A{4PYN^z(z)@w0_%NQbKbc)sRzAvAScnkTP z)9y{4lda1}E1zAR*;ytRc5>F7F?XP2e#?GQMa!PWaX5ZH2SiIcsKck6@6-vu0e93j zm+mwsUsl|BG`l(*${yPk9_KRdbqW%BYGu|bD_XH6>+j9aFCfYn!Of~@s&t2$hrSQ5 zPOXS2WO)jcDDr~L=kH^!jZE!EgbC|>7IlxyjZUj{*(6qF@T<{>#p>64;-LLF&Xl@Z z^E^lR8hHNje9nHYabj<2y8i{L9?@kFN*9J|8FRf zU?^4t7ru)5Z7n8k94bUSP?KIdRa1{&5=Kn+5hFYuoUKk2$dt-lnZ1NQOvm~K00lZq z_?O7Pg4)RdNLJ<&bRQ3lO8OC3TLrO?d_N1suLgpI*Bl-9{XijvWhfA${O^pTto8B& zrap+k9j{B{W?{qz3E_D?giQCf3h^)~MX#m)yn+eMz0Y?9>g*f)hpyCG-V8CNJKo-v z5MkiGt0;Fv@yq>ob+yK>%wogn0-aPJoT;`)5FKIis+$%g`7?nx^ zWx(lP)8z{YJ{H@;W!skp@f21$oqS|!^GxG(7oL5T zDK(L4w1xuj6J6^p8*oRb@bBhfsR3Eq`BAI9^~aCEZAt$tk4#AwGnbyYK`c<#JX2Je zHkXnvAi$%~E1hnhcp^rhzKS$x_>fE;o#yb<7$@WINnSwoTY#3YkCo{+5D$Z>*p=!o z%kOEYY{NXErjS2-VyU*8Sm zVPwEfNB0nEy4x4d9CyG*jUw*AFDS#LbJ=OC0xl z7ukY0%RuWkg*G+sAAY8@9b4wLopZdqWxeO89uor_O-1=CxTg;YLX zKiBRtSdO}JwVJJCt{V3@r&4Nf<`18VUza@HcZnTd+n1QGhamlX0u;wNpnKBYjCKK+ zS>OMiTrnhLzEQTWkX;5SiRU%T=V@{?Lf!hhK;^&677m<* zvXxc(X?O($nN(b{v>L0#fWM!I5wtD5%jdQ7l9K zc3#gAdWIDGemAT(;!vujkr)z1Jz|neAzREvL%!134czv$+>Ft)w}H@U3D3&67d`ww$@L0az=-zV$n6I6W9{JiWltaVZ=Y45WNzl=B- zei(yHZ0T51)yXtf9%?GQY(70;;a`x;fWo4&@<Ds*f+uOmUJ?>HwZHmI{5w3o z9&CHm@isXD30kd^S^byC{F-ndo#v`yZ9JoI3sWiucAXY+A0ZQu9|`GEhUmsQP3fkSpExm` zN^-vZ*xLMjySa#vVZR~_Z|sdYO?1%cET~v|y+296Qjkv)q7Q{c4^^(6{quVPA6Q8q zz%%>0V_KCWlGe!(um^+%60_qfX{#wj7E2r8bRrYD>T$Jw3%He@yL>d!Q=o){Q%0w` z*;EBkImwY2VDHnwok*J1My@n^O@ooNj5*8&jBie>)4heBq{f{Y-53$$TFJb~^Cb%l ze)tK*SG~C5tGAbuw$khvW%snm?wQEUVa^QIQ&y=GIMydfS$%t)#a35>V5_#=a4tne z*reDW_c(B!GD;5Y60KWCDOkL(9v+}>LleWE{eI!aqo%Pi=41QhpV-$-SPAsp-1DV< zlx3Qg^a-$nQ)I+B=r4wrmH{JY-+%fK!j4l-sRbUQYh9sh$M4u8jTil+2C_S8KgW)L zf|M}s{!STxeR9}O1q~=Q1~x7CYWT(j1hW7mOo+@4rXHUs9joA!nnSVu2UGTqERYai zfilO&XoZS0W+%RakC%*{u+NS}28D7HXhp>G*QC7 zLWoNi2D~JStg+udE{9c{qZaOj=SVzzCH{DHL3u%mjsxPR=Q@nNQrOmbi6ZtHL-5mah} z?V!&?btZYQH|(oY{6!safI?p8#?z#0hrXbOEdwUi<*u=Jb?T<;2h|iX!3%xB$*iox z<2?NaYE}W&g>h)y@QX7O+*ts{@yng~P3&EEj}MkpXK8*5!0u^+O!s>?KVtkPePT7d#pO#YQ$MaO&*J}A15HKzgE{6ZYFPVb6-p8HcnwX;w8Jw+W}+FX4^UQw8| z+-{#>r*QmbU>jq9+QIdy0^+A=zdRp(M1^|p%d*rok7>o>zQnTa&X3pTq5E{DlXYy7 zp_kDL6v>@gH3Rh6_R!T83fGiJNc-N0M{$>D2X-a2Q2|h}dSx14F7{VXxWJE3PJw4J zjy2{!I@d%3jWMSKZC_AJE7E`b;TCNHGq>LC=Gy^BsVffyVBY#bJB<66H01#OG0ve& zULr_CPR{BB2E=Wdy05~t_;wuR`gVzwBd6Zq4{5_BQD z6_vTtOKvELH&b?ZW>v?2*2)A-K#f6q><2Nf+!KNFNjP&jN-3LAT>1TE8N(6sX6f7v zu}iHl%mp_^ey`Kw8DmFPO*N2T?9!x53br2A) ziVc-H6nzxc2)r5^i!_!G?>n?DL|24{V z%tKi$^mlI6H@wm8_zkH1i9r}ob948WKKv2R6}A7Tb@|>G6%~Nf#~Am{=7Y0@L=Hyj zse}U_Xcl~OlTVcY@yI=H`lR4^5Y2<8ZdECrt1jw0d`7}qW{l*!C)B3%fG*2G)*a8L*mZUSW#JX&FYHa4aae|zl*8DH}>52#MdzH6bR%B z;pt?^V`CaE@nyWbPz`AZulLD9dX?V?*B$a1TxGPEZZ^lz|8np77;$P|mGbiy%)o`6 zzR}j}xW@J-%e^b&Q|(4&b_A{wPo5X&20Wen!zJg6OFBXB`;O4oa}D$CBPI2CBSXST z-3U_GFZRR^YEwr+rNmE@-`9;vG9kxuU;~3UcR$y;VTEj&ox_qip9lIFnYlu}r=)@&leH3ihdXLCBJl#nR7Vvlk_!JY zUR22xaA)|CuN^Z4HW!O84QD$QZ62Bo%h$1;dvdwUTl3Gv*9KQ%WM2coaxE!nd3aou z?Y|`c0;R?=V_3u3EUWtLPi0meV_In~!5BFO7Y|lJ*#6Q+W4ByDF=8c=ll!Za@pEqM z$tw;MS&_$j=j)_vRm{e3=Y11zD%p-yFuq>9@nUemDrC~FzWf| zGis4r+)avxLQ>!KjwbOob;bRejU1N?muij5QvYOvp{r4QkI=7NQegu^Kc-Zuo_I2( z?5Ar`0M(f&SIuUoP(*e4+`@RN5|>P-m6Xqrj}eJ#;3~;Nev~_%1WDtEWqpddl+Wah zmzsc)TA+l8z)79O#>YST4KB|&Mg@?+`yJ`y*3%gEegM9d8%JTmJy!Mlqe?$&Pyd9i zi+L<*Z)3%R5i6WW*S4=_o8z~Uv%JC*^3%QXPT{Aubd5ia;lW{iV}zi!_?a>vuV3{X zZfhZxL!WII&Mk{n_qEefu=hrEf6P&zHO)xh^XyZr3s%g`^#<3eq%DwB`CAI1&n18F za2{cwm`({DNWq%_y7n_g7Z4}?OYZF;dA)}*6f4Hw1tFD4ILhX-sG|Q*nnB#s-tPIL z2BX-JAEzF=ILD;NigE-pz3cFz380x0u2pk+X18SSo`H5fzA`!fM!L-tX>hF;d@D#yTeHjB&t|ywh{`h%+E;aFnNJBWJf!PC__SEI|x}8y3WZ}fu zFR~e)t_@D3)BUM^F7-wM(|=;(b*gD)`#{}yHVt0+B<+1-qLyFD0>( zQqiJNmrw)D0PI84wT%>`1B1+ZFjH2Z@~gAYRoz9+aO5r_>+@Bz#8!yf0(HU`$2%Xf zkj2R>c)>r+$Pk)tcKt*l%0)2k$v8~l!PhNp8Q9Ohe89w*3xwr~8y3nA4RuR#K-ZW5 z6Vf5xHWeG|UAk+OLSO_o?srWF8@K{Wb2moqsLJWm1&)nn^p}WVMifSEu&03loRJO2 zbDIWkc6a-KyZ*&-q=?BOOZby{UCa?znCI%z+J*{2DaY=!p)X#1Z34rOyo$;NLd@cQ%6imsI2Z{ZJP6zPHb$m>?cV- zc;3JqY?bmfR_j73D%Ygix6+I4wmp+BpGmc6x=uL#*Kgp|)hG|0xbolGIRz?V9Ev4H zf*%d_t;cRZ#ZnF{T1(|rsc0xBcQ#LdN|uXZ$lgl3acrI0z*z3bXt5-?8SV ze}MU~n-&qOD39R^3G3?I_aD0*?BT-ODlgr0hvg040Od4aIY@E78#M|FkdorhYvE0* za=o7-{CnqT$^dn-lk1@Cb6a$B3Rc8rov8yTj*T@JRgN-XJhnHFt$ymBpVV4R%KxJ# z`z2qX%sv~V8*Q(ZIdC66LYEqgh&|W<>j7#GX?sXkNiQWqY^!1!fH+Y&l7Wd0RxA0{ z?V-He$c&6MZ{erzeZux3Ma@&rCTn0!h(XCtG9m9fhYPbf=Y77{(8&Zlr>|uFnU*%z zdz`nWrPz+l%tS{)G@MoDiWT5WP=}lV;W5US?_iK30H4@diF0pHgoX9k9<8ok4vUC& zGRZ9{1)N}_xtL5eECItxGMFD+5sEW5k*VC2nNfj%hZ|`w<_YDTZcq6P<*g82-&$3O zq*w%dp#Swe;=va6q$_v0M^-x@q;`bJNtoASN}s67U*Y6>TD0 znxMgoqyVfpPlV34sCBQbj=OmD^G@TEZLedJ#F0#Car4gZ26cQjU8G{y3-AAWFdZjV z46`y~HHsV8!(v)$rz&UNP$i_7&yQ{>oDRD34g9c;p!QkJT=nu<7E`zfwO4qTsvc+O zE;O@rv;Q8v486oLHN9iYpLY28Ppw_PdjZkDeY@k0gWWX-2lGj{D3AnFkJ|!mj-{ z*1!_l3y4(hecn}yA?Lh324ANX$C~Q+IgSj6PcOW=hDE!)cuVrnauBm$i>SFAL#RnS z9wu$W6J}GUa+E)G*?y8Z%Hcq?crtqU4jfpVfZe^jSofa)+QwU>1f())>r+C60G@;B z`yskU64qmN%|$Eq_|^A&ZtvKB4Q`o@6TSp;8JyUfw>U)#p~@fOt4wR7_Z_28DN;lZ zM(f-i#;l}m-QD6=y7H2x0y64>%{-VS6IgkL`R|s|fvjvjO{)8cxt=cgRrao&=HE{8 zrQyjgB?s6nPPHesZf~(E=nIW?OFk7PXocpdRVl_(rO!do5OdrfeH1${-yyas<(HZx z%e-^4Fa7AFnF&)~6?Dj99MEg=;gQ>PkQQc_2f%=OYB>BLy^8|R%?0jYa(?_wD9DG` zZ_Rh<=$DB zH6yMd8DIrbMJq_+rh#J(;=Wm4j)-3Vk~JvV8(X$h#uMWmk}iki(h)Y>o-MN@xGG)% zc0;E5gczi03%G+cRqVh;AS#6X4szGAC}>2pM{!Z|$Sn1bf6gDYh#t3xtW>gTP^m=}e)~*-E&e$+FOin>$~k_BXBc%`k?K=P zH#pj){zIYNz~@uwd-1R-HCNRjr&PYJQb-%~f^F?C=%bPXq;0ZVrS z{&udR2qQ?$5HIs!$4-su;@=)UNfhf#RQTv$#_d!pHdTx1fece|`HilTPV7ljC|ZHn z74~%T{uPY}BVQs>mNGv`wotN!4&Vry&j+?|H$YZ|P2^>QGgU5CXJJX6OWx>dDqu-= z8dgCfruMDHfKam#`^9gL81O*^SRMFH=2DpdIZP9{0hoT|Tygw#e<>LCv3o;7Ybfb) z!p*Fm%A1~4cb}C$6fvq)dLpIk`S!Pwi|`2=-#DxEOT4(;uUOkc?nCcMyYAG2tTR}% z&M*&b@^rFNU4*BRce0m{PO1j-@f5`7m5x93i#N>@gkjfD(!?Cx^39~AubS8aY3|#2P;v{cXwgsy++ttIpX@w?@9*)Z zj;9n4N+p*rEQLBa**M?R-|DqG+8Kh8jqQmthELU%ex`3ydHobV{`5w!Z4#92KpJ@0 zN;gu?_4fT8+}dpPLryr`Gq33M7IZtUNx)>4+|%ro_WjxI&nYmmw3{tA!vE1_o&7mv zPCB~?2T>9*;B$skp%(^S2+mx-???P0Dudo&l)qSySAyC=#Nn{*i{-UpEOm%GI4sL_ z;}{&msBEn|vZ-5snsczi(yqb?&bp+^m_+)B&_v~`go>M|c-lsA^E@nizuGy#I}rTz z6gx4!VhX(1(=-`TTYzdX{p`~-a2R*UO@g2L#2v`_SGc1JOlqK10^!JAGBH6TDtxO7 zLFA&=cPBCFTxGtj8mmdOj9{$bDEw5mx~wLsD%vE{=ve}zFAm=mJ2AQAH-rK*)$yem zvAAgl1K#3w+x0l=;jvC7gOg(^4+`~BoM{hs{sBXE4!g7R=9EyeXX@-#IGO}bi)khSL{ zMZK%O67Dh!O5oTfW^&}OHPEREPhO&56BDqm)|S$TT$bL??B#9@H|N3VRSjjveySii zT70L1G`#bNSKMLBN_p87b14%|T@U$>jF0d-hI1XeMx;-h4jRIzal7B+a(rXAT!ev} z=48$?uvHO*W~G?Rt_NFp0}IG780WF3Aa%YZbi3_K;d=NRFdF2Rq3ns``%jLN(mo_G z!Q%PTN{~KfCjv2J40)DK#V)G`3{km{>sFghntz`Ammd}$CFZPj)oj%CN#PC|CY$}^ zhXir~vhBOPsRzyrpLyi?Mj3))?!G?sXdT~9WmT1S(y3fVw|_{TL&Ux=$-m|nSF#UP zRIb)3l%0Dj`*F(m0&GLTTQ(6V1fzz#=&asL3g{o0;u$>P8)CV~f2xOuoHij)mZw5H{C{Hg{5;{FW`ttEYX7K- zFBeUt6DZ<=wFAjL5xfYkxj!>1-*i@*#hE^2SkDNTf;!p(6<4IHVX7v|sImLcLeG|< z3pn1%EY0y=`^U-XC=bB$8+W76Y*(kVS=_)Xg9oz!B!K_YD)&i6qmVZinmg`i-h(@Y zws{K}K|ocr+v@9~Nu!{R;CG4$g$*EZ6FGb*RkrL>P4m{>P^*yAp=0BrI~`%n3{!KC zoh-#MeO;dCwW(1ao8y3{t^^B|u0%4&!)~rU3av0t%;NEc@~C|}jQ>^anIoCfb5$x4 zt3N6c@8E8$4H9Jh3Ti_U@?XDdW9&TU3lG_d!!vE8Of)7iy6#Tn7ryw5f1^}verTqf zUTjt3Q+KxvR1ll2nT&S3XQTD$6A#`x#Ljz{txr6?=uy_Pe*OJZLXStvm!Uj(*M%*f zKI<&L^AT^J_1Y5+hKxi zC#yq-gB0M=B8|Kw2YnqG_Ftd0W#u%>UlXgMq|f2`dmI3C=yg2^6|Br>Geoaag`&_) zDB(Kj-OgBEDHf(@8hg=1kZYD3GW+pSPSZGdFOy_%>tlj!*x6VTn$LWcZ!kP>2qQ9P zq+|6A(r_1kGoMBX{aog-#Reh$d7vjHS7w|f@~4oHey;Qhox{o>RGv?^#f~J34Ri^$ zEk!$WM?C}x8Uxg$kdF9};rj|g3(b7KyFbe8 ztO!2FYYt9+^Vx1>PV^!rK;XIt@1}WYo7F#FhTfLH{OpjU@ItS%x^`FBv>1b&{qnuK9UoK~{D-z`y_OL|u{&@9eSIS;W$$ z$5O<+^iQg4WCu4h?RDg%PGyoH596LCwe1~0toKagjz9JT%eYJ5*~m8^3dvKR*Trr7 zt8S}HjT?h|ez(tL7N`GNXTGv_z!+QeGj*CieTw#UN47K9H-Y5MqV=a^@u)4l*X1{c zqt|?__WG*kKRd^Mb1;$h3RTP@iMpwn%PNcxjJ0)_F8JvQCca)ZXX6^ggKDSAeXR(Qf5D;E{SonOp^j!3ar`wCIu z!tz0$p6>5E^)5dj1h{q#`xH=4+NgbMySe@FY>9B59WS4Eve>y)7TU<3`p#G0BjoPg znllQxiT=|5twvGlUqBCLq8!@^?e-yQJ*n}--J}(^1hI^DgWSdtr)&I z!Hea7J?_u;bxLe0oosme_;X;0$pf`!i|=jw#orpOurk}|{en#+&KQdWCOy-`=ifXG zF9pyiufOLtGbFQSl#Xi@Wzt==FcsOupGBK4d(VoG4OL)JF{>SR!?88XHVxkNXigvGdmc}=1k9~5VZ1~}D zx<$3S(YLOgLt&R5LsENEd;1nCR?6Bo;rWSqY1c$4LwKgbst#GJT+=U#lq{{#2O_MJ%sk8*Ytl7j|`zntgScl@D)iS-*UeCBPr`^=DTbwMh zeete{4c@&kWUOQi46lvEx+NDZ19*S%V$!|{D%^HcFM=Gu`n>a{W9X(t=F^6UOYl;H z%dgf~i^X&d#HMNCO*{61EYxW=Wdsdf3&>#;H%7sKq>j`=Ru{EeEJBWx4nFef2s~Ad zkiRbS*5o4f9{juAPH@ykJSDQ^JLVs{IL6Y@W+nKg)gZC0vS{ZGG|EbXD70k zfGw#7QNWR}m`b__@`jK6Ga|LI5}c~PCPc>KoI*0LSlq?Ck6JC=`S^5sAJZaF9o9qp z%ZV&Al4C!6^FcMGacAjc!NIO)_FODR9TSae<4clAA||gSs>7>e!qDk3N6YzS!8PT5 zg=X3LIwbk^&H^?&0+|zGoh-8>ev2m!3SGoTC*eGsT5gqg+~ptkv+gbv1Y6osDLg|` zjoXpi=e8z;=oiV2Cx;R}5h;pn`e`-%*jii{{DUR_bkM?2w(D}_1`bOx7{&r~fv-MW zji&;*vjEPIh-F0jO6tEi)Rb>ZY+4S{^GNizR`;7O@fD z5Pe2A12NSzle2~#Uf8tTE+i1he4cmdrr|Jtu({!<_OODP^_4o7(b!6Yj;4srOY185 zcP=c}36E3jX%@@uWt&N~A;KT|s_*l3SQ8T5T0eMC^|{Q(VUFdztA&ue6Ti!e7;&JdGM`U`Ag5wy{Jw`GU@r1I)soAh>SPG0OmL8&K`q}8aG+=nE} zx}FLhhqT-1I>k_S>wd-VBK(4v5_Fi__<8Qdkvp0}&48v< zRf{5-j`2qHuJ~%b`pWF(2p&qR_6@7Ne*Xo`}qvi^#rR1ygE%7a9n8X0(@oCZL&p zeV2Eab{Q=BisQlq-uV_3EM_FsJnuH=Ix6EP$dQ*@27%da+S-kw_obX3l;2;-8Y^7E z`Z*nn;k8_13-ZiLBEl2<(zp!ucE-TuX(wX2-)_p^TnqI3+z*qJ_w%s+Mja$4CL2%F zb9Z4$;!>lSGC5(W!nWwysEwLTmw!P1;nMr-axK$$yxxkW^%{3bv6hEG6Nx2R1C5=| zO0f)qVr~g9HQM`Mt4qSktxw-tjaoZ&tf51yivoUg6QA_CQp&v-R<4WNwClEzM3Y(> z_=Zl>&=jT@i|#O$ETb*1Wy=oy>K*!O9{79v?M8~`zC$JZ89SASzQUTs%)@Jm5-RGX zei{5H>jN>D@BP3`@%?z%r+Q%{L5)QAm-M?OLcDV2&ZFqP2R}njdmERHhT*d2Q!NDE zH1?J3+p7B^7w>Yx$SiVQ;*nc?I*>EjUrQaI18RLU_tlr)1*Vc&q-)6jd3q%o%8q{A;Qe}$k?$^hQHcsWUJmg7rOit6DByJj zqGCw-#S4ctY?&TbvTbVq(ZKW3yn zuvz!NneuB-jz>FQnPp<;;E~wQSXGX?`i;5^{~Og!75lp4i8oUk1v9B)1c+B1Ijt^J z$(4wJ$l&6qC2yomdtn&0pvt1N(zRdp;EkDF-2d*OfVGu8_&37_jY||y_LIHKddMO+ zk+*fdY!-S9UnYP zuo@LbZp??v96WM%Q_fiZM~IVf#=Sf7aM^`+578|<5EL*PR$#4<8h)`NJYdPLUZ}+z z>$Y9sW}{gqY~}e$$77lhZ`|#s*Zq#bhrU1dG&HFQ0(B^}@+TaY*|w? z`Q$T!rmZ71SvZxv5v8ypQ*c{B%>(6BD zVT#m8yd*?OjPPot@(qL9Nij(4i^UNA2jg4NQZ9}f6{I|(bJ(DSduOuEoK-6b|H1vT z53!m*3V(!aQ<+~Q#v`ld94`O*xWrBlA~AA+R3YF#+~s<_inZ!V0ow0;n>)l4=$k^+cW^wXp8o52+m_4Ik6?g>dBVWOWEQMYiJX?-ME_Z1u zg1q_ut-v%qFpJcctf!QNzblOzFKp85CHM{9OXFpi5N`y?aZkcjqLQYp+=GZ;23S~_ z@|G+I=z87wg6MNt`KRSbxcAd^1vj>1dsmO77XerD)mjsCs+~rmZFZjRYLi7pP)5!%tLb?vf_1e3^BMZj z0J53?OAN3oqsEPgKD=6+K!y}nFy5Z_R~#3@LNJ}wJtq%miv6FKpb%z{1*%(?=-{6+}H)b%9UN^C${m*C#EcJYa^zTGCPp7GsSaP*R{+jw1GJOs$|h4 zi4-7;NaW6*enxw>Ft-8Gub&X;Z@A4F@r_48QMN*s)JSW+t|vP2NW2tMCc&QzuQhRF zA{*e@2`0HXSdCD~jCgewqOknd&Zl_oSF%28y-#kjSK#*USy*+^M&3lJv78w_M9}Jq zH+r$%`h(FIJJLQqDH<znb;E^yPoyh9e%ObBK@Wn$ubC#Q+Btf2 zv4Y4LpG7QWh@aaxcrnR?AMZ)$P5J&#_k2s8`~yW@A@W2`Ghc=hD@0-#!cK=&LYsd5 za|sqZ)1S!6X}#jPp3q+=xGh5!wR0~W$&_EmTx=(z+*;EnK`>Z7zxyQrg`J8a5b(P6 z#zG$~BeEW5Y3`vtcD)DHr`LX2u`0Yk%m_yK@fOFJ zVCp@nh7Dc^U*gR8tT1?DV<@0TVnV;xeSJ9G((pM?_@yBA;BRij9-0nSRYoV8fa?Dr zn!Y-$s^|Oq(j_5CT^a?H21#j<6eJ`sU2*}D?v_$%5v1YLofni&K|tx2l;_A`(rvLwj073x5OchQN_i6V1QylWblE1tu#^_Id2#}ET&X)eYED1a1?vzj zo9)OH-j?#DLW6DJCsnVA$F{q7Tn@vi=ydn zwxc4w6&f#Lh%RTRO*k7K`dVh*lKm{SnDy2V@-(~x?vnTbGX@>zCAVfyT59N<<)u;T zQ{w&iHL7aQ5*YMl>dh~)t_F=Ia_N>fAtl!%4F%kzpm(d`c;*nHj@9a%htSAQ|IJ%W zdN?zo(b+SfoZ&JfI^9s{__lZIgP6AKQe@FUm9k%+mk$rs=7sAo?@iv1KJbEmi-lWd zwo8ZgsPQ~)ZSmuonQ=b&bNi~k5sR3!xwCbnI}s;QHsiowz=k+Nc?}&1B7NLbG%+!W z7~8XkdRZKj-^oFU9vfQFBOlRS%xDzt@-hE{+H;Px=ce_#x!G(2Am8-F=NCd8fzUcLIki|gP`o@_vudR zEwLGZr6axTsvdvZ8{Y81p+;*q1BJSulM{1cxLxFQ!5-qH)_xTAIeV#;HEfmEP08Q%9H-8u zyT7`jUZZS#oL(LDWa-;2L)u@8YE0XA(;LL)OZD*&x*!EFRWexo-jU+{Z9b!Bx%MN1 z;!_8zqbqjAG6(foaPdL=w*e8{j49th%aUzDd|h8nJ!AWGs4#ztJFl%~0Q#Y(teX0V z&!DJ_{Fb2gn&D8TNjQODILc6wet%+Pv-RX{Be!*Px$(+;=YC8fVhF2ra?|_JztmKn zwjTvue$VyI|9vXFVn02iS=;^dX>&G}w7Ffj&i3JfiK+QPu`?c&R%pyK_y|{E;L=Is z8%9MZD*XlXs{Bog-{krAv6q8^P`4_{57mQ+1sp_iCVp=FjrcrHSBX?Z%pN5QnfI}e zvBRtqra446ep2kj4Mn0Zh%a&;wQlXRpmt7IxvUGctK4c~$|2ZB)XZ$QiqcJ_#K+osf z^U|)qGxXdiL}Z3r2j#~}SzmY@7d`*wTP2kC=l;R&)(V!p-xvs9oCxuskF=4XpwP2C zB0VilPW#ShhW<5D!>Xm4`WFvAq4k>`Jsr9aL3NedKD6#hlhtg4Qm_$v$rfJhm*ZKM z;eJ+SIe${csT-1sbhq3S+MWlPC~D!iXZ6&L+B{RnM8kfK9{xZ}E9uu?Z$UgQ=$pDb zzfsog%HNMkRUYed`wg#6%EI?5#cjC`9Sj*Hy>caZI6kc4ndBG#X^^ERyGh6}Zk8Zv zBD{9x8obDz?01&VjzWha%5`Hehg`}u)`nE^oHux)E$2$VlpQfO;}UIG&0SJQb`|Jr zb(rI~ua!~l^l%S5!r{_>a(zKV#iEa3r<5w?V?RItTf?zGjoVY~5@k@nf0Op4%j$`^ zx<`79)s>HOyo72evDrXI(m>r_Xi)1@X-i@XmFD_&TfWq(38f63`s$(XKa22U0J`LZ zZVi@0pAZIoKu%sz3P>uj7;Yf?{S-6m^BAQ$?8#(qgH?Q) z34DPqhv-+W2oAD~j}ZA-(yMGuj(yGDGXAa4GtHq$rcsab?k%w=mt;xCtf^}}A@T#q zKWr-lXR>7>=4tuTlK(PyHSDLFSvL5YCGKRJ^LL$cx|@!Y>Oc|0we)xXTD$MqwAf~R z(IOA#qVrIsj10-b+r!?Bq2nv$v^qwGt^srH*0g>iOK>c9M^@YWAMC8?3RfX(Z;eA% z2pGD*n~=bS0I<-^GtGyM^nT^lO#v<_Z;rx8?5k4DuD6x2$8cDwYyqhWz&pC`W{mb)>WuWtLi68x|ZIlU?)%tIScv%LnY3^Of^Jr!KG5B2=murtE6%?fK3Opxdt zzew88+5P48FH*Z%(Tk2)19-v;zw@YKlT^B}T;-khujAU=i3xT)5Q>ki{f3d|;*tWV zE&sPT<}BKbznbh)_Wv95X=%KK;4%ETF0AEL8j0dvL*_ z@&mkt&_RA0b0_8q@K|0(6SxEQDH_atswaUG67HvIqXy(K1iIl3Dnw0Ru~r*{=$oYU zE2-d@^Hq-xCZgep+dr&^D4SRmZ|f%NGM?x+kV?DWw2N9PWVC60Sjrvj$QpolePn)j zn^3x7us*SGiysjTO^yEIw@W8RebwX_-T2;QzEV-*cpoV=gsb-xcRxS9>-ib<^h7*? z^>>osiSq+rQLn5WM?G`#7^Ck59MWcQ_TwS$nxuPjMxI|irMFdz3B&xFS*jO#uhztV z(CuIOcQGTm0UP=8HGcG`wrc`tzmMl2VDrS?NvSu2ySsr`9wuYUaXaDtG4|=B1bdT( zLIb1tQN@+2{?1kZW0=C8d~Vm~zgS^gNu~P%%yewAXlN{1>#% zq6*zVUg1O3UdbcV(1{?%q|X{P;Z+EtvTy)OM}9^upk;^P!Vxhchkk}HwATL}_Z_)Q zj=gElJB^?-v7NFrb~P33sgQm$vCjpCTKe@xnG>G1nxGLAyTOqxcun({53+Z%8~6pA z9SnjQmj~46zU9r8cL$QMt> zRCCdvt1=Q9IxjJ~BNISi1OU%t!TZaj=*=sOHvp&D>-d)isT&L0Tr^DpvbmI0K1i1+ zrp*h~lK&nwC$c}Fp`s1#@w^^ax%GR#H}v}P?&)D(|4~{vmIpTlUvlVmTrM7&av~Z; zgTam^EaDl8$8tSmdmN{qoN-NAg<6Kr-UCmmL-sG5uNg3Co5$+1mFY=;r)$`nQ=R|Y z6~tFysjFzPV|zzxR`WB&=)3KKr{-@hg&dl-I=M=n!p;zOV4Y21gHgCzx4R&pNIk7>B195P2G6JKE3 z)uUFc&P{}bXno3!zjel3AC~|Y_$(t@GATG(A&n}=VHHq5F?o$^EwA{MRKuLI&=ys! zgnzMRdLaZKQ!(@rNxRfcIdY-lBqhdTu}ys%hO6Fk^;PY885xe!vtO2Ik%?RCo&J40 zcQx^L?zx`Db7AAeN0^XITC(X-a)<^{w>6A>Mf2dD2f5~x!F`5+4$nXFQxeY1!76o! z*LQ5DL;hc6xi4t9gm1SObeP9J0~c(FBcC?G&z?MD`-6S=wkSSU+A&XxJb24!k7`zT z0vFdju>nG0^^Fayv~I=unrU@_=W5uAD6vOLKa7b7!Am>2INT%G2t! zsinJxK82O`^hJW=bISF2gA(q#1X}PMT3i%}!n~plhYreU`fqQavJOUz*xS+;uhcWo zz8a5Ms#z6lvIt!qb^6cd`Ke`VD4FK({xrH99bEVp zM8ixUntF~`Xz`xm*U_Rwp-p8DV(W_KA%8SK2d(?Q=p1}%x6z^2XN6SOAM0~B#1joU zFR6|6lzyl0u?fFQP<=KlWB1g+Rvg2nv<`R=165Ia?$1O?%F2$h-$89mtRI%${JO+% z|A)C=k@W459szBxYx9<5^6qNyo!hs#FJYlKZcSW0HCf+k&3M(o6N*8hJ)=8gx<043 zCoyKm_Tz=1r-+4_F3azH0j#>}@n3xJl;VMUZgVla)*B;WQgL!YGvmfqKIriuNpeP+ zw1FRo)OnBDy&$EdqZ6KASyu(#;PKSIU%5-?*OHyr7HW~Hi&8D*CHrdsqF=Ex#JK=5 z#OkAR0;7P#j~N5Z3mRY&*(=8){zVgqgJ2JkLr?KGUR>soj@ue*9rIt#@u6)RIgo5( z{`-29ziSU^AJ!v?z$9goe(iB1c7uoud2?e>5AAo}c^MSgaUc4Y5y3DT8X29SPgppa z)Bm)7$Bhpg8DTwLyOh{c<3X+~7EkxO+O3%KtWTLd+(u^IcXeE(tky%9?6hEP%+R2+ zesfq-817x%T7(-~zp&m=|I}0+?b1+UB4LV-qv@SZ>H5k{^MeI7CwC}zhsd(%sO*?5 z%g4Jf)v)2K|`1ZXqnJUG)lrkfwN zPpNHc2FkP-jw{38xCt2m?8j0n1e=oVlqo!v8CWw@v!}VtsUe^~>P7s4)?RpwZ*maA z#P~YH7O;W_Ks6NtIblrk?NV+=`Pd07=Hu~425Em?-|G6K##duz^wbA4vpIKCV-l;h z%cE#5VZ%rBbLqNFq3%hb^K3+VoQ5OM(I`WS0xM?vVd}!nT78W}f^*4G`mf7TIuz$O+o94xcNF31${#&~UL zQt{ZYUzv>J7n%uLIJey%H+^{@IRX&~+x{|tSIB4d7^Wxw)tftIfPJtI!5%QUo>(;h ze((z|f`>AKgyIB9%f#RfK%_IjTWoj}SzDY@57)UPpli8v#wNjyJk8NNIz1e|T&u4A zW)JuFOxjM8<6D$@Mb?s@XSdUcEuev^QY z(8A9*?_~W+S`oi$>U;i`iD9m}kuHm>>w})8|GOhVI1poQvvg~;{u=-0m;Q+V$d*ZI zG2Gg6>=IHjQ}yuUrsEq?M?{LE8V_p( zBL=!0T4p?tF{>{yIgo~Lo9leqE~)=t9hff<yr8U!JN3vK?HQg_1TNi`*S{Am%&pNYoYLH&N+1k~<<+d94 z1~*vltx$yP^*+Ok03*6^RHx9JW@}X3foVys6iOqFeXFL~emBJ(>{x(4hi#_wf$SJ0 z>IdwDO4%~2IJV}Sh@)-JU|BXIh1{Xa+Pv}`kMByF6tZw-=3qSt_U{sY*l*ZloXFd( zQVgH%{l-DJlW2hRKw&;A$ZzDfd)eT2vFK;CGGFh%SfnU$_=jrWhXfb|`EkO_PJ9v! z`CHztHZ1{4jl_AWlbI-jcJlE>6!3`+oCUNHdTw`E?x#9Nw99u_9Kq%|1)pomuMBv{ z*k$t8e2&uUUcrWRf_~28T05d-kpAi~z|~9wLX8QFgKL&0?BWPN05zUBO@yjhesv|d z+AYkn)oG2F<3g$k6X!X%F(9+Kn!5x`YWJN>{SKjzh$KIuza$I@7vwzQSj<&5+# zZ?HV}TkX%=%}w9*AB+p_d}W=|Ruh9rJ}nrwke7)0nc+NwA4y6bW~)i=s?d)20;_hK z=d9-SMtznRi_Ez6L12}%W|H=H9hAF))^r>0}&y)U+(5r+2OS>!MET}cICLWt^X<{IuUdGsPi6{APm zb=Om`7Nj`gQ<(G+_D21zQo}t|T@SuwmXd(zi-%;oG~SQ-l3}f;)UBcWB4rb=Pfjkn zs!I<6CJJ6 z%vW5X<`Oq&krU#FEs~ZN`lyCO%}5OS9`W%?l)7k<*e&mRduSnb0ZITVn6Acrx_oO_ zsu3B5MjIYq@7;vEd)o)g?oeS8)$jWmA54;E z98CpD!;e*Oz_IFSHEM+|1u}P-?@W_WV^On^hFT~GP=h!UifD;d_H~8IqJ@s z7w7e(TjnRux%fr1>qf?e|HPE}%cNTmlPG-uz_Oazv|op_Q$LihJmv5v(m(%P9-%$% zGvoLr;2vF{?(_aQ&qNWQczwL+Yy(?7C6$s;>wW*cu5wf6%oXreyp%^O%$J}Vrb1=} z6rRfc$AF~xP*K2-4w_19#ax|Fu=n|af@^n`0y9a!7MD=(xukzIx*XLDqnj?4>moeE zkC>1N-H#;!;w)E*`Ok0>^>NYW9*oNWqMOg?_$Cu!S}*7Q2cCyF_^nru#OE9PKa#AV zcLULs{4lYi@5a|&6BbQ304E^bJsLov8OJ#aR)k^>mM(mbRM^FENJE%Q$$~u@aI^NHtGXw_9}Kbt=uYMGrETXopj>gk0IJ}x2u(`0;F<% z7cX^3#isUw?A5~V##=H>Rxm3Veuh79s32asx)*I^Wnpj`78d03OuLOb(--7C3|r-4?ggBDAa$roOkOy!4|=V|FTuW1;Z2AvI=m->nfR;s7-EMv=_1 zG&1qxkLYSr?eZbdR)tOCwF7ykV(edi-nlU!AFI=vaPHKk6#EU6V58>D*|l+x(2L}u z{Y*$2iE}P5vgo8^#u8c>=6j;EHv|# z$s@_1aL4s=)__`P@gaPiXZtkc7LJ1Sut3Lk)3QX??5jAnPSQn~Ih(3hdF?Do`g0ai z96-NTm!m!fg4e{o1*!4h7#J(PBdalnIaKNydvA^#B~%>@v_F}iU(-;C9IjAY91)P^j0y&GZ-i7H@WyR61I1#%mpR;gs zgB|F6$6LvI52g}sQ(5!9~%nNA$~J z%1W=xboHKIi^NNyLr@=fnuTr$0NoTq2Q~KekT>!CtL0RAil?fZU2jnC%({-KiCkDO z8zjK@*UjD8pT=*?jER^@$uL7i+2jgdCa!G}UtRvdiuj(jIIJD**#Tw0cNVGdppWJU zk)cdhqqC$)w3YSAm7XJ0HHL8l0qV}cBQePr5|q6&PsXg4R-HxfeqWFvoBia<4n$2K z0u@OV7G`kQm6=?&nDE$aG@|BZ(?o4`OP?vIo4nE6kHaD_!hh1{s)Y)X^s&hP<+<7L z!PQ`7wB>sgr;8{qq*in!zKiVxn`AJUv|eeWR=%(6OP(P||6I?4OG*#wgyRRx#Tug* zmwX8IpGq~5lxlqXwSHYTP|}vD^ks4Q52|<(771hRN1zMoE@R4R?|oekH1B0nHuH;3 zR(AYX!a#8#0E+cMG>A+$tXLv~1i(DUi?U|xBT$z2MwLJZJt1m3aLJSjiI(lP*~3W& z0mD|tjHOC%)F1VQasMVCiM^u*+qGSc^R)|GYs&F+ zC&Q_mk7XFAz)%9XzZzW+rz~YSWgqX8m!7mO$$+C2aPN!a#-_%wB`IM|9}LK3Rhf*| z(8RB|lr*4oP@DjyWHD)2B%N@UQ6Xd)`8jUDMlu59aVBX>0q&zvELOPQif#7841}x? z%Ll+{5H!Q&>>ECB`xZBWZ%wtFs_8xhc^8nuXwL+(1N;y}ePm6~3@VSvYHB{<oQ zMGTRY`Vc}srmkIi(CCvBQ-t7)lMn80ir+^6o|K6R#xT;1{KL6XF`Ps=uE^xrZ1r$| zlGiIm6a5JvlstpVXZ%G*6!yB+N`3EBLp=5iRbqb(jm3r#fqhn-(9N71&&BSyOaoOU zg>19J-X-1cCEea;i<9XtUu6BdKVwM&)E_p0jJ!xuPUV}9`+a~(H(#$wh#Fuq<6Wk$ z%ffrt`Hvz(9UnU^hF%efxkWfX>=Q!^ZOL&VScnP?wK?6jH6NFJ_Aj2&yEg~>b!}l6 z%jZ8h-z97a_f}ge0>_vsSrlqBSXJ28H$u_=dpgVBDn&f)sW?}#VzqR9lp&tDt#2t7 z{zr}bu28~n6lJn`X|ITqg*nks{K1aQ zRI1X>q7ry0ftoF;m=3a|cIVemkkFFI51P#5$)1ugU1rVK&l?%IZBT%WwPq#LW}Tr7 zER)SRRF6;x-WTZOP~*(_T0fNbIW`&uC=1DaJAhbY>qno}7>kb4^zfTBTF4jriNijq z-&gSu_D*;2cqM{}ecn%+wsA^BpVT>DM>su8*>IVILX6%_@H{oq$JUZ7NuPs#Xzm6kiEacAG}vp&8pL3J)xQ)WpaeAFy`V1ZvCfWlm)|xBj!lWb)pJ{{ zYiuqgMF4(^C>CTM-KYmBE>h!G-gG*gE>_?=AD%-!z4%BW?nT=+N!;}p*UD@&~@7xl0|Ocgmp=?0!NLaO|keC(nw zQkkqoMH+pTc#y^*)@+Y>x4QlK?>|Ox<3A?$BS0&Jjz&eCyHBUTP@c%6eaWvc@F}|! z&ZS>xu=t?EnlE5g`Vb8qu_9vBQ{LuHdef`M!f#gcxurZW%#9BM3AY$-8A=~L;b@v5 z5Dm)J)riMPHn^dRpO37OF+RKMxSgk&o4zjwDV2^31Q)D zIFs}lE97B&PbR(+V}pBzq(8(!vBkKqa~jp&|5Ij*B!+^OVMr!6c`8qC$;5pr|J%P? zBoT=M>D+EJH#yElQ^^*y=3Q;4_2Jm}Ge^_YFu;;4nSgv9(wfK=u&Gjj3_n`;cfr@d@6n0G%wGNbL!OBpDUw2@7Ft z*%59I)Wo9VmSEO#Y%D{Dt}KdA^OBf-jA&2#@0FMD9{AVF0dUKe$o4Sj}lnsMEsu|1LP?*2klJh=%Qr?nu=Cczje0J=O0x76AZ4Xpu{(u7E zIcmy;9m|5ADM{HwAsUcPz0vbRWI^@C3a?4@(DQ zEh+~NU0mw8ho}9|^xk7>$-H8#CI5!wjpNol^YulalwbNZHrX6l#NT{;Me#hzUJ+N+ z1G0|8uZvB@AN`%0DE%S{DgfN_^JZ!;hsfB3^OvJSn3tRwb`$ zYScW8__6w2Cr@?g@@LNlAZl}<)1CK;a$dj zs6LIN@V72t{uOI71furkUE@JxM~jj&I+!T|e3!YZNyJM(GEM4I8e_c1;H%-<;R^*F z-31ItYiY=m_f72^@ja7suk;^;`%M92WfLeqZgU^W0BpCI@A+-!D(^PHRur1xrZ%QV z0>tss>2wy1^X+{!*6?F+OqX(Jbz%@cKYZ{`WF+HT+G8kV6Bn|||6?w$RJK^si#e1d zs2=(kBE@;B6(ZVE2xQ;NVCej}U>7+g0N&&Lqc-4dtY}Y+<%J)`mS|VvX7q+FlBJtA z9F2%VC1nf4=$>g6%1IgbZC(j(Xm7*h)6EzDY8B@El;zOpH|BH)C@=pi(Q;SD0fu(u zWdshmT7ya(d#D^a%zAWvH6M?*Hfl}mZ|Qah9?Ebdv9+4uK|p&}-Q!Z~FIv=4ra=@) zfJFF9v}|ScyAgW?4H0BP#oP{!uIh2=qhs~I>#Bt>lKC>0*vLFcm;;>&@f!2dXHg&y zn{(zUtODWal3m8Mg0hFAvEmanc>}`b1TsLJ*3;4$ujwA~5R9m}(WU?gKT?1Ll_Do1$kuWU79GY6ad#@QGkk_T6ZwaFc zFe%XOsRk3tOED%U5}5;s?k9;m$I#udrBxH3gQMOC33%<|csaWeIVK?w%S%)sWGO(X z&$k6nz^2w1hU3H2Rs+Lior$6GYIdY+m!Qe8>HWyO$#5{`JeyGZ z0Dl4vuMq-yqP)5;Iv_zzaWXJ*9&R<@#4^M_Q{U#8;2*Yz#AQnuOhD1Nd~sJ6u!oP8Kem-v)xEKBE<#3zB&n z`bPhqq|6wf`@dsf01ek%RHx_y)m8cEr^WaT0EwNBN8*Zyn)6j7O6c8h3QGhxjpxC|x(l_a+2*Xw=NstIQSxIsf*rlQJGa z@j{ACRI6T;d=k_gsiWG~ek)_bL*eRHJc#5W#l+aR0erYiX$j$kR)p#^ry2nN39Kh8 ziY4+)Qy-V|K9(>BIaqK;US1n_V^k3$cdQodJ)C|r*2rnxNby8Wr|+zhUDVLo+S%UQ zIwJZV*sbAwZF}KiI$5!zu^u&jkJab|MzIEXh;>X*8<+#K4b+`CW>msHHNvQnR#xAx z?{yb~n14}**GeJJj!@rUz!nUzE`!wZ-f^QrS~)2D<~_UydW_EhgQ`&{i!bP20t+Le zaFqhR(HQ}u2Cjg@fB1RXK87&UMX*)|1CP-r~SqWdeZy8m3&6~C^8Fhdhwy&G8cyXPGu(2SS#DtK_0L!%#=n(Ez8G-zU$X5wIm#aBts7$ChVkG# zozeuCzD8zcUYFgL@c%=2VF7n9H#YdW$NhUXwNC|d(jf;i^k!t>)6eLjUuW6g3i@TB zBr&o17DVdIedKaeyLP;K99D4tm-~wbwri_?R5GE9`lCR;R3)9|fsi?Bu9ZIc&fQJV z9z#JdrnY-Rnj+-BSr{>bCaUp@+Kw(lh`>c0h0E_H_dkl}AIE>F;W#J9<8s$5 z;Ijd<`!hOZcVC?AdY~R@@V9#w$SK}i1H40)P*mr}fi9;)OQgGx0?CBd&;_A^B;rq_ z;OHQnbT-$_T|1QyGzn1Ex7$uq0jT^kFQ>1vYm96^@`Xc1eeG-yBR%v}_^o4MFDQ8aXGbg~Du;pX__e|6sp zRTDwu{l5=pkV7*$Fco~ol+U$Ufj+~g%u*(sh0JX!NtC{#6%8VZkprz8Fuc`@k;Gpl zf=Jp}7)kC;1J zTOPg_lchv}b{nEK1&&drU^FMHc+Ltl;KtqJ;d;F3tQ1 z8VKYWBcDwU1$`3-vf|tRI%|7NH)%%T5HOTye`hfd9DK_zY$9gANal;^cc6DM zJpmMYqy8_cdH0a)ln(eGXIk3a%XNwKl1`U412NX+YFp6b)b*!O!&@ZeE0?HhriPPW zHHxbQx*W$)Zx&*gmQa9C2?UD*Xgv#l_^sZdWf?(QYQO$p`_(4LPxC3?pM58V#vgU| z^wZc0UjE3A_pQXWE-BsKQ>yjz?^Ek2V2;bN#F}EBFu6xHn%!%h{s2d3{;wN_(_QmQ z6PxVnOU@JdqG&%hO}f%6QFJYe*b+cODNBOh-T9m1wU_}gMs6D^P~gEF4ZY>(Z4($9 zEI10YD++~4jj74-_E;cNZ}N#3Te&n0&7SY6oSpC82Kn?K&W;Vt%4J^4$z;oQztKj- z-{9T&91>a6K8Z9H^h_e!PqENI4yCgz3o<6|rVbtbMwfDhK#Jm~6I=936hPYY*@8ep zCYz0QRWB|~z6HK@$rCRh_$2KJgw}vJ;%M3Lf>BLU2BN6dfkXMC{i}qX&mu|lATD4s zKIkmC2(elKwVD61EbL(qW6=B?ZUw_x970~#gvyn(bMnpDczbanh8WW(=f`@R^?pPJ zYlj>cwr^$$FSWuy1;!|fTt@5zwjlnW9(WsePh()eJ^Q4P{cMu9!ymS2u=9>R z51C<{TnQUUIF=mi=XexXjDecHqVo7Jli!pc!6kq^F2CZ75ZY?|NHYK`^ch`P(m6s) z$YB>l@FC+59Waz(ZMe$}(Bw!zcw%c?nbBV2>3t_1GipWDbZEF+HHa`ZC&@!&h-+PM z+5qWoK-0bt1SGjGaqp@D-(Jm5rD2C8=8BiUr^x%6B5en&((l0#WU_tcF=Uh|aEKWA zGw#+R_bZC~9G0Q&<$rEgjS%@+X3D{3LfW>N<)-;pR%@>`TvNKj|Ep*oy96x`qRTVF%(5Hs4@>&;Lx9>oipU=j{2xOFi-Z{@K{odOvlbv6gmVbH6@j0FkeCvfa>2Ohd zLK!M7*BgB9|Fi(LWfF1DLH+;^LjrJn@+g_@P#_;Ow1GfJFqLOaj*aPP3Y^G)xjGdF zu;=E1E`&OC9C`mC%1JJorA(;VtM6)wdjGDlgQH;*Zm(^GY6?+Q^%wta^*hCcl>loB zEkdAxOkwGZX)4s5Z-Vem?U;NW?OJ=T$G8sq>%P~$%Q-G~0jo-X2RR$8Z^L699*8&~}F?I{h}ACTn5YeSnju3(?CZQv^FIbKP_Cj*wS&X&cdy{m4(X{c2d zNxd0x)JLdaq(9(j)Ly53Fpxx2_!@TbK{{!$tVgWtNoM`@Qp+Q!_TvA7;_>5L?X)XXYvhQzuWYtXi0QRt_HuGP zKnX%I)L|T&)(^xU8wB92@`nm|GP34erAMMnf2ea`ygW&VpF3+JJ`Xd;?#Bi@mTLj; zrS89Zs7{D4*r~#t&Mi~Fy|FNafw>f#t`L{y|Myy5=A$>mC|1w+(l)8rX!aa)EnXiW zEWQJ93Vs5f5>bd5Bq?SOkcNzA++#Oup(wEHBEAV`iQ}Q zyjMSrzNt?->uv6Ojim{?e(jHaOyARiJsS)C3k{a*)BN=x`%4y`TVR}ZGBdp}pE&my z@Q(MdQe&8U1pd{+FE7?|3aewyuThn?&TnF~?Ea4sNDm{om(YmY1n~k?$59eW{K}It zya?ejfVA)~bEThWz`-t43YDYdyS8HEdjVuTUEL%v@Q)Ha%S*x{I5hkCaoUp6n5)bE3+EQd)7Y3qZRP=&skea%f)e-%SiTO6cJZICni$RcXY-91R$sb* zV(2k!3q-0h#XGz(8ppK3oJnAK!hSnQ5A%K7rjJ?LO#)RdnIZF4zUs{w-*4ongWg>u$@5#yhcsn zNwtFo&q_OWYn}F3*dx{uLvI14lXVZVA3XsFz4J-2=6#=$Q2asi`)UjD72%17 ze;ISiAP~#ArwNfiy92=HrC1vdY42BS92`%8 zqyyVfPHHU(H~AwI5U5@hnx!=Ig=GHzg#Wt*7vpHG-BIC@9=|1`eSoxicr};&gDdQlSE|pfX7FW?BL`3ffqHt>c}4P@snJm zI^2IcCjVzz&GhEweNaHeHHXK?( zvo%m=Kl(z-=WJCND#xFyFQDsPxYKJ?gTY)b%|^645eAUN)zIx$iLjp7p(iVA}#6{JC*8 z)g2SE83u;HqZ5pee6#(_#ag!CX}Go3E+%hWbKr_^CeG#9#@k>eS!Dt zLH_%K`w-CeZeyxCp`l~4k<&vYpbFnwI65tTkctD548TPhCZA@?rJ&EvTDep3zAWiN z42j^STm*jK2cS>V*J1vw!-`xxokgu?u;Bq#Q0uy>GAnv>Os^l9+|8V3Yo?JkJ=;gl zq?)6_+5zk$!aK#JO&bAcqvIw(333ApMJo6qf3d6P;c&iOsWf!Cy9WzG1dhg8G|2$) zSrw_EK^8P^Nad2SF(LG*FZ^pkq6A>uWP5WzipG@7HRxmHqqTYby}o=jODC;lx>kO#n%N~P^U#rp9p`!*n%Hk^jRtL}QMwguDV zFSM%%H0$X&khsN!>?QXrz`gREuG2t}2phzzr+Vt8!;SY%=`Lzrzxx|RIfO#FX6a578>WhXKFpX7G~$A3rOZ_6^zJ$RE|z)LU1RqnqJt<+HlWEyWx^ z9zIKR;kQmPbXoDtSEh^)K?hjx-WkT!9+DYx0+-EvoMyXu!qXX894rf_k9(?fpAVGL zmGB_}BHR0*W;QfzveY-@d@qft`^a{aA^*vzszc73H_~JGKrKQQ*u-%Ad09HGk8ACh z-70=BKhzJhAwwPUI66x4Lx?{?L;DsU#>>&X``}NyyMR*Weg_pI(p!MM%L!~ zfO7Il&I9y-*7e2SP|No@cJ(Vu&qj(w67~lOXBQpT143HiITlu*Ue_j2=9B?F+fl8W zJvXI?meH&e0GS-2i~|~Ykv86!13(&$&hA&Erl*-W7Y4@%{I)(bX`2k$^;=mmqAuV) zf2H<;CK$chEt@l%B`L3R__J>nNZNBud&jzje$=%lx>LB;LQ~Kb0S1K} zJN*8I%UNH5kJXgD@P{=*+2v87y6^l1=*`I+zW~FJd=<)+@hbRVks0!hd`ESBa*do} zA7aRHtb2PHyiMtQi)V=!!&%zMpGGU&{rALq-#(9O-g}&fG-%q*o*>e;3MTCh?!%2d zk=xe|gxVtEJ8f%uMr0oMtGM87AkV_s;uGrX30htd4MZ(rV0^)a@vv|0 z2%QEm0768;&uts2oxoi5gy`&kw_TyjGd%^z)t*!wHCa$yrfwMySSE$WV@Jq=!gFc@ zHr86R-D=ZUR)hMIFMpBL(8!$^N6dEW^T6iF!%6krW4(W~SobmGF(nwMxE(_voeV!e zJ!hjb0Ur0#h2U)g$CiX{YI)I@K2PrdKcLp9N5Qn#h$qHA>FyA)D9*mG%AJ8|DbgS= z_h)0Md;^rgZu5>aUV)DPsh9P}nUd>JLD3z+(A}U1;nWFrbci39a>(T8TiH*6->RiX zO874kam}?wf-}GP{|wt=m_OeWQ!3`ekD4xVW8jz&hpQor2CFyLq=0A-7`x*V)jf6F z+6F3@F3DRR!o)mdnm(F`IA(gp?nx_N!SDLnk>IUAwl zhTFY~1lG@FKVc`)tl2mqqjNG!kH5w%t=T2`*oT|#2|rfq_%1M$M*;nFOAxg>#_1Pd zlnK3@aS!k<=h*fCRvu8BUe5k|UXP&c0ov63F^EkN0+5Xz;ijX=EL0PN&Y&MtgP-sV zT{0|apKz|V`fvgr3$XN$iBW64j~rRUJ*Dq_LfwE}iCz)XNj}rCJRZP~z7nt9%)$Uo ztZ0>(H#Kb3)2U~D?GLBOd`}u1{OD9M!|(dBl!4e*s)*W_%87~)ZwVY`q%XH!toeXU zI-_+lxtXYuqpNkp@S_vONCBNjHOcR*Z5D!m(H%g>PVZDTRSWnQ7r*9hpDeox5ZLzM zb8Br&%-POG(!kn1|y?ocM}rVj61Pv*r`8L&*Hezvl2G> zCY=c~%E}OwHZRaDz^)_|Dx_~ z7!YA+*7D#|feNghbH=6ohJZOPoj~=B@a_9L2&D7Rn#b-ZXBKSn43>Um0R==AJ7U_A zRI%pk0QB~A+Qz)337_O}R*!Vy5#lvLH+}uA2B<=!Hf69kQ6tp-mD%<+3?JQplJ9$j zhL5jV{hoQgyw|u16$-w^D&<@G18}Q;$*6-I ze6$qwwJh7aLZp*S!NKz%xm2CL7*Y)v05?lzYP^t#7r%L}7eBDFfm%KwO{PhCZ*_5d zj?_oNiUHnWGERUuw$nFtb zi5Se_k#txE|6#vn`0Inp)oy{Ng*6eJ#+AXj#bjXlN9K10SC9Z z+!4?cLC(cYW4Netl#sw|$1ZMtL{n@Jj-dL=ylBIG%wP0P>H!8;!SPY-B3o3iq}`n4 zAMYC!GrjjtY-=rt)*SrymAHO`t$&j@J~b+?2_fPsrOGHYLM4aty$7b&zqg|t20u`G zlCZU}Inry_zyAxP{O05O6D3f&8tJ;5&MQo)M#F1g5$h=$h z9?F3!Jv<8DU!m)LMFOI%au5tQEKbbSdS^5Eqvt%5+wl1S%OB$tFO-x>+&t2iaFbw7 zy>}rzS~B=>y@72PRh}g?))i2q{zS-W`dsgDO_6Ku3&FUPQ3do6Y!;ZVm&JqiPlOwC zdHi~05hFDX=amlXOAw)b7lG|U+M62+TlvWpb-v(^miM%KqIe0G8b{i`yXZpkS}L5&84Kz zIf)}CuG5zngKiAwjxP?HFCfIP%@bZb{Xr2xCIf;wyqqiS%kjp{r-(=T0DoJ-QL2hV z&s34VH4KFDHmKMX3UkK3@Wu-7!dDP+y}vt(xA+bb86y}|t5)YQ!t^Wf316ZISMjy#aD z+x$xjT8tOzmg*Z1?`(Vm4HxSnxJH4J?aXuOD{sAllRgBv!i}eC(`BTuhqwM?W8Iv! z>jTK8TR+e7P%Kks0z6UA9#E#_=Q(7w$q;($^ z+_Aec10-;Av{@s)Zx(sa<_=i=4NmgkhVI@Wp262Qg>ep5fF{Oc3cD(4dS`nG&G?F$-lmUcJlvaxZKzGDI1d!g ztZqd|T#4vU#y-(A9w;)Hb^}45>s`|Ud5==|``Ry-TBp{xHus^si#lLcu3@eTAMmTv z@w%lROm8mKfb{QIF@njf;7FpR%{!ZsxzATgF-E%R#GTl!9X@~C0V$xs?{c2milh1v z9N9w^=7n^mkfsJBVRr<~jj_Ew8{I4UAQJ3)zkl47K;h04uGXD`^#Ut!6LCxP^~+TL zX8;FHVpJ%}JWk2vk5>r*E6{AbdwBx%j>LSgK+<5WStVxo)x^`nOVx`eA_q6yLAg%v zYJ~dgmJEGqy}-$Q9S#DaGS{RM11{Q)<>NT93rLI%dtLZb47!d>YP8Rs+8lh3SOef% z2tV7Of(n@`;>yE)o;8_^ zGh&yXUG3-F?CtO%hwwXr{+(4(E3)?x(u6+g&Iea{z}xG3Lq(;k$ik0|ChOBL`u5Ab zmm3~#Y2Vs7Ozo1yRy#nh0>Ff#(!JI@@8{l37xCWVf33b{x9XAnz3}5AjvAH^L~G}E zS84WaFFTZEFMR-}{Aj}koaS!>w=*}TfL45V{g-z}-$!8uE`U@r|Ey`831VJETc};u zPiO&4NBj>+&X<0EWke2`V7lxi8KX#WbvjL7(Q~kAelW+p?Dg1O; zDlfhWbu;oquSxNG(Pb|Zb;tl{8vQdoPhe^MXS1BE_S4jyl`p^T!gyT4p>SsI5cv*Z zZ8pByK0=4#L-0R_$QLi2W&9N6PP|VE)Dk{M@$?Z^bU?f(1hhSr%Y6xq=QEd#g7f>T zbhJ2zrbwMyLbQSj^%HDpa5h0CDL7tqkzu00%02_;`qX=?B5=XEAA$Hec&CKyns%VP z&!FyA6@Xbm9Q0(>u;WA_GSJ@*ry-G)R!<1ZoXEoZB0-TPFc&0ZZ)qMdszX80d&f|4 zgle$=b~=Nr3ciT?2)5&1;NeM>BR>He^xEOzEE@&QQ-dAvj|qN`iw>Ht@g$Uo%9nA6 zy)gm4(o5t~KvsO$Bb1WZdBid}toKr&diz=B73P$fQ&jMCEcg2-Fc8S=^>rUu_4pi9 z76O~{&tOc%t>^tcAxlX9>;DY-hXdD3?~?q7WFT)k(}Zkrc`M+FJQG{s_GpQVn{{$u zSJq!sIEG|rV0+?>B71ZDEb~F&oxFsEK(sSSPSZfs<3q(qTl>H&ejv zb9$L^Sw(ENA6BEQW9lbxg7n5dILae-V4tjL0<0mvRu9!JBnN^_ZNA-&{mlnp|Hf;M zw4iJ!gGlblKTKQ)xsy`+<8K}76LOcqSNj}6C-@rAiYe^jlYY4e6;E8$`iqh_{z)@k zBRC+lMJQ8ff;K%LHA_1eK$orshgT~KE=~q9o}cW*-s#@U4w%U$K>a#zNV|wKM}q4> z5~r+nKk8!U#+f8vSb#RnqDsK_qmcq^hebZdDOgTo$TKd$kfZ8E`n*vQp1_D^Og0f{ zv~KEe@Vw+F^e0=ppFL^T# z!jcV*3%k?uaA|3)ocs0IYoM8{=bSQ_w;r4jC_FRfqg-AR2=SBqBJ|4n?z%ewsk8On zKEH=z@&*aynY0EIQ2}uTVNb_ARdcdA`^*E*_jr)^*;GuVRP?(_^YBL_y$!jhgg0d~ zBvd_LQqh}$6FZr<2FT6@(4nR2&fs!3O>d(H`1w3AU6!roh(aDSZ7VyD_jAap0**=e2QK zbV<`JhU6t>v=u`d6J(OyT=V13(|NT=$7I7gslI?mudL<5;-yqG`<^_!0k(i3d1fq! zsSW~- zj7zzX?LheeA=AG+ovDIQD&Lim#g1odk^?A5P{l)B(ZnN!ptTe}D$1|!cEaz=p@$n!N_IVEGua)MNY5kmJYH;5@GQxw@jM#(T4hPa?G;d#OW5Wk^&Tk+! z)N=hxU097K_gK79ZNf}s+&H+F1F26eMp}R)NHCY>o zi~+AudNnp~*Gu~rnZ?YV*cMFULYn(|*6zvjur(1RH>lYk959wA$Xpfm3ZL-@fG4>8 zqdXEv`Gh;n^pVDk`|}@#(?L&=bBVrIHnm7h)hJC zcKtnyh*DiKMk6`fk%s%!w|GX3%bw9mk%xHWY9_FR)Q}BkIn>o}V~B6858zc4){JE0 z!8Bpgu}<=v{u?$yFo$#8#OW;NLzWk$$WCzoo%Xd?mov1ljW7`32AGSl_|Lwktjn!= z(z5cUf6Ka?$;R-yg@j&<|1*9w$x;84#?+!$a`;6W&Zln7-xO=0O4Tcz+9})O>&NXn zPaICl*2R6j6DV>z)Hrt9Ncaw(cuwg^9?u8K3_umrOyS}_S~qmE%6;bXQEkQ@7L@Q- z^pW02sUUgOs@J`6+UORNj97iz9*E=D=&FkI_1#kwK`vxjlR%X#uIabdqPwO3Krc6t ztaMuS@@=9>zU<{NVXGrP3$Ix0FXHx! z=$m2|$DEUC#>SIZTwZ;L`mPwko_*biMNA(b84^6~7I{vT^#ZSCA)h0y+A;OX=-rln z8A_7+;%kOAX~SB+FSa%5vQ*Iq$#tQ^z4g$0yAPqp=jD?&G$~}^G?etpaQ+99<7K&m zBkxGQ%(T7CpN1XC#y6KiUPqZ{-CV5oU@lfXm)F6F4Z!dgf)aMlHoguWg(-$IROVe- zX48U;A)dk0StBV-+sLWlQw^2r=hZifAru=D=rtE-6^H1CvMSSlP1@DZNztb*U!|hr z+koE)dGc6R(?c*Qw4I!q)W+xZ$5>Z%C=>kKW0RN(um+3Mzcqq}JjR9wckJ({&1BY_ z-~EL%C^&FPIb-}0(%%x=Vfkx2_-b!4vh%*{jpqgs-!UZ`f?l@(?21Lu@s{S3P43hH zq0sx?%J84oW3HOvUBas)$_UJ@K2gq;d}i10h`U#s9%YKwglT>nndr><@*LYN@|;g_ zb;J;Xk)5u0{VF#~8-V?O_@d>K{9+Pkes0?C0qcCj8*J3(L@4sS#L{)oEn@U1@%EG> zpE<&@E&+2SXJ3~fftelmy1Fzj2T9M7Li3Wys4~JCqn_;VXscz#?Ecbm8Ujq{XKUzr zc<Aj~(r`Oaye^SQZG&(#KC9ycduqR2h6S@{K*?g>m?o*6q-uCNW zCDW|>I8uf?{EF+}L;6vnw=ff!?5M}*wV|rfY#+;W4uiqW;re~|Qi}hf#9L4Ko`DrE zGAU4>+SdZP@pQHj>>xEnj0-(Kd$puKjAoZ}d*BXRvhWQlYsa^|?RRj(CK_w*(2n3M zEkf+&=|2B-eLa?-^(UI7G?E|ig3dB?&u_&9ecWku zs&}hv<7LbKT0I-TqA~o+vt6W+7uok&x-p5@emY@BWXW-IsLD^NTP?2lExJi6H(#ii zH?Kcmc?9uIE<&YnKpYKYVaec{hFw0)TtT<4^7`nbv5JhP7mxJ&X|qgQ?$|{jNqXVI zKVAfBeS}E#mBLOxz*wcV^d>)`W@HT=Z!Kj6S7h@kW~g26wxk|9x41Q(MAkT)Rq~G>YUO@JfAbr2>5>5W4gS+zcIF#c?v^I5$D_ zAzon@#0@6t++qFc6jh~Am3x?aCxpQ8t@|>+X-;o)pVL|~=hVD58rN;e0mzuANc~qvddWzcSGU`$vn+>4(#0UpLR3rq z884R6OjEb@kj?`Czy%(M4*R^2ozGYPYSY#+$a3jGE%Ep1-5>@gTDR7~Ls+o=xt&{I zaAyrf@JAW3Q&teG{zPhH>;B&QCjwn35p4-{BPhZpRS_4yRXq5?_o>ytAzuj&Yiz5> zB?9vF)yhA2`VL2v`4zRl`+XKdHd<~agbgSdDQa}#;${_0wYW;N zsA!*B<3Yye2a)`c8eTSb;rny(Y=%Xea0BH>%}9&plNT+c8|+|z=8)Umt=g&w8e1yd{=V&^ZwtP)3j>c z!wGw#)84Gm_Xk;X$M4K-^n8B-;MvZIr-(Aai!`p4fcTlV%m>E2H;7tsl{T*(1GLC@ zcejEqq>{6H=9iUJ_cPzc`lAfJV`D|*x2Jwias&QKhBZSL{AS3twsf`N)Qnz%!GW-_ z3L4rf(O;@Fqx0m$F~{8cifEB-;;eV$W2>x<^C@7}H8*%^LU(vY=xtKt;jnz_7~Pzf zTpiUOSnMxV@(EH^c=F6y)s~Pu8ud`qAv?RC^?vy%9`ZKUIM8lRs zY+PcdSOp6+X|?i=^=)ZNpJre;UzEmLI9=F}8Y+0mLr$Z3tsRIAl~*W<=1T#(?z7K) zZoZCD^`;76+*?~8Gp#1FEbCrU*nh#g_g!Jq2ljrl;MUB`As)MqYKGO$q_I6mT^m6r zD$bs<+m~S|4{Qya)4Jl92^-RGhz>S7;ar?9{MF*|S?)59!@k=`(_yw@?1y2LnzKV` zBQ0r2V)Anxtq0z?5Qi=_tENZQ7uY9-yuMO=r(KQbF%w?DqeN{ae%IsSd_N_2m***J8Ax@L=)cAg! z+d%r2Z(4lmQQ!K;>2NlsW==OdTA5rpLv1;^^&T%&bx&D98rI#FkLRD{O)sd?=Hq8u zeR5PT3l{FF{zmXREk!8xHd-fDKinlghBNfcyoE`k?{GuM(f(}7;n(PDaS*FG6 z4Y2IEP2U-D$}Sl*X4vsgsjRD6&{q4B#S?u#Yl|i<*}uB4CdSp+lv!>4@n3TATv2ubm~971Jz-Y0TWVKa?@M~f} zF)N;qAL`nC(Kjgj9t|fa<>A!U%A)_heb9IIjJj>zUbg=m+!^;FvbsyDXV+5WkC@Ot ztjjM=qyRMQuNp%ZY7j2Jeh<1m?Rmktz)3^7Z;5Ab31hw$6i+^!T>3-lW!u5pGNEXEU+q^I90|yGy7L#s*I0aPlpJ3B)B~K_h9(S_xMR}>Gf+%PI=PtvDqBq ze;2x01e?9xMp8O^(}lN3F@HsU7pvt(qfObdsxsHRpeNlm({@LptM*ah0-i6uXXFzI zm_vm!iR3$2Yhye*()kVEW(ud$I=#+iVIHKkRg?2essVLerFB_6RlTv;SjC=2UgCn! z3&yH%t5#_sjP11$C9RPdEL}oWC#cNG!6>gwhA$7}W{&EW3+LBs>O_CV6HSJ&3XWvM z5U?@T%;WB)BHka|!A0fkROqTHH2;XTMNGB+`N`Dn0FC-#S8L&r~!Z)>6sB z4b&c&Nb>6j@A7c6-10{f3E`+S;Wrz}<;%8GEGkwky}-z* zKF~anOryvFYTUZnDNMU%+ccgtfuNW6!ei;nL_9k#@6gd=QeNHRy`AnfvR9IxKs}=0 zLZ=6NVgg5~`n<) z=s*7XX-~%`%h#E)()^RHlFz$d#JCMqg=>O;;#F=q86;nENXRDn@mz;34+%lKogOni z$l>0TaHo&fFQlurntC~C6ki-K_F8F7lDY4+OSeBNgnjEXS)yRcz{#VmrCOSFJVzap zXKnjhMb*{l)K;P2L(5;QB1NN~EyGXTKFl2P%z37@@9#QG)YN=Fv(ot~aw}>8AO$uf z+CCLlHAZgtZ+Ro%70$6NY3QXdj%k3iGzS&+p|q7frscBHM32#0LP|es8RfonloN)L zhAO*NGFq=J#ami_W6r?h9!SpGUuv|fUSQy3>$t(&=bXf-M3hc@^@R+ko9ffa_?v50 zWzZ$ONhzi)5xkO79kt5Z`bVh=-}MN8j|hH>ZGP>1oK* zpQRstdl3)mE)k{=iP6v->gktft#bs;bmCPDt5g{~mqSl09XAqOE36;ro}9W?g(}ok z9)8?TJWzp_=48#JhxtVhanSYpu0l+d`&bjdAmV^f8g|Yk7+%pC__8k+>%=Ztvr~Ay z+|_rwJ9K!QEZbKVKf!r}euy}K)S(G?)5Ut+5%htebD*A30v$J2s&0DkT&UIwB_~pKu6!Qeshlv8V zn82=>(Ra*!?@d?FC~alQ!S~ZkG89hy*W}eacj{GmB}Eh@_{O&<>eJY@1<$ajPHF4+ zK30OAtUx{-X>^S)XuS*jVmp5G#Tzwk{pF6gAAgGK-Q7jbs>8|2!gzP@3f81pTszjy z>qBIH^BvAccl@v^pvU)Zf4iq4E{7^)<0gUJ1#cq)z7aAcdV+85+V^05>`GV~sve7n!!*uH1(+S;oKBU^-c;`iw?@&ko+qx%wafh~K} zm*8!>m)e9^*_-X)2K#xleQ0_{8biHc`AX)!p3!@puf01lQ$0^(h0w#@ukq@V@<%uz zzp0}G#BUU(=t@6VgB%1Oc%M`*Gc+$Z@FAm2r^`+KKWg5$c&bC~JH0+cf{AzT{mHhL z^ZnMd3Ioqq-Y>c@$xjg>1rP&aNHhKAoTL#PmQ7H>*S^w@C!X1gKaojLg1csC*WpI2 zx%eN)>~vnKc=gcrex73n(3uom!8$k)kTQOOgM>^kg6m2cev?ZK8-{l^$H-$naZ$B| z#b8^bRb)Q$5FgSRD6g-q6Eo7r*c|6!9f}mGJg{~E1mJ<)Ku_mwY;Vjns#h$TObfpg zB_J{2ac_B@A^cU|$CVH^yBVguH*?QX73_`m;k3!c0)g`TT4WL4i4-}ak*1K`*j_|F zG59t?Op)Ci9>pxfzE-!PjvnT=JAC$hCHJ^VnksXD;uP0|V7l>D!DHWsd1CesT^&m8 zHh&Rb32!EVnKhvUntL8n(@S>u5+08=Ed*ffn@_t<1V=5JoPV;8RL&P%`Y4AYf~L(( zei(JQAx!`&yeT;1TG@<*Y=~G~pP75<^U&%1^RhcEHp`>+eu>S;N|&NAsvK({LQse~x7d^V-p)%cNBoan+|4fG~F+n9T45=(k__MYlg(UurGuGqTTw1px95 zV#?u>F8+hvQcL~i5a3^O+}vkt(a0JU&~~!3O9BQnM+oDr3bnz}!Zx>4I+p2@yF`hP zt@#t3+##M)$wp>I-wwLK{x1A1Y0O+DDPFP)=@qjl|5VoZaKIpO^0M!eA{CC4%Xw%y zq3Smt>&u&DeTH59?`U=)5_%S%pDpjHZq-D>X8rZ84HlV4-H=JPrn1>IJLEveL;<@V zzbcgP4mU>#;)jc-o=20m5s7~7-D`Zy*F}B(RNK-auJau5GHu+Av14+qpEznx&!+!S zi{M5SE^7vO=Z4Qcys7EEm`2GUg;{VjQ_o0Q6H+zE%>Z?9L3e(^Fh}KxO);CHdArm3 zv9PW5{!szRH3+<|e7k5luwA4wH#wUl<(?zoLfq_OyZ`rKO1dEEW8Y<`U@vWR#E#%2 z-%T<9_a{1$OUYe)T`eIhzZF~^x%$bq=JUg8+@f%-ktJoJ8Q3rnITXu$OWwCD5HZwG z^v7KfhOiF>6PJCzvp$Y(eyIu;A_r{)$))tcW&Xpx}ncGbF zEE*S#KBG_yR$igJm*?)%E16Jp0^<1or6W~1q^SEbPJFGH4N*&1)( z*&oc*vf|EX*Q?SGAih(zXBV|2b;>1X!f^*r#s-sWRNF&2QA!u6g~g(re$1S?_nbc7 z22ll!a-~!4syIkg4h2uW z86-d7ve>!0ikHr#7|^UmJim|Y4UoM784jn@9#uCi*_1*fbsBT8Vi=&VPV|WEZS`X} zxHFzklKQem*wdvCYp^pW+-Ca~i!618s5DHTUBR-*&`MRKErp-jSJQ_YwRzkb*n_?? z-4e4in>73YIM`(=h$ju`m18m6BD4WF{7O@WN??P-_*a+Xj$0QQwQ`(OP7B1+v<(dxNC5j*A5+p zqvQ5C`RVrrm>OYomR}&fZZ7Bu`UTzY>cJ0?UZwkZsqIW>cSUOKEAM|Y^Xn$eYFndZ zg!{QTT!P^tl8G5?YUOjey?V}H+EW{v-O{@q6M%d}$pdbM{uCpJ66G~9&Key2B!@DO zLa3dqZJlG$7z*hED-`S78q2JrdjZ&jE%Rb!fslqU;t1v)({(%`iN~w$0ik`{p?i?&qm5LLu%Q=^Ft& z+8c=rmwn?n4A&Ee(Z-_e=<6N?ZJWO`B z-ShS~p%h|X*RYuSl#Wm!4wrkiJ+;H?fY^Cam+UTY^`ceh@*qsbr^hs`HJ zJq3TkIkTB`B4DKC{6LZxvJ)&j<&mP01>thVOP^i4H0vE{#SB;2)=^26hEm%^i7;0@ z?I0L1kSBKG((7CL2O_gfcl)I1Y3$h~9~SMMPBmQJiqYOc z10)#}6_vh678;dS-@&D8^x%`AT}j_2|Jy5ya0QmN8ck(Kak(n|eh6kF<$iONB!$0M zq}7DYP~AqO;(XZbX*n82QMsbBp)sb-Qk}L?KHa+`hZCoizYGD% z&*l}dfiy2mlJ7CLHlul;M%OQjjo;ye>^%0I&nan~sapFL(eX6R&%>}?s6Wpt+xFf5 zgZfH^0Iaq68iBLNYnBb2#fB-tHHL6W{9cRu@~+n$#%k}&4~Mw*F3)=Tqo+3bUDk># zr;|(WiEm05-R^fp>%bbpszv~T3Pae{3jt@y!P*Ivn=?ulQ_LPz))DsP!+k;bM9zpG zAsXvf3f5Tl`pXUbdk74NJCx%I1=5@+oeI2OZ~e&JRI%t2=_q%mN?0*unHL*DJ{cl6 zJUn|Llq6!ecXW2hFojJ4^iw7r>%lEVGt8Rnh^9vlZsN0!X7M{2-}C61X_)%%^~kwl zz2-U9P~k+!N#xA2_xOmT(dw$`@{jU2+O%?bID^Z1-fGodFP=Oiu+W2?&Y`7 z-rSgv*O-q#J8G0ZJ3y@;Lo8`mbAf1tA&MPux>kUSg7rZ3!m<f<{Vx zZ8J$qs`0!H&WyE?_d~!(;vwI{@JP7d*e4pC&jAP4AJdknxU!nZl2c!^vSb?g@ew2JwN%nR6wAe`@d$DAeNZuJRk zn@C4lWsyefO0=gy7M-7Ry*KJG_c=VoJ$90WAnMhjUuMf?sfweD*8{7DL2NgOp8qAZrE|-hC=#s1XMF=1T8cL2lEe>5)$iiT_ zl4w*#cGv5%peJwMgk3h9oyp+n_+sXH)-#+z3M zCn}n3!}uQ@+F2f!^H63G`456kBM}kSlx)mu2MpDR{WqtCC|%Gop>c`v5=;0iIQqq4 zjg5_FO9_`C354N#bbHri>sHV7hXBd>tVZKD(;Kb_NE>QMuVxni;`nm<8-*D7y)SXM zsy$Z)pDptP{bT>Sl9lK0LW6-T}iW+L=XzR--MinjP0O3&4z7a9F1&f`zjXpd} zwTMNtw;_5TtOsR<=^nG>^3Yu&RpcLyi%{v8Jlerkxfhp!X$qR*2{x}CNP z*h6the~w+z?u~|1kb90q$t|E@3`f)^o<{cKVyC*GloFx_-EounftE04tP}+&>)>() zf8Y0-g=aD2IkI`uK6nym7>`97buH5aqNp*R(Nmv_!;m5%B`59uRXP%ll1d+c3DS9s(0DTrp~r5M!UJCj6(M-GAVQ4cP= zM0Xv&MoUWmrT7<6zyW4OdOC5Obj+aarnp*lSPK@&|J}M?jlP(+A*i?1gAHPl6?Zbz z_FK^OZRMM{(7#W|0g|S`1||t-I!zawMoVQo&*>S@1GiM&Z20Haxi0C$^Ho4a*5{WR zkAo%xpe?n)G%aqT`R;n}|GF)QmSzQ;>)#;*#Jm@uOHo*`hd` zy~FNrXDS30s8SG(P<4P?*VY>I87V+*i%VMyptt@VU@hMfXE2#tyLb(`sZo0Xx$HDY zUSwvPz3lW#t5Xm#X(OcyWnSm);mY-&{ z7ka1pyNvwQen8jFYl$XK8oj8xn)3|*@Q48AdW38mSF-L)k_cOcEUdxkZcAqlrKWO} zS=%tRATF{o9yh465BdK=5I;PSq^8f6V98Zf$<>bIeHUQA?&*=38OjE}q zrGjRS1`bGgHD=onUym%2#3c1?^dhWUkd)1!(q;Vfz)Ff1RuYfGp9euESLk%SMAEJ~ z0>d84rv`W8GrinDN{q}QY0>_a3Yx6^8NR{5*NY1Sm$ZKxwZ}OzPl?_tRj@=LFtN|E zh)QoAhkuTQ(aX%Uo_(MWyG);t=Jg6aolN*Etqa6{YC2`a*Ve|8SIn)>83*K5t%|C2 zh(8pvO>EcXf01gn@!aD@OAj0saIot=hK-NcNa28hRpF)ibyP)IqC!@Gz5I_9$Qtmu ztb~D3z1~m`%7Xl|5yUN#8nEIOMf>ym4HY^) zhi>_!^a03k$4-}S>(kNmEM4ux|L?V*nSqj%55RJlN1d1NF!0@OjW3)w4P1#FdS-*U z#^_ZCUIQJFH!fP#Hd5dh)rXDN6pFGR?b3lHqE&tk2n2r&I(p7IG92d#rQQBZ>zN9$ zEqDC=X>qx{ZVY}6Gs*g;?Uvxeu!Wh60L9g3iR9{pJpXPk4;(hcNIkAoySw>f3#I@&e(E7VQy#No|}K!vctp#><|X@9P8O_62XJSC%~e*d~vJ)kjVV*lrs zxJK2(dZLbMg0D`1FB}rJmF-%q^J+W}rg-KV;4saYmb*fUTw2)L?CnIwt5pjJQS<<2>ubz90S^=Q%+YZ5SEJUpP3QcZ zx&-Tvk_KsEH7kRmwcX z#v+M%_0Y$nDO+N2{_u6pXp!mRw&D2a@Z`hs-SQ985s*a-G>I5p z_tgjGKA?T`w=)aF^?qRdo;S3Rm$#lV91jO_Vz1fh2Rs;G^0mUgpG4SK z4m4KmCP8EQUFff0SN?IEePqOW;YNPgM0 z^6+pdJ)fM^Hl(L!G!!!?{0C@9`DTn4)c||^Ei5g$+hHL^C@)aR)F$Uy)BOh7`{n~A z`J2o+GfTQ{s|VGj>!640pCO|SC?o}9sD0Ct=?7q^OCm;=oGg+?O>MJ)`lgMU_*VA2 zefOLlMBn}XPw5MNVD-Mt&qGsLgJ3RbEgVow=FrO7e!$WEQgGDi-|8|^u0MyeVn3}=k1?9_qM4a$zhG)N8pY7%#*&}QAQrkCo(nwT4 z+~syR_u%DB=DKi0`x!_7?j{*;e3aveiC`CY%`L^dLG+iM=WZ);T$Yzd@5aqFw@Z@^ zQb)|J5&Jf^)K|xCH;!Ex223ex^(39^5;AAz9|h$caO|7AgCBfPi=nA<`u{e<0I=ZT z>vUOVNb=Kl9?(1#FoA;MGZO`kO+ow@tT5)7kM*%ToaCdC6VH3P^)3Or8nSl{Ks+T| zoir~=b;3o<@-FOsYlDIPwiHNP2Z$MsfWm(MhMOC3Nc5eU|5~O6aOtv`1%D#krm)U- z9+)_B8*e_Oz?5v5|2r1|3X4A&JK68a*18*^wWWIdS`Ar{l+EFpbSgm4kq^g}gKeS> z^&dP(b1i^Ubzk*DCDi8xWJ)WJ4F2xIUN~S-pNZhpTaEKP?ly%8*!eZ=-pNhQsEWIv zkFUj3-cjycAAl{M^GS*z=#Q5ra~IY#0rZT%*}9=d;|rFJodra^Q`=(mf&l z{8O_8|YmZ$86(I4ul-F08ZD#-X8hMi# zmA95@@cgjs-q8s!D6DZF5abH9w>-n#JhXD=rs+Yskm3|J0HNPlAfe~vqaM|DP{&f7 z2NS!+#6pgnmjb5){~hE2TBgmY>B+>wUE=`I9&dU%(EhdnD>)|$3uH4Iby2e+IM?Ivb7|0Qh{*C=12ckEzek>@{9hQQ<=+S z&#}`zdpp`dCrYl@Leu0Xm;8`IM?PHdX?zGL;_&)-ZuQyKM4BwL3EV+P9LQ&Soxce( zKDonrV+hJwyjADl{y!evfZLH|Ihx_gF<4FDqAJ|(*F6ORA z>E#2dCtrH=i8l>l{5van(jD#v^mNKs0m*7>>FYq<&ZlZaQ=8|(;iNw!Jp))P|3av7 z3kW05HVMwg6)&U-_(T2B7Hw5^{bS8MOwsCc(y{+X%J&|6B+Pv0p4g9uj?6yU&lcim z)BH0EpUp}t%I@?h54;5~cK@5K3FzV2Up5QIG_f0gwiNo|bSX)T9AH@kp>&16JVgvE zdK>tj)xb&a#<#fXow52_`5o(*%Z%$oH3lEYWeg|r6zXUFlgQrl(YNN zi1LGlP^AmHZJ!a8e&Cenzk_~>WrEJVU~j|N@~QDuM_G9yA$`6j=>aR6E)58eLfQks zJKIah{9xEclomPkca3F~!RbGfM@awG9I{5q%Thj|vTsg_e_FI-*VlLvRgu6GyCQ`# z7u_xX4A+}Hi+CnwS5b^CD6JHN7f!pHj7+U zTZ@5fPvP=_uN_GCiI<)ws&P|Tp_QhV!`DY<DVlJ_d`4sx2KNW*IK!*!o5R6y?UwCG8~G{iuWplK=tY0(W?&> z{}=R^PID-iuT*QN9~MN=VTqOGi1#3*)1q!Wrm4S21n_Qde`Cw^yBBLvMp8WWxoAil z1QN6L=3Hk7W^>(mdL{rY=g^l!W4+V?o*o1B{Qt(J-^D|mepp&3%^OVNuT;5$)$;4u z(%UX@R4tEY(!js61Qma^u&)9jei!L&bs%tQ&LPJ9 zU7TLf{L8V;B4}vy&SgQ6DkMVbqU-k2%^G7Mm80z@Dbv**($5F7b(POo7jdNE5YxW{ z6A#4I-eA*7qKA?mkd#-t5E44d%1z=4#p@n|=CfdKsPl;FIM|T0vos};gWER@%;7> zCAH-CjlYyH5CV7Lu0|}jR^4^S7GWvQT@D z40i8B4(%NQTT_KM`ge;!WQa~%k`#wPOzZP*@4L@i9O5emnDhip$UM)|oYZL4 zOpTg|eE90$H$%jffR`AGPD!IbIa1egyvK?B%NmG`4G=U8km35^VEG`}|GuUJvQuJw zawug>)pOnd-JWSU#9=Fnc4X*KE`n3}||$z`t#=+pw$-%ST;x}b;)EV_jnbq7qX z&R~04g=b+O=ubY+>HpEC^UzPtodX?_t(eAV>3v$cxDRH>a+vu)Dn4@n#Hn7y7wj^U zUHzY(1Hh7}xcvwJdqEN?M)?pbRE8plm2GCikgg5WQcrGS%ZT5l{wn_8UkQSP#$zBa zlu85pz!_!RF=KHOR(Nm>TfC>vzr%Uz21qwE{vStrA77jZ|2vrWDeS<>vj2um<&}RM zhyD54G5O*y$iKhp_p&zZJ;IX>lLx=&y#kBj6?{Qp1I C=c%y( literal 10227 zcmYLPbzBwA*I!CHq(mg88|emtONlf{cOxL(Al)eq(jZ88r%HDzA>AcN_d7nnKi>T; zb?@F~=Iog>-#Fn)3euQp#Apx%Vam!#sz4B21NdDZ1s;6wLND?LKTto&XgNU;I^Mr8 zI4C`n2)u~wEc+IQybgy1Pbd)kH4h2AL~5m`=`3MyYinlb4E}^52}d&{XERd@H!EjL z3TfH5N~Wx4$Ph#U$x6OfbDux-@zBNlIsfS29|MOgM#(&Z;xAV~p~iq;F9+-X`ZF5s zSBsZNccZV`nSP3=p7!he9RgqdaPs`_@c+oXeT$q;IWU~bKjfCR%|DhmGdG8d^Y6fa{lWzf9vpcxh~SWnM*&CJ%ctNi zPdJGi$93&Z+cDzE3RFM)_x9QXjA3INxo{FDB zHS_LNTuxov+U16%b=UG2_}@?J@zbbA5unsPj^CQB5EPPpvHh(b4#KoeaI_*I<^eZ) z{Wd;kI6e+XXVdoh6IuyQG~s`lLh(1bwavC`{> zU09#me)zBIW)XDgL*9n2xotz|IrTeEizTHBj+QfrT49%S)Wy7RkqxIgFjC7l5Xc)i zSn``vU%#X@m87C$9eZ@_fRp$_8#Ks$UK0_LDUl1Ln}6Q^Jk$b7zM{{Uz zoYl9AV18Fl=OeP1(X;UM&GYN^E6X-*Ozty-Qag)Ta}%K#0>&|og>;aZ=2RUwEsN_E zS5rQC?jOg$FsLleIn)XWcWjVkjrOB)Ai-$8U1e6Wt#u?(3sYX@;in3~_*h=^yZs9U zCJZ;Pe0vUgwC4PsYz&TQ618}3K1am&u)lq4Q^$lwEyGDzlw9jg|CXN!HB4@PS#s+j zsC8zEknHth8sW|Rn=gGrv4OZRxna1H3dKF4F_{vd!v0;In2X$KGgzp;yiFc-$2TT| zV~+lecd){ujn8jAtyr0``Fn-}78wXVLBQ9x{Bpe`+uhxMl4cq=MTrdRLdLhhblWFr z+Qw>3Z6oR1w_Y1-C3Kz_K%?8%?fl^GH|_XZ^bFUmZsbC}9I2c5YN79&!XDF_(^mbX zRg*lJrM3Ur{~$E~IdSKE?0Ul7l-H@z zUbENdqtOr5i=k%5>^TMf^Mp?~q*e8{h_^R29p0rH!b2;Y*55c@$tbW@l-e(yWS>PM zKckPAP`4mfY|+Cwy^$mx66#og11gDB%7^|056m&Q?Y;~6HxJ6kXxUzv*c`A2P4DzO z-A$F%*dJebQ8n;H-nq`9X6d?FjEIJnowJ#_9bkDD*Q2y!^epc1M^G>BEWp-9S5&7q zu{)I5;3#riVn;0-) z^$QkwcyWV*{%kqiK+c}dq2DQ*$@ua0G+Q2-H9wgD9Q?XHV6PLcaL|vr>}<}y)o@PV zI!)%Hi(T&tZ1q!PY{M7I={NF6-4LXrU4=Oay(v;FB#@K4k#RGD*-1zbJD4=@ymAHw zH^Le?+E9IzP@? zF(VZ+B;I8N^?1MJg{uHRHhRIyc@>>Ey4K`q7y|Um-~&Dr-mEmeis6O92%eynde2jw z8)CJEm$98P(4W1R7*o8Dg=da(>cq^mawt&@iHThTXdv&esPXO3&Ab|#(~R_dPRqh- zSJ6e`AZp#QPy>(FITbEwtBT^^s@(4CW+Y-?LWB;{N|{<87G3Ajy-RnBTS)ZZ(c zICGWKXl@V`Q^;gv|dz-l}p}L{4!F`qTC# zHr6)#wL?p=h}SE7U+;nDwvg>KZY?4vA9{I`3x=F1? zZ|3h(R|a6*emHS?<@g#;zODGft(O|vP;^t8fb?;V{`F;#OBc=gm@Z%isZVsLG zZ}4byGc3Sz@7U0loK3z;a7wjlCzE(KB7+$8AQ>-DYGOGmc)fi+PpDklb~q^*|3#Nk z*Fp5}mjp?AadGAT(J%@2*SKBPBrO)74SENDJg0m1g$zBIoOQudo%N5>Qb_I}Ws9^O zu$MrPNJ_iWz!%2sxZJ|_9!jE`$@_U}=hRoit?ze}nM}6HKeyk!S9o1-?saFg8N(R{8{m$2lKG9E#X$U(Z!gx$W&` z zU?M3#zK=q0&D2cxh@|;gv{rE(_s@SX>zl=Jx}k5#zy3=FFW#B>GB=O7+W^Dol0x1k z?Nydw-f!Mp9mT(ft>UDF5CywS9w8;c*QKh9*^*L4l)u^zstnhcFD@T7-YsJ^CzwCi#(05NBthto(#&-)kcE#kHXLTU$yXOcX9pZRu75VIC1SL5pJWfza;_!53GOqXmTa|Gx!^48IyfOU zfQR7e_=fd;8miP16O<>}#CJd?(%Lt$hq)9zK>eu>}19~t)O{v)4no?etMe`)JcnE-)KoP(Q&P1^o=>N}gEAi_;B%ClyMHbX z#HF=^+qssKrZy4JH600uB?CiKN=E?d5Y{WRN{BBjyl+G<#K)uhl5>-?;q6##3F<{- zWUuOn*C2I!Jm6K&K7EdyISdNgSH!E>mJqHK6dotVPYSnN@+zr%{>yP?E_H(367##I zgu+>47UJV)3v!fpDx=k))SfGk3;*e@hAvdBXZD1PL*?=cY`r00*Y3Y!>|Yl3Sj6b> z|E#~n{kc`u5Ln%+I0JS3QNN;}=e4eveis)y?k z9)@O<-s7gP+5{lC%lSyP1zB&FNB?$>39sVMrti~cI2!JI)k>qSKR#vYH9T(L;wvU* z>St!^ldSBE(Y5;PE;$J2Qwy+uYBQ8+M_f6``{aW*hAv7nl7scWGerlrgx`CWj%mBO zw5>^VG9a7!Dm-3QseN5^cTfO%Kkc^-E#8re8tr`5pQ1s$#!$rr_{fq2#4_aRLw%R! zR*)T7GoBwsrAnSAjsI>ijt|o(JR1Mb(`R=Ru86%dk`itvYp{IJv*n@NTqif@ zV$A!Nla~__dLJVjxWsd%i@iH%rb+QO<@ebcQOgIn2Yp*CNQ}`2)Z#dHEB5vE0vb_r zx&EwH+NT%p%gpRv^pp$sbFyZqWU)Oe@yj)zLUC(uo01FxJW*fxv@4|S=vl33DO5Dh zfJaZua{Zi<{>;GeQ}UhsRP?H|V-|Sp{yRAc;$$aZ+J2{K`qP@#yundf1vdR^7F4IF z@k4@GG^S$_d~deDmoO=+UM15@db-Y)%~)8uif?pGH{0NH>rdz}((ANr%Jqjtav`@= zba>2MY!vELj9baNG2ou+lW;pDW<|BJ2xG@C+_U|xAE$c?#c5@brqyzCSiH+E2s+5N z%&Dy9G{$`|{2;tFiVH#1lRWfv_So6Q8hrqW{hM$54YO&e!J;B;z(>`R$0*``USRZLg-FtkCBuU9u)Xp0b`)cG&+=x0)sVL8zqE zz9TDFN<46Cz@xN*u5@M~xG+zvbj&)A=0|i5_=S-xxA=fYfMvHL@AE%Gha&$OpP1xY zeu=i*TJ|<$de2$@_JP$cyM)!Lq5Sc?QTAHBy#e+uWPscxgF>RM{~HD3t8(}%#{%~F zv!<1qPkBEXp0gcOA#-ymUi5){!L-Gdy?s#>b5hG>B>9O+!fQ-GbhrSQ!z8K`du+x* z(|`)!dL%xb=0&^Be%^*6?tv`CzY0fwM zudb9<6Q5>{JjP1LNO_sc-yae@x>5T3(yHXRyn{V%dVL0POZ}sRK~@DBk9ijpQ;d!# z#$!UdLPOl%KV9zLUEwk5zrlC4+UQWLvC^E=i9Sv^cC3FcIG=uV6BlMtE)uz%aTv4d zpz&QR*t|6MbEsE~<^q-p@ytk8pJ!rEM^!!QyJOAkcf)7idQHiNrW!fEuB8m(@v%bb`=rB?K`;KI_S&Pkg z%G9=i!kI|xAV7Q}_JP4Vo8@A%1nkM-uQlVOHRD2cmnJMqhg_q{DHZg5I4;q*Me6+n zxBCFg+B{tx7(vIvq30@w^_e9!)1EpY896Z4A%NIu(msi~I9P$;x`($nt@Gtw!XjGV zG&KQ|JiH^{#P{Cy!^ahndj4-;UKY0SV}CIr?Oh%kn%VLr^jv%K1k+l!F>-*%t}T=; zduZven?VmA5;DqU?GR62!PBBCdu+Mgab6K&*Q*{ybKg-qAuMM>21dO2AJ;86TgQ4o zbC+dO;0Ta<$5zq5OzLe+9C;RwMg|xxG6}YCd25}MXx`LVr3h{uu^~8e^>`La&zLrPO~#tfd@Rwt-~YMut*Z*( zeG4o;;i8B&8~w7wV(vz?#P11I^{UQzH*K|0NcbsMyK*w8#UA5>UZKLG%@6Y0;47uZj;bY&&~i3(7|l0|k?_PT0$ZR^h{kfuN% z=`y3`!6d%unqYFy#mo%vHolshEt~V>*d=PT@=zsS=czwtMNjTy*H=P`Hw)35YrT%u z;})ECbv8PZu|KYPzuM_l+?|tt+I#KPdswEj6OXsDUuCsvkz!1^p{MI|vJy~5f$e6& z@k1zfVF&lfa_p#CTl`+F zmi7;JoUNECCmxyBwYrb#RByjnJH1QoWq5cF5vVFC=_Xt25q2Rl5QO4zvt=FvK#JR{ zfX>5FBzdAi!fDA5`WxoOp_0f}obJ~edw(`HT#XFQ0H(X^q~z>%reqE<;m2o_nFRqR zY^RNdRjL3PL}qbT>82=8<}cl1h4K?sJz7-@%VxDyz$=ti6{}63dODHq$$$SM>o>w1JI$~Iqc7MG<5bG%<|J!o-$%7ZE-}va)BDg@t+PZG=UxPuWWwbtiW96U8fM`SmewhrT1~OU z0s#jkSfpeGXyJ-4BI$Wz-nyjaza=3}qiXJLVAfL&e&7XU9w^Jv!*g@vwElM6P6tJQ zw6;TE8S43PF5Sbolou_r_%8!gS@RYnhrXJhmgW9C@3BQ($;^DfpPxAzV*N)Ph-Xth z7!@(!j=6&3pAzQn|KS|$8zH=LpK8u2EBtJjyC0)5RBN)jkZhnk`6h zu29$|R?A1_|K>Ge-ZdWF&N#LsL1lH60+43^+E4As!pYHr%{p}ghDzn6skQI#;fnxU=;m>aq2 zw_@XnD3++uyD*QZTiim+pUDM(iioL-zdRXHq7AT<)69z?k1_Acd}ouH;+N8j4IS$p zb@(+`Y;t-4T^5|)J zspDr}I-U)K3l}1S811b|;StV3lGZFwp5U(HsmmX4__p5m76zhU(YAph{m}f|cQPTb zdqC0WNDt0wRiJtM4k@gazex2(#I!D)M01~I_c`t6qyiH4lWmotrm)!%-BL7KR-oI_Lwl)8Lh{rh|6yb z4%HTa$z3Iu2Qyls(&zDZF4+|!(Jq!0^;$uM9R(jbb0t5#CVf3k- zw`51c#pWAeM1$0Fmf5K2e{pyR|3=zykiDZvHxFiAn>P9#+W6%7Kr@}vp`9|&nt87^ z1=Tiy?pdYQHwuTh^|pKqGTbmvE%)y!nE;_M-dPoucNVK-l)4!?(!h;#_bq$kGMnWS zg2yu+9y(@*N8(U@$zx8MYQo!IqL;3cH2Xin&FB`T2jCaA>G#Yf+}T6=wU%ulpP`%v z7~?ZiojX+&3^zivH$JvudG%Zn&#d@~sna(-^+Z_D-RJ~=Ad`Bs7O zvRjA}{S7R2%s^4B+a*n)m)(d@KH$aq8IB9wuQVP52vYevSv7)_9ycV|q3s7pM!tMf zSQGl4jsb;~A9-shctLt&x-E|8vh38$+3vb973mo^>+yM2=(~^-7C6XNYBtb9KxKPV zDl*fApNqMlip5U3AH!H&)EmFbzXOXh{nWPthoTtUF)q6$rZGl0?^uC;Zgw3X6}oEd zy8tx)S0^yPezaW=G7|R)R6`%%D_3Y=#sFBW9P>n{-JO^Ie~NPy4V!v_l4j1abt|yO zjA3Y9Gryiw^Bw!vJ$mg0lwxXErxHJmZHi=Q_)6g@9`DaBTi`z|Ot;b$FSkIB?PeUC z%7F*o4^1oT_Nvq!h$FG<`Jl+Wt;Qm{R^7#j;d5Z#dq0c>M z*t!DU_6T6>tEiTC=D(0_lC5xHW_knu8h5^n=)RS6Vg_^o%mgK+#r<_2S1|EXK3Hln9=yOsWJRK_4CU z&|`r%|0RkBc#{b7LEryXbx_ZI&o*=a`G$NUF<1AQZ{av5zhee@Mr+&v6abU4jd&Kibq7)%zqfWsDL@yC(Xa zZY7S}p(hy-9y+e9j$}F;D}!p5T`)JGkz9OE?EUQ_F|})?_2w2pqyBnp;E%e^X<4LW z#~KnS+cb5zqOAZ?%AmYcth+q@OiOPS2fkM9X!!OL+wPG$=q3*cx(w#q@xa)Id9)-fWYvK|;1d~Ir!(E=EM@$3qUi-O$xaTGs zBjj)5UBqW6J03R>dKS!jC6OL#yW!v+U2^(;N`TI$^z1sJzOK(;@W<5mn(}wtD)i^y z;4i9r(85;q{8fUHMs_V!-_gpfo%*i6s0Aqb^+P^`c~eypgZDPKd_j4sJy_MHWsWtp z3@Wj?twzq8(^C?gDwN5{&tl7~8|mA;`#Fw(WFQ^8YkZ#rlIN7SI~w^5TGK%0aEonV zS4C+7FYs;kUs4$oWG5%*LtV2?*CdvYM?Wh7b#kFOJjn)a1bY8^0o;z-M=5PO*p-p8 zhq46E^TV@rA*j!>Sa<0}jyh>@9te!Ggdc#=ubiCo{xW@FP3n1bRyJ5%!~NSFg8>$O z#;?wSnh+uPqp?BOZ*dMCepV|LO!cY?$`T&Q4WbY&XT zBTuU2F=BvyQIU`76xd_z2UJ#e22(3_1b$(nkTRj%vO>Mn&G!BeCe31LSVMu4K1t54 zL_9I03lcz@nYlr~(jsVP;}&35ozroP`F-|v>m9qDqnj@?Mw@J<2L!Rtm`U-mP)63> z6UV=48!CML^u3097#Rhf3*eTh-&}7uVG6pV!DA%AU0yARtIAZ>^8t!ZQjKcQ)$$60eN}Rq zgT6waeaSKU{_P1B?VBY&T>0WFDe?)M`{MmXE8qGqh-jyd2<%zThh(EX0dV>tkJYWb zK}}+>{OiYh37oR!+EMHqnXIPFH$DbCUjOV1U?+mD1L$OCuU{iT-EaZZCb`Z5rfs^m zDzU*T+h#OOG&8PnPYx>kaS1PnNk zoqk!prAh#-9B6I3F_^=3v`~|VuTC^?GIDE|kB*LJ?eGCf?z6{ZhzH&OOlk{DE7gM* z9C{V@TXxW=(ER5%|9Kp5wjOGJ2ivd;)}P3qXI@xfA5rH<^_^PT9hf@rL;@b8*~51y3kn2KER^fHk$;n@=FxX@nP`L#%I#j zQNJ07fbuGR6Cbegf$;$8B^sC-l>R}ZTcS*V? zBsd7z9vtqBQYGkZ1~Q74t!q+gW#^UMLTEL~u9|~+(oVkOrB&(F|lz9qJ;+A{bt_q zicy?pqxCWjX7Y#!py%S>pD3O6^_k~KhZM78Yt=*rRc1sCeIH>t{9$H19K+c+mNwi| zGSld`L0W88f&hhS@B4&Cb54*lvOBe$-;hBov-@5WE>RrTHAmAzSEcK0F}X<%$%Mr2 zpa}w-0XDcv%G5w}a-dm-OG&%gZ4bIwoDLDx(A;{`ix5PY{ghe%6(K8;d=!BATJ3+mL)=pF?91(v^A`m>Ei0fs* z`813&|2mXuyZ>htx0vAa<(~r_p3WBCcz7xgiZ&J@4ZfhbH{1t02f8CzNbF5{`=Nhl}hTiy~a8^EA$1~6V4ef%$SA{NL8+G4L}H<{qwdxMdw<& zo5@ku7)9h$pT%ELdQdV92)GE%BD3h3CZoUnIN37M8_D#LXm-8})%3KleV)=$^3hch z(1QX}vj{2BeWe!>q)y(hYnv0xH1yr@0)HD&fq~s0)7^U*EVhDFHsI8}g4>&hJ`Qf} zZFQPLkPcPMKiO_P_+=fq1VjYP;2D!jR4CCLu>WM1^E@Xy~w|OLlhVvj%g?>6b7QVqmn7Gk$tG4(gqmyr$(%= zX!9U}e1ZYKa`fnn|4!naAkyR#*g^yc2dBKD(*-?Xzi5s5*L5S!fD2wTqJSIl{QrJ6 zoPXh_pojoVNlXks99;L2{@-PA%G97UOkDKaBXYm%Pv;*kem?)Y@nm5Nk`>~H0sjZi C)^VZ$ From efbf91be0c09f88fac5f451247e48a699c1b393d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 12:33:46 -0500 Subject: [PATCH 324/644] Removing sphinx refs from README --- README.rst | 13 ++----------- docs/conf.py | 2 +- docs/quickstart.rst | 3 +-- docs/rst_formatting_guidelines | 4 ++-- docs/user_guide.rst | 4 ++-- 5 files changed, 8 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index f45f2dcef..b9a6527ae 100644 --- a/README.rst +++ b/README.rst @@ -166,14 +166,6 @@ Basic Usage The examples directory contains example libEnsemble calling scripts, sim functions, gen functions, alloc functions and job submission scripts. -The user creates a python script to call the libEnsemble :doc:`libE -` function. This must supply the -:ref:`sim_specs` and -:ref:`gen_specs`, and optionally -:ref:`libE_specs`, -:ref:`alloc_specs` and -:ref:`persis_info`. - The default manager/worker communications mode is MPI. The user script is launched as:: @@ -183,9 +175,8 @@ where ``N`` is the number of processors. This will launch one manager and ``N-1`` workers. If running in local mode, which uses Python's multiprocessing module, the -'local' comms option and the number of workers must be specified in -:ref:`libE_specs`. The script can then be run as a -regular python script:: +'local' comms option and the number of workers must be specified. The script +can then be run as a regular python script:: python myscript.py diff --git a/docs/conf.py b/docs/conf.py index 0a0e70d4b..c92efaa9d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -188,7 +188,7 @@ def __getattr__(cls, name): 'Stephen Hudson, Jeffrey Larson, Stefan M. Wild, \\\\ \\hfill David Bindel, John-Luke Navarro', 'manual'), ] -latex_logo = 'images/libEnsemble_Logo.png' +latex_logo = 'images/libE_logo.png' # -- Options for manual page output --------------------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 755fd6fbd..8d1e85bbf 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -20,5 +20,4 @@ | .. include:: ../README.rst - :start-after: after_badges_tag - + :start-after: after_badges_tag diff --git a/docs/rst_formatting_guidelines b/docs/rst_formatting_guidelines index 83329a529..bedb5707f 100644 --- a/docs/rst_formatting_guidelines +++ b/docs/rst_formatting_guidelines @@ -7,9 +7,9 @@ Put referenced file names in between double ticks ``stuff here`` (no quotes) Put environment variables in between double ticks ``stuff here`` (no quotes) -Put warnings line: **WARNING:** +Put warnings line: **WARNING:** -Put breakout code snippets into a box using two colons :: +Put breakout code snippets into a box using two colons :: No italics on text of links diff --git a/docs/user_guide.rst b/docs/user_guide.rst index cbdbe20fb..78ac30302 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -13,7 +13,7 @@ three routines: * :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. * :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. -Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts +Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts be found in the ``examples/`` directory. To enable portability, a :doc:`job_controller` interface is supplied for users to launch and monitor jobs in their @@ -110,7 +110,7 @@ libEnsemble Output The history array :ref:`H` and :ref:`persis_info` dictionary are returned to the user by libEnsemble. If libEnsemble aborts on an exception, these structures are -dumped to the respective files, +dumped to the respective files, * ``libE_history_at_abort_.npy`` * ``libE_history_at_abort_.pickle`` From 5db1aaef56550f54cc9d99d00649b236845739d4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 13:25:17 -0500 Subject: [PATCH 325/644] Removing blank pages in pdf --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index c92efaa9d..9c7caf46a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -163,6 +163,9 @@ def __getattr__(cls, name): # -- Options for LaTeX output --------------------------------------------- latex_elements = { + # Sonny, Lenny, or Bjornstrup + 'fncychap': '\\usepackage[Lenny]{fncychap}', + 'extraclassoptions': 'openany', # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', From cabaeb1abbfc2f22d0820712997ca9345bbae538 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 13:56:42 -0500 Subject: [PATCH 326/644] Avoiding linewraps in pdf --- docs/conf.py | 10 ++++++ docs/tutorials/local_sine_tutorial.rst | 46 +++++++++++++------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9c7caf46a..3440c1048 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,6 +68,12 @@ def __getattr__(cls, name): #breathe_projects_source = {"libEnsemble" : ( "../code/src/", ["test.cpp","test2.cpp"] )} autodoc_mock_imports = ["balsam"] +extlinks = {'duref': ('http://docutils.sourceforge.net/docs/ref/rst/' + 'restructuredtext.html#%s', ''), + 'durole': ('http://docutils.sourceforge.net/docs/ref/rst/' + 'roles.html#%s', ''), + 'dudir': ('http://docutils.sourceforge.net/docs/ref/rst/' + 'directives.html#%s', '')} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -166,6 +172,10 @@ def __getattr__(cls, name): # Sonny, Lenny, or Bjornstrup 'fncychap': '\\usepackage[Lenny]{fncychap}', 'extraclassoptions': 'openany', + 'preamble': + r''' + + ''', # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 0e23fb40a..54c8fccf5 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -182,16 +182,16 @@ inputs and outputs from those functions to expect. .. code-block:: python :linenos: - gen_specs = {'gen_f': gen_random_sample, # Our generator function - 'in': ['sim_id'], # Input field names. 'sim_id' necessary default - 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'lower': np.array([-3]), # lower boundary for random sampling. - 'upper': np.array([3]), # upper boundary for random sampling. - 'gen_batch_size': 5} # number of values gen_f will generate per call - - sim_specs = {'sim_f': sim_find_sine, # Our simulator function - 'in': ['x'], # Input field names. 'x' from gen_f output - 'out': [('y', float)]} # sim_f output. 'y' = sine('x') + gen_specs = {'gen_f': gen_random_sample, # Our generator function + 'in': ['sim_id'], # Input field names. 'sim_id' necessary default + 'out': [('x', float, (1,))], # gen_f output (name, type, size) + 'lower': np.array([-3]), # lower boundary for random sampling + 'upper': np.array([3]), # upper boundary for random sampling + 'gen_batch_size': 5} # number of x's gen_f generates per call + + sim_specs = {'sim_f': sim_find_sine, # Our simulator function + 'in': ['x'], # Input field names. 'x' from gen_f output + 'out': [('y', float)]} # sim_f output. 'y' = sine('x') Recall that each worker is assigned an entry in the :ref:`persis_info` @@ -204,12 +204,12 @@ circumstances where libEnsemble should stop execution in :ref:`exit_criteria` function call. This :ref:`H` is the final version of the History array. @@ -221,7 +221,7 @@ This :ref:`H` is the final version of the History arra H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) - print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array + print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array print(H) That's it! Now that these files are complete, we can run our simulation. @@ -237,11 +237,11 @@ be rearranged. .. code-block:: ['y', 'given_time', 'gen_worker', 'sim_worker', 'given', 'returned', 'x', 'allocated', 'sim_id', 'gen_time'] - [(-0.37466051, 1.55968252e+09, 2, 2, True, True, [-0.38403059], True, 0, 1.55968252e+09) - (-0.29279634, 1.55968252e+09, 2, 3, True, True, [-2.84444261], True, 1, 1.55968252e+09) - ( 0.29358492, 1.55968252e+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.55968252e+09) - (-0.3783986 , 1.55968252e+09, 2, 1, True, True, [-0.38806564], True, 3, 1.55968252e+09) - (-0.45982062, 1.55968252e+09, 2, 2, True, True, [-0.47779319], True, 4, 1.55968252e+09) + [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) + (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) + ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) + (-0.3783986 , 1.559+09, 2, 1, True, True, [-0.38806564], True, 3, 1.559+09) + (-0.45982062, 1.559+09, 2, 2, True, True, [-0.47779319], True, 4, 1.559+09) ... In this arrangement, our output values are listed on the far-left with the generated @@ -320,11 +320,11 @@ of the calling script as follows: from simulator import sim_find_sine from mpi4py import MPI - # nworkers = 4 # nworkers will come from MPI - libE_specs = {'comms': 'mpi'} # 'nworkers' removed, 'comms' now 'mpi' + # nworkers = 4 # nworkers will come from MPI + libE_specs = {'comms': 'mpi'} # 'nworkers' removed, 'comms' now 'mpi' nworkers = MPI.COMM_WORLD.Get_size() - 1 - is_master = (MPI.COMM_WORLD.Get_rank() == 0) # master process has MPI rank 0 + is_master = (MPI.COMM_WORLD.Get_rank() == 0) # master process has MPI rank 0 So that only one process executes the graphing and printing portion of our code, modify the bottom of the calling script like this: @@ -337,8 +337,8 @@ modify the bottom of the calling script like this: libE_specs=libE_specs) if is_master: - - print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array + # Some (optional) statements to visualize our History array + print([i for i in H.dtype.fields]) print(H) import matplotlib.pyplot as plt From 47713d7ba991c78f439a4929f40daa94fa5342c1 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 14:07:07 -0500 Subject: [PATCH 327/644] Editing comments for clarity/space --- docs/data_structures/work_dict.rst | 17 ++++++----------- libensemble/libE_fields.py | 12 ++++++------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 4c1310b34..4dd50482c 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -11,19 +11,14 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` Required keys : 'persis_info' [dict]: Any persistent info to be sent to worker 'i' - 'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' + 'tag' [int]: 'EVAL_SIM_TAG'/'EVAL_GEN_TAG') if worker 'i' is to call sim/gen_func + 'libE_info' [dict]: Info sent to/from worker to help manager update the 'H' - 'tag' [int]: 'EVAL_SIM_TAG' (resp. 'EVAL_GEN_TAG') if worker 'i' is to call sim_func (resp. gen_func) - - 'libE_info' [dict]: This information is sent to and returned from the worker to help libEnsemble quickly update the 'H' and 'W'. - Available keys are: - - H_rows' [list of ints]: History rows to send to worker 'i' - - blocking' [list of ints]: Workers to be blocked by the calculation given to worker 'i' - - persistent' [bool]: True if worker 'i' will enter persistent mode + Available keys are: + 'H_rows' [list of ints]: History rows to send to worker 'i' + 'blocking' [list of ints]: Workers to be blocked by this calculation + 'persistent' [bool]: True if worker 'i' will enter persistent mode :Examples: diff --git a/libensemble/libE_fields.py b/libensemble/libE_fields.py index fc74f8346..00176bcce 100644 --- a/libensemble/libE_fields.py +++ b/libensemble/libE_fields.py @@ -1,11 +1,11 @@ """ Below are the fields used within libEnsemble """ -libE_fields = [('sim_id', int), # The number of the entry in H that was generated - ('given', bool), # True if the entry has been given to a worker for a sim eval - ('given_time', float), # Time (since epoch) that the entry was given to a worker for a sim eval - ('sim_worker', int), # Worker that did (or is doing) the sim eval +libE_fields = [('sim_id', int), # Unique id of entry in H that was generated ('gen_worker', int), # Worker that generated the entry - ('gen_time', float), # Time (since epoch) that entry was entered into H - ('returned', bool), # True if the entry has been returned from the sim eval + ('gen_time', float), # Time (since epoch) entry was entered into H + ('given', bool), # True if entry has been given for sim eval + ('returned', bool), # True if entry has been returned from sim eval + ('given_time', float), # Time (since epoch) that the entry was given + ('sim_worker', int), # Worker that did (or is doing) the sim eval ] From 92bd3e0b29c2acceb6e2433cc3f403dfa7ce0af5 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 14:09:47 -0500 Subject: [PATCH 328/644] Seeing if i can control this image --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index b9a6527ae..60ef78d63 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,6 @@ .. image:: docs/images/libE_logo.png + :scale: 90 + :align: center :alt: libEnsemble | From 5b8fdf7b13c4ff52d0a1f16da395695b7b5de18e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 14:10:18 -0500 Subject: [PATCH 329/644] Seeing if i can control this image --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 60ef78d63..fc29af6e0 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ .. image:: docs/images/libE_logo.png - :scale: 90 + :scale: 10 :align: center :alt: libEnsemble From 29763f5063cd8fa1965200ea46efaecd0fdf1b74 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 14:12:01 -0500 Subject: [PATCH 330/644] Doesn't seem to work --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index fc29af6e0..8618d9eaa 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ .. image:: docs/images/libE_logo.png - :scale: 10 :align: center :alt: libEnsemble From a47d1dc96c00f588b5fb8f6255b14a24c822966f Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 4 Oct 2019 14:28:27 -0500 Subject: [PATCH 331/644] flake8 --- libensemble/libE_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/libE_fields.py b/libensemble/libE_fields.py index 00176bcce..32f23eba0 100644 --- a/libensemble/libE_fields.py +++ b/libensemble/libE_fields.py @@ -6,6 +6,6 @@ ('gen_time', float), # Time (since epoch) entry was entered into H ('given', bool), # True if entry has been given for sim eval ('returned', bool), # True if entry has been returned from sim eval - ('given_time', float), # Time (since epoch) that the entry was given + ('given_time', float), # Time (since epoch) that the entry was given ('sim_worker', int), # Worker that did (or is doing) the sim eval ] From e5c4add0d8e0b0aaf1f9414db59c36cde2b88937 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 4 Oct 2019 20:28:10 -0500 Subject: [PATCH 332/644] Making pdf crossrefs texttt --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 3440c1048..f742f4460 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -174,7 +174,7 @@ def __getattr__(cls, name): 'extraclassoptions': 'openany', 'preamble': r''' - + \protected\def\sphinxcrossref#1{\texttt{#1}} ''', # The paper size ('letterpaper' or 'a4paper'). # From 29eabfe45e8242021b3d9f04195a60f024d77fbe Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 8 Oct 2019 11:15:09 -0500 Subject: [PATCH 333/644] Changing default alloc func to only give gen_specs['in'] if it's a field with length --- libensemble/alloc_funcs/give_sim_work_first.py | 5 ++++- .../tests/regression_tests/test_1d_uniform_sampling.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index f52118972..6b2f9d78b 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -78,6 +78,9 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # Give gen work gen_count += 1 - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + if 'in' in gen_specs and len(gen_specs['in']): + gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + else: + gen_work(Work, i, [], [], persis_info[i]) return Work, persis_info diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling.py index 131d9cd5a..2ce48ab30 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling.py @@ -26,7 +26,6 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} gen_specs = {'gen_f': gen_f, - 'in': ['sim_id'], 'out': [('x', float, (1,))], 'lb': np.array([-3]), 'ub': np.array([3]), From 1d54948abb1b960a44f27ee0802038196277fe55 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 8 Oct 2019 11:35:28 -0500 Subject: [PATCH 334/644] removes gen_specs['in'] --- examples/tutorials/tutorial_calling.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 83ae83efb..8db2e8609 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -8,7 +8,6 @@ libE_specs = {'nprocesses': nworkers, 'comms': 'local'} gen_specs = {'gen_f': gen_random_sample, # Our generator function - 'in': ['sim_id'], # Input field names. 'sim_id' necessary default 'out': [('x', float, (1,))], # gen_f output (name, type, size). 'lower': np.array([-3]), # lower boundary for random sampling. 'upper': np.array([3]), # upper boundary for random sampling. From b062156b071a02fcf40e78f2059408423483159c Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 8 Oct 2019 11:36:36 -0500 Subject: [PATCH 335/644] also removes from mpi version --- examples/tutorials/tutorial_calling_mpi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 84a00fe31..2ba751353 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -12,7 +12,6 @@ is_master = (MPI.COMM_WORLD.Get_rank() == 0) # master process has MPI rank 0 gen_specs = {'gen_f': gen_random_sample, # Our generator function - 'in': ['sim_id'], # Input field names. 'sim_id' necessary default 'out': [('x', float, (1,))], # gen_f output (name, type, size). 'lower': np.array([-3]), # lower boundary for random sampling. 'upper': np.array([3]), # upper boundary for random sampling. From c27d72b5a07eba93aa322599051413e8e422e2b6 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 8 Oct 2019 11:44:00 -0500 Subject: [PATCH 336/644] update tutorial, touch-up formatting --- docs/tutorials/local_sine_tutorial.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 1f0b46d62..f1739fe16 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -74,7 +74,7 @@ An available libEnsemble worker will call this generator function with the follo Later on, we'll populate ``gen_specs`` and ``persis_info`` in our calling script. -For now, create a new Python file named 'generator.py'. Write the following: +For now, create a new Python file named ``generator.py``. Write the following: .. code-block:: python :linenos: @@ -117,7 +117,7 @@ functions perform calculations based on values from the generator function. The only new parameter here is :ref:`sim_specs`, which serves a similar purpose to ``gen_specs``. -Create a new Python file named 'simulator.py'. Write the following: +Create a new Python file named ``simulator.py``. Write the following: .. code-block:: python :linenos: @@ -148,13 +148,13 @@ Calling Script Now we can write the calling script that configures our generator and simulator functions and calls libEnsemble. -Create an empty Python file named 'calling_script.py'. +Create an empty Python file named ``calling_script.py``. In this file, we'll start by importing NumPy, libEnsemble, and the generator and simulator functions we just created. Next, in a dictionary called :ref:`libE_specs` we'll specify the number of workers and the type of manager/worker communication libEnsemble will -use. Our communication method, 'local', refers to Python's Multiprocessing. +use. Our communication method, ``'local'``, refers to Python's Multiprocessing. .. code-block:: python :linenos: @@ -177,7 +177,6 @@ inputs and outputs from those functions to expect. :linenos: gen_specs = {'gen_f': gen_random_sample, # Our generator function - 'in': ['sim_id'], # Input field names. 'sim_id' necessary default 'out': [('x', float, (1,))], # gen_f output (name, type, size). 'lower': np.array([-3]), # lower boundary for random sampling. 'upper': np.array([3]), # upper boundary for random sampling. @@ -242,8 +241,8 @@ In this arrangement, our output values are listed on the far-left with the gener values being the fourth column from the right. Two additional log files should also have been created. -'ensemble.log' contains debugging or informational logging output from libEnsemble, -while 'libE_stats.txt' contains a quick summary of all calculations performed. +``ensemble.log`` contains debugging or informational logging output from libEnsemble, +while ``libE_stats.txt`` contains a quick summary of all calculations performed. I graphed my output using Matplotlib, coloring entries by which worker performed the simulation: From c198ac49d296fb95d94f7f11f31342334ce9ab56 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 11 Oct 2019 14:38:58 -0500 Subject: [PATCH 337/644] Trying to split pdf and html orders --- docs/index.rst | 71 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 6b81824f4..8a624bfca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,42 +36,63 @@ libEnsemble is a library for managing ensemble-like collections of computations. * Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. +.. only:: (latex or latexpdf) -.. toctree:: - :maxdepth: 2 - :caption: Getting Started: + .. toctree:: + :maxdepth: 2 - Quickstart - contributing - release_notes - FAQ + Quickstart + user_guide + tutorials/local_sine_tutorial + contributing + FAQ + libE_module + data_structures/data_structures + user_funcs + job_controller/jc_index + logging + dev_guide/release_management/release_index.rst + dev_guide/dev_API/developer_API.rst + release_notes -.. toctree:: - :maxdepth: 1 - :caption: Tutorials: +.. only:: html - tutorials/local_sine_tutorial + .. toctree:: + :maxdepth: 2 + :caption: Getting Started: + Quickstart + contributing + release_notes + FAQ -.. toctree:: - :maxdepth: 2 - :caption: User Guide: - user_guide - libE_module - data_structures/data_structures - user_funcs - job_controller/jc_index - logging + .. toctree:: + :maxdepth: 1 + :caption: Tutorials: + tutorials/local_sine_tutorial -.. toctree:: - :maxdepth: 2 - :caption: Developer Guide: - dev_guide/release_management/release_index.rst - dev_guide/dev_API/developer_API.rst + .. toctree:: + :maxdepth: 2 + :caption: User Guide: + + user_guide + libE_module + data_structures/data_structures + user_funcs + job_controller/jc_index + logging + + + .. toctree:: + :maxdepth: 2 + :caption: Developer Guide: + + dev_guide/release_management/release_index.rst + dev_guide/dev_API/developer_API.rst From 5fc3eb77c93f4869808f9a4e87f54c08967de60b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 11 Oct 2019 16:03:06 -0500 Subject: [PATCH 338/644] Adding code and test for LHS sampling --- libensemble/gen_funcs/uniform_sampling.py | 42 +++++++++++++++++++ libensemble/tests/regression_tests/.gitignore | 5 +++ ...niform_sampling.py => test_1d_sampling.py} | 10 ++--- .../test_chwirut_pounders_persistent.py | 11 ++++- 4 files changed, 61 insertions(+), 7 deletions(-) rename libensemble/tests/regression_tests/{test_1d_uniform_sampling.py => test_1d_sampling.py} (79%) diff --git a/libensemble/gen_funcs/uniform_sampling.py b/libensemble/gen_funcs/uniform_sampling.py index 162fa276f..30d521f11 100644 --- a/libensemble/gen_funcs/uniform_sampling.py +++ b/libensemble/gen_funcs/uniform_sampling.py @@ -88,3 +88,45 @@ def uniform_random_sample(H, persis_info, gen_specs, _): O['x'] = persis_info['rand_stream'].uniform(lb, ub, (b, n)) return O, persis_info + + +def latin_hypercube_sample(H, persis_info, gen_specs, _): + """ + Generates ``gen_specs['gen_batch_size']`` in a Latin hypercube sample over + the domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. + """ + + ub = gen_specs['ub'] + lb = gen_specs['lb'] + + n = len(lb) + b = gen_specs['gen_batch_size'] + + O = np.zeros(b, dtype=gen_specs['out']) + + A = lhs_sample(n, b) + + O['x'] = A*(ub-lb)+lb + + return O, persis_info + + +def lhs_sample(n, k): + + # Generate the intervals and random values + intervals = np.linspace(0, 1, k + 1) + rand_source = np.random.uniform(0,1,(k, n)) + rand_pts = np.zeros((k, n)) + sample = np.zeros((k, n)) + + # Add a point uniformly in each interval + a = intervals[:k] + b = intervals[1:] + for j in range(n): + rand_pts[:, j] = rand_source[:, j]*(b-a) + a + + # Randomly perturb + for j in range(n): + sample[:, j] = rand_pts[np.random.permutation(k), j] + + return sample diff --git a/libensemble/tests/regression_tests/.gitignore b/libensemble/tests/regression_tests/.gitignore index ea1472ec1..ef2668a41 100644 --- a/libensemble/tests/regression_tests/.gitignore +++ b/libensemble/tests/regression_tests/.gitignore @@ -1 +1,6 @@ output/ +*.npy +*.pickle +ensemble.log +libE_stats.txt + diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py similarity index 79% rename from libensemble/tests/regression_tests/test_1d_uniform_sampling.py rename to libensemble/tests/regression_tests/test_1d_sampling.py index 2ce48ab30..438cf7993 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -1,10 +1,10 @@ # """ -# Runs libEnsemble with random sampling on a simple 1D problem +# Runs libEnsemble with Latin hypercube sampling on a simple 1D problem # # Execute via one of the following commands (e.g. 3 workers): -# mpiexec -np 4 python3 test_1d_uniform_sampling.py -# python3 test_1d_uniform_sampling.py --nworkers 3 --comms local -# python3 test_1d_uniform_sampling.py --nworkers 3 --comms tcp +# mpiexec -np 4 python3 test_1d_sampling.py +# python3 test_1d_sampling.py --nworkers 3 --comms local +# python3 test_1d_sampling.py --nworkers 3 --comms tcp # # The number of concurrent evaluations of the objective function will be 4-1=3. # """ @@ -18,7 +18,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.uniform_sampling import latin_hypercube_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index ca5ea9b5f..974e1ecdb 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -18,9 +18,10 @@ # Import libEnsemble items for this test from libensemble.libE import libE -from math import gamma, pi, sqrt +from math import gamma, pi, sqrt, ceil from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f +from libensemble.gen_funcs.uniform_sampling import lhs_sample from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream @@ -54,7 +55,7 @@ 'gatol': 1e-6, 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, - 'lhs_divisions': 5, + 'lhs_divisions': 50, 'components': m, 'lb': (-2-np.pi/10)*np.ones(n), 'ub': 2*np.ones(n)} @@ -65,6 +66,12 @@ exit_criteria = {'sim_max': 500} +sample_points = np.zeros((0,n)) +for i in range(ceil(exit_criteria['sim_max']/gen_specs['lhs_divisions'])): + sample_points = np.append(sample_points,lhs_sample(n,gen_specs['lhs_divisions']),axis=0) + +gen_specs['sample_points'] = sample_points*(gen_specs['ub']-gen_specs['lb']) + gen_specs['lb'] + # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) From 381efcfafa3c8e28ef83806296888f06b72ebf96 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 11 Oct 2019 16:07:46 -0500 Subject: [PATCH 339/644] Renaming uniform_sampling to sampling as it can contain more methods than just uniform --- libensemble/gen_funcs/{uniform_sampling.py => sampling.py} | 0 libensemble/tests/regression_tests/script_test_balsam_hworld.py | 2 +- libensemble/tests/regression_tests/test_1d_sampling.py | 2 +- .../regression_tests/test_1d_uniform_sampling_with_comm_dup.py | 2 +- .../regression_tests/test_6-hump_camel_elapsed_time_abort.py | 2 +- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 2 +- .../test_6-hump_camel_with_different_nodes_uniform_sample.py | 2 +- libensemble/tests/regression_tests/test_calc_exception.py | 2 +- .../tests/regression_tests/test_chwirut_pounders_persistent.py | 2 +- .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 2 +- libensemble/tests/regression_tests/test_comms.py | 2 +- libensemble/tests/regression_tests/test_fast_alloc.py | 2 +- libensemble/tests/regression_tests/test_jobcontroller_hworld.py | 2 +- libensemble/tests/regression_tests/test_worker_exceptions.py | 2 +- libensemble/tests/scaling_tests/forces/run_libe_forces.py | 2 +- 15 files changed, 14 insertions(+), 14 deletions(-) rename libensemble/gen_funcs/{uniform_sampling.py => sampling.py} (100%) diff --git a/libensemble/gen_funcs/uniform_sampling.py b/libensemble/gen_funcs/sampling.py similarity index 100% rename from libensemble/gen_funcs/uniform_sampling.py rename to libensemble/gen_funcs/sampling.py diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index ee6cc5609..4260b98e5 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -10,7 +10,7 @@ from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import per_worker_stream mpi4py.rc.recv_mprobe = False # Disable matching probes diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 438cf7993..5e7990f02 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -18,7 +18,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f -from libensemble.gen_funcs.uniform_sampling import latin_hypercube_sample as gen_f +from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index 9be7ba91f..089b2d44c 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -22,7 +22,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index 3dca719df..1e3cb393b 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -19,7 +19,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream, eprint nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index ce5603e74..53121d511 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -19,7 +19,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index 72cc2d065..ac319ea8a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -20,7 +20,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel_with_different_ranks_and_nodes as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index 74ba6c575..604779a63 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -17,7 +17,7 @@ from libensemble.libE import libE from libensemble.libE_manager import ManagerException -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 974e1ecdb..7443d5bf9 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -21,7 +21,7 @@ from math import gamma, pi, sqrt, ceil from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f -from libensemble.gen_funcs.uniform_sampling import lhs_sample +from libensemble.gen_funcs.sampling import lhs_sample from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 17f561389..7cdd9548c 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -24,7 +24,7 @@ # Import libEnsemble items from libensemble.libE import libE from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample_obj_components as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample_obj_components as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.support import persis_info_3 as persis_info from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index 41404fc8c..d719fc6fe 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -19,7 +19,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.comms_testing import float_x1000 as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.mpi_controller import MPIJobController # Only used to get workerID in float_x1000 jobctrl = MPIJobController(auto_resources=False) diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 67a4f7742..1741232c7 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -19,7 +19,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.common import parse_args, per_worker_stream diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 2dca7fbf2..14237f09b 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -20,7 +20,7 @@ from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tests.regression_tests.common import build_simfunc, parse_args, per_worker_stream # Do not change these lines - they are parsed by run-tests.sh diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 1229ff228..4ba4ab3be 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -17,7 +17,7 @@ from libensemble.libE import libE from libensemble.tests.regression_tests.support import nan_func as sim_f -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample as gen_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE_manager import ManagerException from libensemble.tests.regression_tests.common import parse_args, per_worker_stream diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index fdf30dd71..f19c8506e 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -6,7 +6,7 @@ # Import libEnsemble modules from libensemble.libE import libE -from libensemble.gen_funcs.uniform_sampling import uniform_random_sample +from libensemble.gen_funcs.sampling import uniform_random_sample from libensemble import libE_logger libE_logger.set_level('INFO') From a245c14a7156e4d9fd2f15fb2651ac031dd3c45c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 11 Oct 2019 16:23:26 -0500 Subject: [PATCH 340/644] flayk ate --- libensemble/gen_funcs/sampling.py | 10 +++++----- .../test_chwirut_pounders_persistent.py | 4 ++-- .../tests/standalone_tests/kill_test/log.autotest.txt | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index 30d521f11..74431fc94 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -114,19 +114,19 @@ def latin_hypercube_sample(H, persis_info, gen_specs, _): def lhs_sample(n, k): # Generate the intervals and random values - intervals = np.linspace(0, 1, k + 1) - rand_source = np.random.uniform(0,1,(k, n)) + intervals = np.linspace(0, 1, k+1) + rand_source = np.random.uniform(0, 1, (k, n)) rand_pts = np.zeros((k, n)) sample = np.zeros((k, n)) - + # Add a point uniformly in each interval a = intervals[:k] b = intervals[1:] for j in range(n): rand_pts[:, j] = rand_source[:, j]*(b-a) + a - + # Randomly perturb for j in range(n): sample[:, j] = rand_pts[np.random.permutation(k), j] - + return sample diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 7443d5bf9..f1fbcb913 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -66,9 +66,9 @@ exit_criteria = {'sim_max': 500} -sample_points = np.zeros((0,n)) +sample_points = np.zeros((0, n)) for i in range(ceil(exit_criteria['sim_max']/gen_specs['lhs_divisions'])): - sample_points = np.append(sample_points,lhs_sample(n,gen_specs['lhs_divisions']),axis=0) + sample_points = np.append(sample_points, lhs_sample(n, gen_specs['lhs_divisions']), axis=0) gen_specs['sample_points'] = sample_points*(gen_specs['ub']-gen_specs['lb']) + gen_specs['lb'] diff --git a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt index 1f0682ad8..baeb7d0ab 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt @@ -55,7 +55,7 @@ Bebop (intelmpi):: kill 1: Fails kill 2: Works - + [*Update 2019: kill 1 seems to also work if use srun instead of mpirun] Cooley (intelmpi):: @@ -94,14 +94,14 @@ Theta (intelmpi):: Alternative - may be to use apkill. This may work without unloading xalt - not tried. - + Cori (intelmpi - launched with srun):: - Single node on 4 processes: - + Single node on 4 processes: + kill 1: Works kill 2: Works - + Two nodes with 2 processes each: kill 1: Works From c7aff91caf02f3b62724848fc226db57a45dc151 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 14 Oct 2019 12:01:06 -0500 Subject: [PATCH 341/644] update symlink --- examples/gen_funcs/sampling.py | 1 + examples/gen_funcs/uniform_sampling.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 examples/gen_funcs/sampling.py delete mode 120000 examples/gen_funcs/uniform_sampling.py diff --git a/examples/gen_funcs/sampling.py b/examples/gen_funcs/sampling.py new file mode 120000 index 000000000..7733bb711 --- /dev/null +++ b/examples/gen_funcs/sampling.py @@ -0,0 +1 @@ +../../libensemble/gen_funcs/sampling.py \ No newline at end of file diff --git a/examples/gen_funcs/uniform_sampling.py b/examples/gen_funcs/uniform_sampling.py deleted file mode 120000 index 29ae9b98c..000000000 --- a/examples/gen_funcs/uniform_sampling.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/gen_funcs/uniform_sampling.py \ No newline at end of file From 223cfa8e24f161511f3b3a0007b43e644a74adc4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 15 Oct 2019 14:33:38 -0500 Subject: [PATCH 342/644] Removing whitespace --- docs/data_structures/work_dict.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 4dd50482c..852156dd8 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -17,7 +17,7 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` Available keys are: 'H_rows' [list of ints]: History rows to send to worker 'i' - 'blocking' [list of ints]: Workers to be blocked by this calculation + 'blocking' [list of ints]: Workers to be blocked by this calculation 'persistent' [bool]: True if worker 'i' will enter persistent mode diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 54c8fccf5..0df9bf50e 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -188,7 +188,7 @@ inputs and outputs from those functions to expect. 'lower': np.array([-3]), # lower boundary for random sampling 'upper': np.array([3]), # upper boundary for random sampling 'gen_batch_size': 5} # number of x's gen_f generates per call - + sim_specs = {'sim_f': sim_find_sine, # Our simulator function 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') @@ -338,7 +338,7 @@ modify the bottom of the calling script like this: if is_master: # Some (optional) statements to visualize our History array - print([i for i in H.dtype.fields]) + print([i for i in H.dtype.fields]) print(H) import matplotlib.pyplot as plt From 7ce771743c0eb155e823bfed81c24930fd95c4b2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 15 Oct 2019 16:40:25 -0500 Subject: [PATCH 343/644] Better rst tags and avoiding line numbers in includes --- README.rst | 2 +- docs/data_structures/history_array.rst | 2 +- docs/quickstart.rst | 2 +- docs/sim_gen_alloc_funcs.rst | 4 ++-- docs/tutorials/local_sine_tutorial.rst | 3 +-- libensemble/message_numbers.py | 4 ++-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 8618d9eaa..f4a4932b7 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,7 @@ :alt: Documentation Status | -.. after_badges_tag +.. after_badges_rst_tag ==================== What is libEnsemble? diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index a097cd57b..017b7430d 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -17,7 +17,7 @@ Fields in ``H`` include those specified in ``sim_specs['out']``, Below are the protected fields used in ``H`` .. literalinclude:: ../../libensemble/libE_fields.py - :lines: 4- + :start-at: libE_fields :Examples: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8d1e85bbf..240804990 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -20,4 +20,4 @@ | .. include:: ../README.rst - :start-after: after_badges_tag + :start-after: after_badges_rst_tag diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index 7e56753ba..cfcaeba66 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -40,7 +40,7 @@ Returns: Used to tell manager why a persistent worker is stopping. .. literalinclude:: ../libensemble/message_numbers.py - :lines: 1-8 + :end-before: last_message_number_rst_tag .. _api_gen_f: @@ -82,7 +82,7 @@ Returns: Used to tell manager why a persistent worker is stopping. .. literalinclude:: ../libensemble/message_numbers.py - :lines: 1-8 + :end-before: last_message_number_rst_tag .. _api_alloc_f: diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 0df9bf50e..244906085 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -183,7 +183,6 @@ inputs and outputs from those functions to expect. :linenos: gen_specs = {'gen_f': gen_random_sample, # Our generator function - 'in': ['sim_id'], # Input field names. 'sim_id' necessary default 'out': [('x', float, (1,))], # gen_f output (name, type, size) 'lower': np.array([-3]), # lower boundary for random sampling 'upper': np.array([3]), # upper boundary for random sampling @@ -221,7 +220,7 @@ This :ref:`H` is the final version of the History arra H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) - print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array + print([i for i in H.dtype.fields]) # (optional) to visualize our History array print(H) That's it! Now that these files are complete, we can run our simulation. diff --git a/libensemble/message_numbers.py b/libensemble/message_numbers.py index fc664e822..d868fb9d4 100644 --- a/libensemble/message_numbers.py +++ b/libensemble/message_numbers.py @@ -1,13 +1,13 @@ # --- Tags -UNSET_TAG = 0 # sh temp - this is a libe feature that is to be reviewed for best solution +UNSET_TAG = 0 # sh temp - this is a libE feature that is to be reviewed for best solution EVAL_SIM_TAG = 1 EVAL_GEN_TAG = 2 STOP_TAG = 3 PERSIS_STOP = 4 # manager tells persistent worker to desist FINISHED_PERSISTENT_SIM_TAG = 11 # tells manager sim_f done persistent mode FINISHED_PERSISTENT_GEN_TAG = 12 # tells manager gen_f done persistent mode -# ABORT_ENSEMBLE = 13 # Worker asks manager to abort (and dump history) +# last_message_number_rst_tag calc_type_strings = { EVAL_SIM_TAG: 'sim', From 0b5310a63dcc1bcb538e3a51e043504712580698 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 15 Oct 2019 16:43:01 -0500 Subject: [PATCH 344/644] Whitespace removal --- README.rst | 2 +- .../tests/standalone_tests/kill_test/log.autotest.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index f4a4932b7..36766cc9f 100644 --- a/README.rst +++ b/README.rst @@ -186,7 +186,7 @@ When specifying these options via command line options, one may use the ``libensemble/tests/regression_tests/common.py`` -See the +See the `user-guide `_ for more information. diff --git a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt index 1f0682ad8..baeb7d0ab 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt @@ -55,7 +55,7 @@ Bebop (intelmpi):: kill 1: Fails kill 2: Works - + [*Update 2019: kill 1 seems to also work if use srun instead of mpirun] Cooley (intelmpi):: @@ -94,14 +94,14 @@ Theta (intelmpi):: Alternative - may be to use apkill. This may work without unloading xalt - not tried. - + Cori (intelmpi - launched with srun):: - Single node on 4 processes: - + Single node on 4 processes: + kill 1: Works kill 2: Works - + Two nodes with 2 processes each: kill 1: Works From 7425ba5e052fd61b9c25514685bcba5899587344 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 08:33:50 -0500 Subject: [PATCH 345/644] Removing copy/paste text and editing source comments --- docs/data_structures/sim_specs.rst | 33 ++++++------------- .../test_6-hump_camel_uniform_sampling.py | 10 +++--- .../scaling_tests/forces/run_libe_forces.py | 29 ++++++++-------- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index 63a6396c1..c0e9c7077 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -44,35 +44,22 @@ Simulation function specifications to be set in user calling script and passed t .. _sim-specs-exmple1: -From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py``:: +From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + +.. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py + :start-at: sim_specs + :end-before: end_sim_specs_rst_tag - sim_specs = {'sim_f': six_hump_camel, # This is the function whose output is being minimized - 'in': ['x'], # These keys will be given to the above function - 'out': [('f',float)], # This is the output from the function being minimized - 'save_every_k': 400 - } Note that the dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding generator ``'out'`` field ``'x'`` (see :ref:`gen_specs example`). Only the variable name is then required in sim_specs. -From: ``libensemble/tests/scaling_tests/forces/run_libe_forces.py``:: - - sim_specs = {'sim_f': run_forces, # This is the function whose output is being minimized (sim func) - 'in': ['x'], # Name of input data structure for sim func - 'out': [('energy', float)], # Output from sim func - 'keys': ['seed'], # Key/keys for input data - 'sim_dir': './sim', # Simulation input dir to be copied for each worker (*currently empty) - 'sim_dir_suffix': 'test', # Suffix for copied sim dirs to indentify run (in case multiple) - 'simdir_basename': 'forces', # User attribute to name sim directories (forces_***) - 'cores': 2, # User attribute to set number of cores for sim func runs (optional) - 'sim_particles': 1e3, # User attribute for number of particles in simulations - 'sim_timesteps': 5, # User attribute for number of timesteps in simulations - 'sim_kill_minutes': 10.0, # User attribute for max time for simulations - 'kill_rate': 0.5, # Between 0 and 1 for proportion of jobs that go bad (for testing kills) - 'particle_variance': 0.2, # Range over which particle count varies (for testing load imbalance) - 'profile': False - } +From: ``libensemble/tests/scaling_tests/forces/run_libe_forces.py`` + +.. literalinclude:: ../../libensemble/tests/scaling_tests/forces/run_libe_forces.py + :start-at: sim_f + :end-before: end_sim_specs_rst_tag This example uses a number of user specific fields, that will be dealt with in the corresponding sim f, which can be found at ``libensemble/tests/scaling_tests/forces/forces_simf.py`` diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index ce5603e74..5c062698a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -25,10 +25,12 @@ nworkers, is_master, libE_specs, _ = parse_args() -sim_specs = {'sim_f': sim_f, - 'in': ['x'], - 'out': [('f', float)], - 'save_every_k': 400} +sim_specs = {'sim_f': sim_f, # Function whose output is being minimized + 'in': ['x'], # Keys to be given to the above function + 'out': [('f', float)], # Output from the function being minimized + 'save_every_k': 400, # Want progress saved every 400 evals + } +# end_sim_specs_rst_tag gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index fdf30dd71..cd3e68559 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -66,21 +66,22 @@ # Note: Attributes such as kill_rate are to control forces tests, this would not be a typical parameter. # State the objective function, its arguments, output, and necessary parameters (and their sizes) -sim_specs = {'sim_f': run_forces, # This is the function whose output is being minimized (sim func) - 'in': ['x'], # Name of input data structure for sim func - 'out': [('energy', float)], # Output from sim func - 'keys': ['seed'], # Key/keys for input data - 'sim_dir': './sim', # Simulation input dir to be copied for each worker (*currently empty) - 'sim_dir_suffix': 'test', # Suffix for copied sim dirs to indentify run (in case multiple) - 'simdir_basename': 'forces', # User attribute to name sim directories (forces_***) - 'cores': 2, # User attribute to set number of cores for sim func runs (optional) - 'sim_particles': 1e3, # User attribute for number of particles in simulations - 'sim_timesteps': 5, # User attribute for number of timesteps in simulations - 'sim_kill_minutes': 10.0, # User attribute for max time for simulations - 'kill_rate': 0.5, # Between 0 and 1 for proportion of jobs that go bad (for testing kills) - 'particle_variance': 0.2, # Range over which particle count varies (for testing load imbalance) - 'profile': False +sim_specs = {'sim_f': run_forces, # Function whose output is being minimized + 'in': ['x'], # Name of input data structure for sim func + 'out': [('energy', float)], # Output from sim func + 'keys': ['seed'], # Key/keys for input data + 'sim_dir': './sim', # Sim dir to be copied for each worker + 'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) + 'profile': False, # Don't have libE profile run + 'simdir_basename': 'force', # Used by sim_f to name sim directories + 'cores': 2, # Used by sim_f to set number of cores used + 'sim_particles': 1e3, # Used by sim_f for number of particles + 'sim_timesteps': 5, # Used by sim_f for number of timesteps + 'sim_kill_minutes': 10.0, # Used by sim_f to set max run time + 'particle_variance': 0.2, # Used by sim_f to vary load imbalance + 'kill_rate': 0.5, # Fraction of bad sim_f evals (tests kills) } +# end_sim_specs_rst_tag # State the generating function, its arguments, output, and necessary parameters. gen_specs = {'gen_f': uniform_random_sample, # Generator function From fbc1d5ff5fd2fc08a8f6161c1fc01e473678aa3c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 08:47:03 -0500 Subject: [PATCH 346/644] Removing copy/paste and editing source comments --- docs/data_structures/persis_info.rst | 9 ++++----- libensemble/tests/regression_tests/support.py | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 3f1a6e6ce..a051e95d2 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -17,10 +17,9 @@ from the allocation function. :Examples: -From: libEnsemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MAA.py:: +From: ``libEnsemble/tests/regression_tests/support.py`` - persis_info = {'next_to_give':0} # used in alloc_funcs/fast_alloc_to_aposmm.py to store the next entry in H to give - persis_info['total_gen_calls'] = 0 # used in alloc_funcs/fast_alloc_to_aposmm.py to count total gen calls +.. literalinclude:: ../../libensemble/tests/regression_tests/support.py + :start-at: persis_info_1 + :end-before: end_persis_info_rst_tag - for i in range(MPI.COMM_WORLD.Get_size()): - persis_info[i] = {'rand_stream': np.random.RandomState(i)} # used as a random number stream for each worker diff --git a/libensemble/tests/regression_tests/support.py b/libensemble/tests/regression_tests/support.py index 4cfc78500..23cf171bf 100644 --- a/libensemble/tests/regression_tests/support.py +++ b/libensemble/tests/regression_tests/support.py @@ -35,15 +35,16 @@ def nan_func(calc_in, persis_info, sim_specs, libE_info): ('pt_id', int)] # Identify the same point evaluated by different sim_f's or components # give_sim_work_first persis_info -persis_info_1 = {} -# Below persis_info fields store APOSMM information, but can be passed to various workers. -persis_info_1['total_gen_calls'] = 0 -persis_info_1['last_worker'] = 0 -persis_info_1['next_to_give'] = 0 -persis_info_1[0] = {'run_order': {}, - 'old_runs': {}, - 'total_runs': 0, - 'rand_stream': np.random.RandomState(1)} +persis_info_1 = {'total_gen_calls': 0, # Counts gen calls in alloc_f + 'last_worker': 0, # Remembers last gen worker in alloc_f + 'next_to_give': 0, # Remembers next H row to give in alloc_f + } +persis_info_1[0] = {'run_order': {}, # Used by manager to remember run order + 'old_runs': {}, # Used by manager to store old runs order + 'total_runs': 0, # Used by manager to count total runs + 'rand_stream': np.random.RandomState(1) + } +# end_persis_info_rst_tag persis_info_2 = copy.deepcopy(persis_info_1) persis_info_2[1] = persis_info_2[0] From 6f037143c6f19bbd3a65c85b47ba6a078978247e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 08:47:33 -0500 Subject: [PATCH 347/644] Whitespace --- docs/data_structures/persis_info.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index a051e95d2..87ff5ca7b 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -22,4 +22,3 @@ From: ``libEnsemble/tests/regression_tests/support.py`` .. literalinclude:: ../../libensemble/tests/regression_tests/support.py :start-at: persis_info_1 :end-before: end_persis_info_rst_tag - From 3ee5cc9d6a003787ff8b4aa4ef1d6f5f02e41ea3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 08:55:39 -0500 Subject: [PATCH 348/644] Removing copy/paste and editing source comments --- docs/data_structures/gen_specs.rst | 15 +++++---------- .../test_6-hump_camel_uniform_sampling.py | 15 ++++++++------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 2a937f492..a90529afa 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -36,16 +36,11 @@ Generation function specifications to be set in user calling script and passed t .. _gen-specs-exmple1: -From: libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py:: - - gen_specs = {'gen_f': uniform_random_sample, - 'in': ['sim_id'], - 'out': [('x',float,2)], - 'lb': np.array([-3,-2]), - 'ub': np.array([ 3, 2]), - 'gen_batch_size': 500, - 'save_every_k': 300 - } +From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + +.. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py + :start-at: gen_specs + :end-before: end_gen_specs_rst_tag In this example, the generation function *uniform_random_sample* will generate 500 random points uniformly over the 2D domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index 5c062698a..c7d15ff75 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -32,13 +32,14 @@ } # end_sim_specs_rst_tag -gen_specs = {'gen_f': gen_f, - 'in': ['sim_id'], - 'gen_batch_size': 500, - 'save_every_k': 300, - 'out': [('x', float, (2,))], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} +gen_specs = {'gen_f': gen_f, # Tell libE function generating sim_f input + 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size + 'save_every_k': 300, # Tell libE to save every 300 gen entries + 'gen_batch_size': 500, # Tell gen_f how much to generate per call + 'lb': np.array([-3, -2]), # Tell gen_f lower bounds + 'ub': np.array([3, 2]), # Tell gen_f upper bounds + } +# end_gen_specs_rst_tag persis_info = per_worker_stream({}, nworkers + 1) From 829e62b2a7c1252c19f726f387849e40bcbd447a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 08:56:59 -0500 Subject: [PATCH 349/644] Whitespace --- .../test_6-hump_camel_uniform_sampling.py | 8 ++++---- libensemble/tests/scaling_tests/forces/run_libe_forces.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index c7d15ff75..f26bad7b0 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -25,21 +25,21 @@ nworkers, is_master, libE_specs, _ = parse_args() -sim_specs = {'sim_f': sim_f, # Function whose output is being minimized +sim_specs = {'sim_f': sim_f, # Function whose output is being minimized 'in': ['x'], # Keys to be given to the above function 'out': [('f', float)], # Output from the function being minimized 'save_every_k': 400, # Want progress saved every 400 evals - } + } # end_sim_specs_rst_tag gen_specs = {'gen_f': gen_f, # Tell libE function generating sim_f input 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size 'save_every_k': 300, # Tell libE to save every 300 gen entries - 'gen_batch_size': 500, # Tell gen_f how much to generate per call + 'gen_batch_size': 500, # Tell gen_f how much to generate per call 'lb': np.array([-3, -2]), # Tell gen_f lower bounds 'ub': np.array([3, 2]), # Tell gen_f upper bounds } -# end_gen_specs_rst_tag +# end_gen_specs_rst_tag persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index cd3e68559..5f8b4cee6 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -70,13 +70,13 @@ 'in': ['x'], # Name of input data structure for sim func 'out': [('energy', float)], # Output from sim func 'keys': ['seed'], # Key/keys for input data - 'sim_dir': './sim', # Sim dir to be copied for each worker + 'sim_dir': './sim', # Sim dir to be copied for each worker 'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) 'profile': False, # Don't have libE profile run - 'simdir_basename': 'force', # Used by sim_f to name sim directories + 'simdir_basename': 'force', # Used by sim_f to name sim directories 'cores': 2, # Used by sim_f to set number of cores used - 'sim_particles': 1e3, # Used by sim_f for number of particles - 'sim_timesteps': 5, # Used by sim_f for number of timesteps + 'sim_particles': 1e3, # Used by sim_f for number of particles + 'sim_timesteps': 5, # Used by sim_f for number of timesteps 'sim_kill_minutes': 10.0, # Used by sim_f to set max run time 'particle_variance': 0.2, # Used by sim_f to vary load imbalance 'kill_rate': 0.5, # Fraction of bad sim_f evals (tests kills) From 82db49b8ca9c509f0535cda3160f31daf484857c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 13:36:50 -0500 Subject: [PATCH 350/644] Better, but open to discussion --- docs/conf.py | 1 + docs/data_structures/sim_specs.rst | 39 +++++++++---------- docs/data_structures/work_dict.rst | 18 ++++----- .../release_management/release_process.rst | 6 +-- libensemble/message_numbers.py | 2 +- 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f742f4460..2359576ea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -172,6 +172,7 @@ def __getattr__(cls, name): # Sonny, Lenny, or Bjornstrup 'fncychap': '\\usepackage[Lenny]{fncychap}', 'extraclassoptions': 'openany', + 'sphinxsetup': 'admonitioncolor={RGB}{255,204,204}', 'preamble': r''' \protected\def\sphinxcrossref#1{\texttt{#1}} diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index c0e9c7077..02fca0801 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -33,33 +33,32 @@ Simulation function specifications to be set in user calling script and passed t Additional entires in sim_specs will be given to sim_f -:Notes: +.. note:: + * The user may define other fields to be passed to the simulator function. + * The tuples defined in the ``'out'`` list are entered into the master :ref:`history array` + * The ``sim_dir_prefix`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. -* The user may define other fields to be passed to the simulator function. -* The tuples defined in the ``'out'`` list are entered into the master :ref:`history array` -* The ``sim_dir_prefix`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. +.. seealso:: -:Examples: + .. _sim-specs-exmple1: -.. _sim-specs-exmple1: + From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` -From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py + :start-at: sim_specs + :end-before: end_sim_specs_rst_tag -.. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py - :start-at: sim_specs - :end-before: end_sim_specs_rst_tag + The dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding + generator ``'out'`` field ``'x'`` (see :ref:`gen_specs example`). + Only the variable name is then required in ``sim_specs['in']``. -Note that the dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding -generator ``'out'`` field ``'x'`` (see :ref:`gen_specs example`). -Only the variable name is then required in sim_specs. + From: ``libensemble/tests/scaling_tests/forces/run_libe_forces.py`` -From: ``libensemble/tests/scaling_tests/forces/run_libe_forces.py`` + .. literalinclude:: ../../libensemble/tests/scaling_tests/forces/run_libe_forces.py + :start-at: sim_f + :end-before: end_sim_specs_rst_tag -.. literalinclude:: ../../libensemble/tests/scaling_tests/forces/run_libe_forces.py - :start-at: sim_f - :end-before: end_sim_specs_rst_tag - -This example uses a number of user specific fields, that will be dealt with in the corresponding sim f, which -can be found at ``libensemble/tests/scaling_tests/forces/forces_simf.py`` + This example uses a number of user specific fields, that will be dealt with in the corresponding sim f, which + can be found at ``libensemble/tests/scaling_tests/forces/forces_simf.py`` diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 852156dd8..1a75d9743 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -15,20 +15,16 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` 'tag' [int]: 'EVAL_SIM_TAG'/'EVAL_GEN_TAG') if worker 'i' is to call sim/gen_func 'libE_info' [dict]: Info sent to/from worker to help manager update the 'H' - Available keys are: + Optional keys are: 'H_rows' [list of ints]: History rows to send to worker 'i' 'blocking' [list of ints]: Workers to be blocked by this calculation 'persistent' [bool]: True if worker 'i' will enter persistent mode -:Examples: +.. Hint:: + For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. + For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. -.. How to link directly to the file? - -| For allocation functions using persistent workers, see -| ``libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` -| or -| ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` -| -| For allocation functions giving work that blocks other workers, see -| ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` +.. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py +.. _start_persistent_local_opt_gens.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +.. _test_6-hump_camel_with_different_nodes_uniform_sample.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index a26c36b7d..86274e1e9 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -15,11 +15,11 @@ Before release - Release notes for this version are added to the documentation with release date, including a list of supported (tested) platforms. -- Version number is updated wherever it appears (in `setup.py`, `libensemble/__init__.py`, `README.rst` and twice in `docs/conf.py`) +- Version number is updated wherever it appears (in ``setup.py``, ``libensemble/__init__.py``, ``README.rst`` and twice in ``docs/conf.py``) -- Check year is correct in README.rst under `Citing libEnsemble` and in `docs/conf.py`. +- Check year is correct in ``README.rst`` under *Citing libEnsemble* and in ``docs/conf.py``. -- setup.py and `libensemble/__init__.py` are checked to ensure all information is up to date. +- ``setup.py`` and ``libensemble/__init__.py`` are checked to ensure all information is up to date. - Tests are run with source to be released (this may iterate): diff --git a/libensemble/message_numbers.py b/libensemble/message_numbers.py index d868fb9d4..59e747179 100644 --- a/libensemble/message_numbers.py +++ b/libensemble/message_numbers.py @@ -1,6 +1,6 @@ # --- Tags -UNSET_TAG = 0 # sh temp - this is a libE feature that is to be reviewed for best solution +UNSET_TAG = 0 # sh temp - this is a libE feature to be reviewed for best solution EVAL_SIM_TAG = 1 EVAL_GEN_TAG = 2 STOP_TAG = 3 From 669846c9fd23257cc06a0b251d5199e8e9149d1d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 14:19:58 -0500 Subject: [PATCH 351/644] Using correct admonitions --- docs/conf.py | 1 - docs/data_structures/alloc_specs.rst | 10 +++--- docs/data_structures/exit_criteria.rst | 3 ++ docs/data_structures/gen_specs.rst | 36 +++++++++---------- docs/data_structures/history_array.rst | 12 +++---- docs/data_structures/persis_info.rst | 10 +++--- docs/data_structures/work_dict.rst | 2 +- docs/data_structures/worker_array.rst | 7 ++-- libensemble/tests/regression_tests/support.py | 7 ++-- 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2359576ea..f742f4460 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -172,7 +172,6 @@ def __getattr__(cls, name): # Sonny, Lenny, or Bjornstrup 'fncychap': '\\usepackage[Lenny]{fncychap}', 'extraclassoptions': 'openany', - 'sphinxsetup': 'admonitioncolor={RGB}{255,204,204}', 'preamble': r''' \protected\def\sphinxcrossref#1{\texttt{#1}} diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index a02c37f8d..0e06b01a2 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -19,10 +19,10 @@ Allocation function specifications to be set in user calling script and passed t Default: [('allocated',bool)] -:Notes: +.. note:: -* The alloc_specs has the default keys as given above, but may be overidden by the user. -* The tuples defined in the 'out' list are entered into the master :ref:`history array` + * The alloc_specs has the default keys as given above, but may be overidden by the user. + * The tuples defined in the 'out' list are entered into the master :ref:`history array` - -:Examples: +.. seealso:: + * JL fill diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index 70492bc76..07de7f50d 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -17,3 +17,6 @@ Exit criteria for libEnsemble:: Stop after this amount of seconds have elapsed (since the libEnsemble manager has been initialized) 'stop_val' [(str,float)] : Stop when H[str] (for some field str returned from sim_out or gen_out) has been observed with a value less than the float given + +.. seealso:: + JL fill diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index a90529afa..528146ed3 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -21,28 +21,28 @@ Generation function specifications to be set in user calling script and passed t 'save_every_k' [int] : Save history array to file after every k generated points. -:Notes: +.. note:: -* The user may define other fields to be passed to the generator function. -* The tuples defined in the 'out' list are entered into the master :ref:`history array` -* The generator 'out' field will generally include a variable(s) which is used for the simulator 'in' field, - in which case only the variable name is required for the simulator 'in' field. E.g. The - **test_6-hump_camel_uniform_sampling.py** example below, matches the corresponding - :ref:`sim_specs example`, where 'x' is defined in the gen_specs 'out' field to give - two positional floats. + * The user may define other fields to be passed to the generator function. + * The tuples defined in the 'out' list are entered into the master :ref:`history array` + * The generator 'out' field will generally include a variable(s) which is used for the simulator 'in' field, + in which case only the variable name is required for the simulator 'in' field. E.g. The + **test_6-hump_camel_uniform_sampling.py** example below, matches the corresponding + :ref:`sim_specs example`, where 'x' is defined in the gen_specs 'out' field to give + two positional floats. -:Examples: +.. seealso:: -.. _gen-specs-exmple1: + .. _gen-specs-exmple1: -From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` -.. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py - :start-at: gen_specs - :end-before: end_gen_specs_rst_tag + .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py + :start-at: gen_specs + :end-before: end_gen_specs_rst_tag -In this example, the generation function *uniform_random_sample* will generate 500 random points -uniformly over the 2D domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. -The libEnsemble manager is set to dump the history array to file after every 300 generated points, -though in this case it will only happen after 500 points due to the batch size. + In this example, the generation function *uniform_random_sample* will generate 500 random points + uniformly over the 2D domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. + The libEnsemble manager is set to dump the history array to file after every 300 generated points, + though in this case it will only happen after 500 points due to the batch size. diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 017b7430d..78a5f8f3c 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -19,13 +19,13 @@ Below are the protected fields used in ``H`` .. literalinclude:: ../../libensemble/libE_fields.py :start-at: libE_fields -:Examples: +.. seealso:: -See example :doc:`sim_specs<./sim_specs>`, :doc:`gen_specs<./gen_specs>`, and :doc:`alloc_specs<./alloc_specs>`. + See example :doc:`sim_specs<./sim_specs>`, :doc:`gen_specs<./gen_specs>`, and :doc:`alloc_specs<./alloc_specs>`. -Users can also check the internal consistency of a History array by importing ``check_inputs()`` -and calling it with their gen, alloc, and sim specs as keyword arguments:: + Users can also check the internal consistency of a History array by importing ``check_inputs()`` + and calling it with their gen, alloc, and sim specs as keyword arguments:: - from libensemble.libE import check_inputs + from libensemble.libE import check_inputs - check_inputs(H0=my_H, sim_specs=sim_specs, alloc_specs=alloc_specs, gen_specs=gen_specs) + check_inputs(H0=my_H, sim_specs=sim_specs, alloc_specs=alloc_specs, gen_specs=gen_specs) diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 87ff5ca7b..29fa51f8c 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -15,10 +15,10 @@ If worker ``i`` sends back ``persis_info``, it is stored in ``persis_info[i]``. can be used to, for example, pass a random stream back to the manager to be included in future work from the allocation function. -:Examples: +.. seealso:: -From: ``libEnsemble/tests/regression_tests/support.py`` + From: ``libEnsemble/tests/regression_tests/support.py`` -.. literalinclude:: ../../libensemble/tests/regression_tests/support.py - :start-at: persis_info_1 - :end-before: end_persis_info_rst_tag + .. literalinclude:: ../../libensemble/tests/regression_tests/support.py + :start-at: persis_info_1 + :end-before: end_persis_info_rst_tag diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 1a75d9743..a7dfaaef7 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -21,7 +21,7 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` 'persistent' [bool]: True if worker 'i' will enter persistent mode -.. Hint:: +.. seealso:: For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index e378caaf8..e592789bd 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -20,7 +20,6 @@ waiting, persistent gen 0 2 0 worker blocked by some other calculation 1 0 1 ========================================= ======= ============ ======= -:Note: - -* libE only receives from workers with 'active' nonzero -* libE only calls the alloc_f if some worker has 'active' zero +.. note:: + * libE only receives from workers with a nonzero 'active' state + * libE only calls the alloc_f if some worker has an 'active' state of zero diff --git a/libensemble/tests/regression_tests/support.py b/libensemble/tests/regression_tests/support.py index 23cf171bf..dcd2ec220 100644 --- a/libensemble/tests/regression_tests/support.py +++ b/libensemble/tests/regression_tests/support.py @@ -37,13 +37,12 @@ def nan_func(calc_in, persis_info, sim_specs, libE_info): # give_sim_work_first persis_info persis_info_1 = {'total_gen_calls': 0, # Counts gen calls in alloc_f 'last_worker': 0, # Remembers last gen worker in alloc_f - 'next_to_give': 0, # Remembers next H row to give in alloc_f - } + 'next_to_give': 0} # Remembers next H row to give in alloc_f + persis_info_1[0] = {'run_order': {}, # Used by manager to remember run order 'old_runs': {}, # Used by manager to store old runs order 'total_runs': 0, # Used by manager to count total runs - 'rand_stream': np.random.RandomState(1) - } + 'rand_stream': np.random.RandomState(1)} # end_persis_info_rst_tag persis_info_2 = copy.deepcopy(persis_info_1) From 44c234bfbd10ccc96a8816f343d17497d0747d80 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 15:34:55 -0500 Subject: [PATCH 352/644] Working on more documentation --- docs/data_structures/alloc_specs.rst | 9 ++++++++- docs/data_structures/exit_criteria.rst | 18 ++++++++++++------ docs/data_structures/gen_specs.rst | 6 ++++-- docs/data_structures/libE_specs.rst | 5 +++++ .../test_branin_aposmm_nlopt_and_then_scipy.py | 1 + ..._uniform_sampling_one_residual_at_a_time.py | 9 +++++---- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 0e06b01a2..4131d6c83 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -25,4 +25,11 @@ Allocation function specifications to be set in user calling script and passed t * The tuples defined in the 'out' list are entered into the master :ref:`history array` .. seealso:: - * JL fill + The script `test_chwirut_uniform_sampling_one_residual_at_a_time.py`_ + specifies fields to be used by the allocation function. + + .. literalinclude:: ../../libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py + :start-at: alloc_specs + :end-before: end_alloc_specs_rst_tag + +.. _test_chwirut_uniform_sampling_one_residual_at_a_time.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index 07de7f50d..5a2cd6114 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -10,13 +10,19 @@ Exit criteria for libEnsemble:: Optional keys (At least one must be given) : 'sim_max' [int] : - Stop after this many points have been evaluated (by sim_f) in current run + Stop when this many new points have been evaluated by sim_f 'gen_max' [int] : - Stop after this many points have been generated by gen_f in current run + Stop when this many new points have been generated by gen_f 'elapsed_wallclock_time' [float] : - Stop after this amount of seconds have elapsed (since the libEnsemble manager has been initialized) - 'stop_val' [(str,float)] : - Stop when H[str] (for some field str returned from sim_out or gen_out) has been observed with a value less than the float given + Stop when this time (since the manager has been initialized) has elapsed + 'stop_val' [(str, float)] : + Stop when H[str] < float for the given (str, float pair) .. seealso:: - JL fill + From `test_branin_aposmm_nlopt_and_then_scipy.py`_. + + .. literalinclude:: ../../libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py + :start-at: exit_criteria + :end-before: end_exit_criteria_rst_tag + +.. _test_branin_aposmm_nlopt_and_then_scipy.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 528146ed3..c2cfd8c8a 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -36,13 +36,15 @@ Generation function specifications to be set in user calling script and passed t .. _gen-specs-exmple1: - From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + From `test_6-hump_camel_uniform_sampling.py`_ .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py :start-at: gen_specs :end-before: end_gen_specs_rst_tag - In this example, the generation function *uniform_random_sample* will generate 500 random points + In this example, the generation function ``uniform_random_sample`` will generate 500 random points uniformly over the 2D domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. The libEnsemble manager is set to dump the history array to file after every 300 generated points, though in this case it will only happen after 500 points due to the batch size. + +.. _test_6-hump_camel_uniform_sampling.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index 6fdad35e4..2d47e4e19 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -16,3 +16,8 @@ Specifications for libEnsemble:: 'abort_on_exception' [boolean] : In MPI mode, whether to call MPI_ABORT on an exception. Default: True IF False, an exception will be raised by the manager. + +.. seealso:: + Examples in `common.py`_ + +.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index 075efc66d..ef22e533e 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -66,6 +66,7 @@ exit_criteria = {'sim_max': 150, 'elapsed_wallclock_time': 100, 'stop_val': ('f', -1)} +# end_exit_criteria_rst_tag # Perform the run for run in range(2): diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 17f561389..fdee1c242 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -63,10 +63,11 @@ 'ub': 2*np.ones(n), 'components': m} -alloc_specs = {'alloc_f': alloc_f, - 'out': [('allocated', bool)], - 'stop_on_NaNs': True, - 'stop_partial_fvec_eval': True} +alloc_specs = {'alloc_f': alloc_f, # Allocation function + 'out': [('allocated', bool)], # Output fields (included in History) + 'stop_on_NaNs': True, # Should alloc_f preempt evals + 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals +# end_alloc_specs_rst_tag persis_info = per_worker_stream(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) From 0c74a26570ff0cda642fca599c366754fcd4f273 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 15:40:42 -0500 Subject: [PATCH 353/644] Whitespace --- docs/data_structures/exit_criteria.rst | 2 +- docs/data_structures/history_array.rst | 2 +- docs/data_structures/work_dict.rst | 2 +- .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index 5a2cd6114..a4a762ac4 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -10,7 +10,7 @@ Exit criteria for libEnsemble:: Optional keys (At least one must be given) : 'sim_max' [int] : - Stop when this many new points have been evaluated by sim_f + Stop when this many new points have been evaluated by sim_f 'gen_max' [int] : Stop when this many new points have been generated by gen_f 'elapsed_wallclock_time' [float] : diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 78a5f8f3c..38ede0a0e 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -19,7 +19,7 @@ Below are the protected fields used in ``H`` .. literalinclude:: ../../libensemble/libE_fields.py :start-at: libE_fields -.. seealso:: +.. seealso:: See example :doc:`sim_specs<./sim_specs>`, :doc:`gen_specs<./gen_specs>`, and :doc:`alloc_specs<./alloc_specs>`. diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index a7dfaaef7..fcf9a3398 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -25,6 +25,6 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. -.. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py +.. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py .. _start_persistent_local_opt_gens.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_persistent_local_opt_gens.py .. _test_6-hump_camel_with_different_nodes_uniform_sample.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index fdee1c242..046048881 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -64,8 +64,8 @@ 'components': m} alloc_specs = {'alloc_f': alloc_f, # Allocation function - 'out': [('allocated', bool)], # Output fields (included in History) - 'stop_on_NaNs': True, # Should alloc_f preempt evals + 'out': [('allocated', bool)], # Output fields (included in History) + 'stop_on_NaNs': True, # Should alloc_f preempt evals 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals # end_alloc_specs_rst_tag From 043bbb4c4cf5eb20824b1fa7201ea053e95e0f10 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 16 Oct 2019 15:54:51 -0500 Subject: [PATCH 354/644] Removing usage of 'color' to avoid confusion --- libensemble/tests/regression_tests/common.py | 12 ++++++------ .../regression_tests/script_test_balsam_hworld.py | 2 +- .../test_chwirut_pounders_splitcomm.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index aaa2f9784..672fd362b 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -36,7 +36,7 @@ def mpi_parse_args(args): from mpi4py import MPI nworkers = MPI.COMM_WORLD.Get_size()-1 is_master = MPI.COMM_WORLD.Get_rank() == 0 - libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} + libE_specs = {'comm': MPI.COMM_WORLD, 'comms': 'mpi'} return nworkers, is_master, libE_specs, args.tester_args @@ -50,16 +50,16 @@ def mpi_comm_excl(exc=[0], comm=None): return mpi_comm, MPI.COMM_NULL -def mpi_comm_split(num_colors, comm=None): +def mpi_comm_split(num_parts, comm=None): "Split COMM_WORLD into sub-communicators for MPI comms." from mpi4py import MPI parent_comm = comm or MPI.COMM_WORLD parent_size = parent_comm.Get_size() key = parent_comm.Get_rank() - row_size = parent_size // num_colors - color = key // row_size - sub_comm = parent_comm.Split(color, key) - return sub_comm, color + row_size = parent_size // num_parts + sub_comm_number = key // row_size + sub_comm = parent_comm.Split(sub_comm_number, key) + return sub_comm, sub_comm_number def local_parse_args(args): diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index ee6cc5609..16ac5b694 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -24,7 +24,7 @@ def build_simfunc(): subprocess.check_call(buildstring.split()) -libE_specs = {'comm': MPI.COMM_WORLD, 'color': 0, 'comms': 'mpi'} +libE_specs = {'comm': MPI.COMM_WORLD, 'comms': 'mpi'} nworkers = MPI.COMM_WORLD.Get_size() - 1 is_master = MPI.COMM_WORLD.Get_rank() == 0 diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 2b2f8dc3f..377d22bf8 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -25,7 +25,7 @@ num_comms = 2 # Must have atleast num_comms*2 processors nworkers, is_master, libE_specs, _ = parse_args() -libE_specs['comm'], color = mpi_comm_split(num_comms) +libE_specs['comm'], sub_comm_number = mpi_comm_split(num_comms) is_master = (libE_specs['comm'].Get_rank() == 0) # Declare the run parameters/functions @@ -72,5 +72,5 @@ J = EvaluateJacobian(H['x'][np.argmin(H['f'])]) assert np.linalg.norm(J) < 2000 - outname = os.path.splitext(__file__)[0] + '_color' + str(color) + outname = os.path.splitext(__file__)[0] + '_sub_comm' + str(sub_comm_number) save_libE_output(H, persis_info, outname, nworkers) From 0aaadbb7236298013804c6b9077fd4db51551113 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 8 Oct 2019 11:15:09 -0500 Subject: [PATCH 355/644] Changing default alloc func to only give gen_specs['in'] if it's a field with length (cherry picked from commit 29eabfe45e8242021b3d9f04195a60f024d77fbe) --- libensemble/alloc_funcs/give_sim_work_first.py | 5 ++++- .../tests/regression_tests/test_1d_uniform_sampling.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index f52118972..6b2f9d78b 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -78,6 +78,9 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # Give gen work gen_count += 1 - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + if 'in' in gen_specs and len(gen_specs['in']): + gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + else: + gen_work(Work, i, [], [], persis_info[i]) return Work, persis_info diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling.py index 131d9cd5a..2ce48ab30 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling.py @@ -26,7 +26,6 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} gen_specs = {'gen_f': gen_f, - 'in': ['sim_id'], 'out': [('x', float, (1,))], 'lb': np.array([-3]), 'ub': np.array([3]), From a6f9ca363abcc8fc5e831a371d018b87f55b998c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 17 Oct 2019 11:14:55 -0500 Subject: [PATCH 356/644] Adding comment --- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 94a145662..d537012a0 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -46,7 +46,7 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, + 'batch_mode': True, # a different comment 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), 'localopt_method': 'LD_MMA', From a6f0445c12ccb15c473fcd7c4d961c3eb3d8c49c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 17 Oct 2019 11:36:49 -0500 Subject: [PATCH 357/644] Revert "Adding comment" This reverts commit a6f9ca363abcc8fc5e831a371d018b87f55b998c. --- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index d537012a0..94a145662 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -46,7 +46,7 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, # a different comment + 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), 'localopt_method': 'LD_MMA', From 119139c4a649f8b086c13b153c9af9e9d7d1ef41 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 21 Oct 2019 07:05:02 -0500 Subject: [PATCH 358/644] Background color on hints --- docs/conf.py | 8 ++++++++ docs/data_structures/history_array.rst | 11 ++++++----- docs/data_structures/worker_array.rst | 22 +++++++++++++++++++--- docs/examples/gen_funcs.rst | 4 ++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f742f4460..7ef1e3929 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -175,6 +175,14 @@ def __getattr__(cls, name): 'preamble': r''' \protected\def\sphinxcrossref#1{\texttt{#1}} + + \newsavebox\mytempbox +\definecolor{sphinxnoteBgColor}{RGB}{221,233,239} +\renewenvironment{sphinxnote}[1] + {\begin{lrbox}{\mytempbox}\begin{minipage}{\columnwidth}% + \begin{sphinxlightbox}\sphinxstrong{#1} } + {\end{sphinxlightbox}\end{minipage}\end{lrbox}% + \colorbox{sphinxnoteBgColor}{\usebox{\mytempbox}}} ''', # The paper size ('letterpaper' or 'a4paper'). # diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 38ede0a0e..ab0c7a1b5 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -2,11 +2,10 @@ history array ============= - -Stores the history of the output from gen_f, sim_f, and alloc_f:: +:: H: numpy structured array - History array storing rows for each point. + History to store output from ``gen_f``/``sim_f``/``alloc_f`` for each entry Fields in ``H`` include those specified in ``sim_specs['out']``, ``gen_specs['out']``, and ``alloc_specs['out']``. All values are initiated to @@ -23,8 +22,10 @@ Below are the protected fields used in ``H`` See example :doc:`sim_specs<./sim_specs>`, :doc:`gen_specs<./gen_specs>`, and :doc:`alloc_specs<./alloc_specs>`. - Users can also check the internal consistency of a History array by importing ``check_inputs()`` - and calling it with their gen, alloc, and sim specs as keyword arguments:: +.. hint:: + Users can check the internal consistency of a History array by importing + ``check_inputs()`` and calling it with their ``gen_specs``, ``alloc_specs``, + and ``sim_specs`` as keyword arguments:: from libensemble.libE import check_inputs diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index e592789bd..bf73f9913 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -2,10 +2,20 @@ worker array ============= +:: -Stores information to inform the allocation function about the current state of -the workers. Workers can be in a variety of states. We take the following -convention: + W: numpy structured array + 'active' [int]: + Is the worker active or not + 'persis_state' [int]: + Is the worker in a persis_state + 'blocked' [int]: + Is the worker's resources blocked by another calculation + +Since workers can be an a variety of states, the worker array ``W`` is contains +information about each workers state. This can allow an allocation functions to +determine what work should be performed. +We take the following convention: ========================================= ======= ============ ======= Worker state active persis_state blocked @@ -23,3 +33,9 @@ worker blocked by some other calculation 1 0 1 .. note:: * libE only receives from workers with a nonzero 'active' state * libE only calls the alloc_f if some worker has an 'active' state of zero + +.. seealso:: + For an example allocation function that queries the worker array, see + `persistent_aposmm_alloc`_. + +.. _persistent_aposmm_alloc: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/persistent_aposmm_alloc.py diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 110216e36..5c3fd2495 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -6,9 +6,9 @@ Below are example generation functions available in libEnsemble. .. IMPORTANT:: See the API for generation functions :ref:`here` -uniform_sampling +sampling ---------------- -.. automodule:: uniform_sampling +.. automodule:: sampling :members: :undoc-members: From d8528262bed80357f8e7d410a3d41efad19b477d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 21 Oct 2019 07:46:11 -0500 Subject: [PATCH 359/644] A space --- docs/data_structures/worker_array.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index bf73f9913..338096d3b 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -34,7 +34,7 @@ worker blocked by some other calculation 1 0 1 * libE only receives from workers with a nonzero 'active' state * libE only calls the alloc_f if some worker has an 'active' state of zero -.. seealso:: +.. seealso:: For an example allocation function that queries the worker array, see `persistent_aposmm_alloc`_. From 337c7919856e469dc3d0e8432e5cb55dd639704c Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 21 Oct 2019 12:56:40 -0500 Subject: [PATCH 360/644] rewording FAQ, move mac-stuff to own question --- docs/FAQ.rst | 127 +++++++++++++++++++++++---------------------------- 1 file changed, 56 insertions(+), 71 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index d0c44346f..f72a72cca 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -7,52 +7,52 @@ If you have any further questions, feel free to contact us through Support_ .. _Support: https://libensemble.readthedocs.io/en/latest/quickstart.html#support -Parallel Debugging ------------------- +Local Parallel Debugging +------------------------ -**How can I perform parallel debugging on libEnsemble, or debug specific processes?** +**How can I debug specific libEnsemble processes?** +This is most easily addressed when running libEnsemble locally. Try -This is most easily addressed when running with MPI. -Try the following: ``mpiexec -np [num processes] xterm -e 'python [calling script].py'`` + ``mpiexec -np [num processes] xterm -e 'python [calling script].py'`` -This will launch an xterm terminal window specific to each process. Mac users will +to launch an xterm terminal window specific to each process. Mac users will need to install xQuartz_. -.. _xQuartz: https://www.xquartz.org/ - -If running in ``local`` comms mode try using one of the ``ForkablePdb`` -routines in ``libensemble/util/forkpdb.py`` to set breakpoints. How well these -work may depend on the system. Usage:: +If running in ``local`` mode try using one of the ``ForkablePdb`` +routines in ``libensemble/util/forkpdb.py`` to set breakpoints and debug similarly +to ``pdb``. How well this works varies by system:: from libensemble.util.forkpdb import ForkablePdb ForkablePdb().set_trace() +.. _xQuartz: https://www.xquartz.org/ + + AssertionError - idle workers ----------------------------- -**AssertionError: Should not wait for workers when all workers are idle.** - -with ``mpiexec -np 1 python [calling script].py`` +**What does "AssertionError: Should not wait for workers when all workers are idle." +mean?** -This error occurs when the manager is waiting although no workers are busy. -In the above case, this occurs because an MPI libEnsemble run was initiated with -only one process, resulting in one manager but no workers. +This error occurs when the manager is waiting although no workers are busy, or +an MPI libEnsemble run was initiated with only one process, resulting in one +manager but no workers. This may also occur with two processes if you are using a persistent generator. The generator will occupy the one worker, leaving none to run simulation functions. -Not enough processors per worker to honour arguments ----------------------------------------------------- +Not enough processors per worker to honor arguments +--------------------------------------------------- -**libensemble.resources.ResourcesException: Not enough processors per worker to honour arguments.** +**I keep getting: "Not enough processors per worker to honor arguments." when +using the job controller. Can I launch jobs anyway?** -This error often occurs when there aren't enough cores/nodes available to launch -jobs with the job controller. Automatic partitioning of resources can be disabled -if you want to oversubscribe (often if testing on a local machine) by configuring -the job controller with ``auto_resources=False``. E.g.:: +Automatic partitioning of resources can be disabled if you want to oversubscribe +(often if testing on a local machine) by configuring the job controller with +``auto_resources=False``. E.g.:: jobctrl = MPIJobController(auto_resources=False) @@ -65,9 +65,8 @@ FileExistsError **FileExistsError: [Errno 17] File exists: './sim_worker1'** -This can happen when libEnsemble tries to create sim directories that already exist. -If these directories do not already exist, another possibility is that you are trying -to run libEnsemble using ``mpiexec`` when the ``libE_specs['comms']`` option is +This can happen when libEnsemble tries to create sim directories that already exist, +or libEnsemble is launched with ``mpiexec`` when the ``libE_specs['comms']`` option is set to ``local``. To create differently named sim directories, you can use the ``sim_dir_suffix`` @@ -77,9 +76,11 @@ option in :ref:`sim_specs`. libEnsemble hangs when using mpi4py ----------------------------------- +**Why does libEnsemble hang on certain systems when running with MPI?** + This may occur if matching probes, which mpi4py uses by default, are not supported by the communications fabric. This has been observed with Intels Truescale (TMI) -fabric at time of writing. This can be solved either by switching fabrics or disabling +fabric at time of writing. This can be solved by switching fabrics or disabling matching probes before the MPI module is first imported. Add these two lines BEFORE ``from mpi4py import MPI``:: @@ -98,12 +99,8 @@ This may manifest with the following error: **_pickle.UnpicklingError: invalid load key, '\x00'.** or some similar variation. This has been observed with the OFA fabric. The solution -is to either switch fabric or turn off matching probes. - -Add these two lines BEFORE 'from mpi4py import MPI':: - - import mpi4py - mpi4py.rc.recv_mprobe = False +is to either switch fabric or turn off matching probes. See the answer for "Why +does libEnsemble hang on certain systems when running with MPI?" For more information see: https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe @@ -115,48 +112,18 @@ PETSc and MPI errors with ``python [test with PETSc].py --comms local --nworkers 4`` -This error occurs on some platforms, including Travis, when using PETSc with libEnsemble +This error occurs on some platforms, including Travis CI, when using PETSc with libEnsemble in ``local`` (multiprocessing) mode. We believe this is due to PETSc initializing MPI before libEnsemble forks processes using multiprocessing. The recommended solution is running libEnsemble in MPI mode. An alternative solution may be using a serial build of PETSc. -Note: This error does not occur on all platforms and may depend on how multiprocessing -handles an existing MPI communicator in a particular platform. - - -Fatal error in MPI_Init_thread ------------------------------- - -**Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed** - - -This error may be macOS specific. MPI uses TCP to initiate connections, -and needs the local hostname to function. MPI checks /etc/hosts for this information, -and causes the above error if it can't find the correct entry. - -Resolve this by appending ``127.0.0.1 [your hostname]`` to /etc/hosts. -Unfortunately, ``127.0.0.1 localhost`` isn't satisfactory for preventing this -error. - - -macOS - Firewall prompts ------------------------- - -**macOS - Constant Firewall Security permission windows throughout Job Controller task** +Note: This error may depend on how multiprocessing handles an existing MPI +communicator in a particular platform. -There are several ways to address this nuisance, but all involve trial and error. -An easy (but insecure) solution is temporarily disabling the Firewall through -System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, -adding a Firewall "Allow incoming connections" rule can be attempted for the offending -Job Controller executable. We've had limited success running ``sudo codesign --force --deep --sign - /path/to/application.app`` -on our Job Controller executables, then confirming the next alerts for the executable -and ``mpiexec.hydra``. - - -Running out of contexts when running libEnsemble in distributed mode on TMI fabric ----------------------------------------------------------------------------------- +Running out of contexts in distributed mode on TMI fabric +--------------------------------------------------------- The error message may be similar to below: @@ -178,8 +145,26 @@ Another alternative is to run libEnsemble in central mode, in which libEnsemble nodes, while launching all sub-jobs to other nodes. -macOS - PETSc Installation issues ---------------------------------- +macOS - Common Issues +------------------- + +**"Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"** + +Resolve this by appending ``127.0.0.1 [your hostname]`` to /etc/hosts. +Unfortunately, ``127.0.0.1 localhost`` isn't satisfactory for preventing this +error. + + +**How do I stop the Firewall Security popups when running with the Job Controller?** + +There are several ways to address this nuisance, but all involve trial and error. +An easy (but insecure) solution is temporarily disabling the Firewall through +System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, +adding a Firewall "Allow incoming connections" rule can be attempted for the offending +Job Controller executable. We've had limited success running ``sudo codesign --force --deep --sign - /path/to/application.app`` +on our Job Controller executables, then confirming the next alerts for the executable +and ``mpiexec.hydra``. + **Frozen PETSc installation following a failed wheel build with** ``pip install petsc petsc4py`` From 56ada89c3a52498c7f192b7ae50f42f18bc85faf Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 21 Oct 2019 14:45:12 -0500 Subject: [PATCH 361/644] notes -> note blocks. Reformatting. fixes reference. first round of platforms info. --- README.rst | 12 +++++----- docs/FAQ.rst | 7 +++--- docs/examples/gen_funcs.rst | 2 +- docs/index.rst | 10 ++++++++ docs/platforms/bebop.rst | 47 +++++++++++++++++++++++++++++++++++++ docs/platforms/theta.rst | 19 +++++++++++++++ 6 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 docs/platforms/bebop.rst create mode 100644 docs/platforms/theta.rst diff --git a/README.rst b/README.rst index 36766cc9f..297ee22c3 100644 --- a/README.rst +++ b/README.rst @@ -155,10 +155,11 @@ which can be viewed after running the tests via the HTML file available online at `Coveralls `_. -Note: The job_controller tests can be run using the direct-launch or -Balsam job controllers. Although only the direct-launch versions can -be run on Travis CI, Balsam integration with libEnsemble is now tested via -``test_balsam_hworld.py``. +.. note:: + The job_controller tests can be run using the direct-launch or + Balsam job controllers. Although only the direct-launch versions can be run + on Travis CI, Balsam integration with libEnsemble is now tested via + ``test_balsam_hworld.py``. Basic Usage @@ -176,7 +177,7 @@ where ``N`` is the number of processors. This will launch one manager and ``N-1`` workers. If running in local mode, which uses Python's multiprocessing module, the -'local' comms option and the number of workers must be specified. The script +``local`` comms option and the number of workers must be specified. The script can then be run as a regular python script:: python myscript.py @@ -240,4 +241,3 @@ or communicate (and establish a private channel, if desired) at: .. _pytest: https://pypi.org/project/pytest/ .. _pytest-cov: https://pypi.org/project/pytest-cov/ .. _pytest-timeout: https://pypi.org/project/pytest-timeout/ - diff --git a/docs/FAQ.rst b/docs/FAQ.rst index f72a72cca..6110e180f 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -118,8 +118,9 @@ before libEnsemble forks processes using multiprocessing. The recommended soluti is running libEnsemble in MPI mode. An alternative solution may be using a serial build of PETSc. -Note: This error may depend on how multiprocessing handles an existing MPI -communicator in a particular platform. +.. note:: + This error may depend on how multiprocessing handles an existing MPI + communicator in a particular platform. Running out of contexts in distributed mode on TMI fabric @@ -146,7 +147,7 @@ nodes, while launching all sub-jobs to other nodes. macOS - Common Issues -------------------- +--------------------- **"Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"** diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 110216e36..30fe00f12 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -8,7 +8,7 @@ Below are example generation functions available in libEnsemble. uniform_sampling ---------------- -.. automodule:: uniform_sampling +.. automodule:: sampling :members: :undoc-members: diff --git a/docs/index.rst b/docs/index.rst index 8a624bfca..f3eaac54d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,8 @@ libEnsemble is a library for managing ensemble-like collections of computations. Quickstart user_guide tutorials/local_sine_tutorial + platforms/bebop + platforms/theta contributing FAQ libE_module @@ -75,6 +77,14 @@ libEnsemble is a library for managing ensemble-like collections of computations. tutorials/local_sine_tutorial + .. toctree:: + :maxdepth: 2 + :caption: Running on: + + platforms/bebop + platforms/theta + + .. toctree:: :maxdepth: 2 :caption: User Guide: diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst new file mode 100644 index 000000000..551f21042 --- /dev/null +++ b/docs/platforms/bebop.rst @@ -0,0 +1,47 @@ +===== +Bebop +===== + +Bebop_ is the newest addition to the computational power of LCRC at Argonne +National Laboratory, featuring both Intel Broadwell and Knights Landing nodes. + + + +Before getting started +---------------------- + +An Argonne LCRC_ account is required to access Bebop. Interested users will need +to apply for and be granted an account before continuing. + + + +Configuring Python +------------------ + +Begin by loading the Python 3 Anaconda_ module:: + + module load anaconda3/2018.12 + +Create a Conda_ virtual environment in which to install libEnsemble and all +dependencies:: + + conda config --add channels intel + conda create --name my_env intelpython3_core python=3 + source activate my_env + +You should have an indication that the virtual environment is activated. +Install mpi4py_ and libEnsemble in this environment, making sure to reference +the pre-installed Intel MPI Compiler. Your prompt should be similar to the +following block: + +.. code-block:: console + + (my_env) user@beboplogin4:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py + (my_env) user@beboplogin4:~$ pip install libensemble + + +.. _Bebop: https://www.lcrc.anl.gov/systems/resources/bebop/ +.. _LCRC: https://www.lcrc.anl.gov +.. _Anaconda: https://www.anaconda.com/distribution/ +.. _Conda: https://conda.io/en/latest/ +.. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst new file mode 100644 index 000000000..4395272f9 --- /dev/null +++ b/docs/platforms/theta.rst @@ -0,0 +1,19 @@ +===== +Theta +===== + +Theta_ is a 11.69 petaflops system based on the second-generation Intel® Xeon Phi™ + processor, available within ALCF at Argonne National Laboratory. + + + +Before getting started +---------------------- + +An Argonne ALCF_ account is required to access Theta. Interested users will need +to apply for and be granted an account before continuing. + + + +.. _ALCF: https://www.alcf.anl.gov/ +.. _Theta: https://www.alcf.anl.gov/theta From a574b99138936565a4c75a77eb9c0734a315ee4f Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 21 Oct 2019 16:17:25 -0500 Subject: [PATCH 362/644] complete first draft for bebop --- docs/platforms/bebop.rst | 92 +++++++++++++++++++++++++++++++++++++++- docs/platforms/theta.rst | 4 +- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 551f21042..284a588c7 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -11,8 +11,8 @@ Before getting started ---------------------- An Argonne LCRC_ account is required to access Bebop. Interested users will need -to apply for and be granted an account before continuing. - +to apply for and be granted an account before continuing. To submit jobs to Bebop, +users can charge hours to a project or their personal allocation (default). Configuring Python @@ -29,6 +29,10 @@ dependencies:: conda create --name my_env intelpython3_core python=3 source activate my_env + +Installing Dependencies and libEnsemble +--------------------------------------- + You should have an indication that the virtual environment is activated. Install mpi4py_ and libEnsemble in this environment, making sure to reference the pre-installed Intel MPI Compiler. Your prompt should be similar to the @@ -40,8 +44,92 @@ following block: (my_env) user@beboplogin4:~$ pip install libensemble +Job Submission +-------------- + +Bebop uses Slurm_ for job management. The two commands you'll likely use the most +to run jobs are ``srun`` and ``sbatch`` for running interactively and batch, respectively. + + +Interactive Runs +^^^^^^^^^^^^^^^^ + +You can allocate four Knights Landing nodes for thirty minutes through the following:: + + salloc -N 4 -p knl -A [username OR project] -t 00:30:00 + + +With your nodes allocated, queue your job to start with five MPI ranks:: + + srun -n 5 python calling.py + + +``mpirun`` should also work. This launches a worker on every node, on which every +worker can perform stand-alone calculations or launch jobs through the job controller. + +.. note:: + When performing an MPI libEnsemble run and not oversubscribing, specify one + more MPI process than the number of allocated nodes. The Manager and first + worker run together on a node. + + +Batch Runs +^^^^^^^^^^ + +Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example +for a libEnsemble use-case running on Broadwell nodes resembles the following: + +.. code-block:: bash + + #!/bin/bash + #SBATCH -J myjob + #SBATCH -N 4 + #SBATCH -p bdwall + #SBATCH -A myproject + #SBATCH -o myjob.out + #SBATCH -e myjob.error + #SBATCH -t 00:15:00 + + # These four lines construct a machinefile for the job controller and slurm + srun hostname | sort -u > node_list + head -n 1 node_list > machinefile.$SLURM_JOBID + cat node_list >> machinefile.$SLURM_JOBID + export SLURM_HOSTFILE=machinefile.$SLURM_JOBID + + srun --ntasks 5 python3 calling_script.py + + +With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble +on Bebop is as simple as:: + + sbatch myscript.sh + + +Debugging Strategies +-------------------- + +View the status of your submitted jobs with ``squeue``. + +It's not recommended to debug compute-intensive tasks on the login nodes. Start +a bash session on a Knights Landing node for thirty minutes with:: + + srun --pty -A [username OR project] -p knl -t 00:30:00 /bin/bash + +.. note:: + You will need to re-activate your conda virtual environment and reload your + modules! Configuring this routine to occur automatically is recommended. + + +Additional Information +---------------------- + +See the LCRC Bebop docs here_ for much more information. + + .. _Bebop: https://www.lcrc.anl.gov/systems/resources/bebop/ .. _LCRC: https://www.lcrc.anl.gov .. _Anaconda: https://www.anaconda.com/distribution/ .. _Conda: https://conda.io/en/latest/ .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ +.. _Slurm: https://slurm.schedmd.com/ +.. _here: https://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs/running-jobs-on-bebop/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 4395272f9..88d3ca871 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -11,7 +11,9 @@ Before getting started ---------------------- An Argonne ALCF_ account is required to access Theta. Interested users will need -to apply for and be granted an account before continuing. +to apply for and be granted an account before continuing. To submit jobs to Theta, +users must charge their jobs to a project (likely specified while requesting an +account.) From 6ee450772c5470ce4981dc881516ec99e3f36ba4 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 21 Oct 2019 17:27:45 -0500 Subject: [PATCH 363/644] bebop clarification. beginning theta info --- docs/platforms/bebop.rst | 14 +++--- docs/platforms/theta.rst | 97 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 284a588c7..232e35f5e 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -7,13 +7,16 @@ National Laboratory, featuring both Intel Broadwell and Knights Landing nodes. -Before getting started +Before Getting Started ---------------------- An Argonne LCRC_ account is required to access Bebop. Interested users will need to apply for and be granted an account before continuing. To submit jobs to Bebop, users can charge hours to a project or their personal allocation (default). +Bebop consists primarily of login and compute nodes. Users start on the login +nodes, and schedule work for execution on the compute nodes. + Configuring Python ------------------ @@ -30,7 +33,7 @@ dependencies:: source activate my_env -Installing Dependencies and libEnsemble +Installing libEnsemble and Dependencies --------------------------------------- You should have an indication that the virtual environment is activated. @@ -47,7 +50,7 @@ following block: Job Submission -------------- -Bebop uses Slurm_ for job management. The two commands you'll likely use the most +Bebop uses Slurm_ for job submission and management. The two commands you'll likely use the most to run jobs are ``srun`` and ``sbatch`` for running interactively and batch, respectively. @@ -80,6 +83,7 @@ Batch scripts specify run-settings using ``#SBATCH`` statements. A simple exampl for a libEnsemble use-case running on Broadwell nodes resembles the following: .. code-block:: bash + :linenos: #!/bin/bash #SBATCH -J myjob @@ -100,7 +104,7 @@ for a libEnsemble use-case running on Broadwell nodes resembles the following: With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble -on Bebop is as simple as:: +on Bebop becomes:: sbatch myscript.sh @@ -123,7 +127,7 @@ a bash session on a Knights Landing node for thirty minutes with:: Additional Information ---------------------- -See the LCRC Bebop docs here_ for much more information. +See the LCRC Bebop docs here_ for much more information about Bebop. .. _Bebop: https://www.lcrc.anl.gov/systems/resources/bebop/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 88d3ca871..e6a969e48 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -2,8 +2,8 @@ Theta ===== -Theta_ is a 11.69 petaflops system based on the second-generation Intel® Xeon Phi™ - processor, available within ALCF at Argonne National Laboratory. +Theta_ is a 11.69 petaflops system based on the second-generation Intel Xeon Phi +processor, available within ALCF at Argonne National Laboratory. @@ -15,7 +15,100 @@ to apply for and be granted an account before continuing. To submit jobs to Thet users must charge their jobs to a project (likely specified while requesting an account.) +Theta consists of numerous types of nodes, but libEnsemble users will mostly interact +with login, Machine Oriented Mini-server (MOM) and compute nodes. MOM nodes execute +user batch scripts to run on the compute nodes. + +Configuring Python +------------------ + + + +Installing libEnsemble and Dependencies +--------------------------------------- + + +Balsam +^^^^^^ + + +Job Submission +-------------- + +Theta features one default production queue, ``default``, and two debug queues, +``debug-cache-quad`` and ``debug-flat-quad``. + +.. note:: + For the default queue, the minimum number of nodes to allocate is 128! + +Theta uses Cobalt for job submission and management. The most important two +commands to run and manage jobs are ``qsub`` and ``aprun``, for submitting batch +scripts from the login nodes and launching jobs to the compute nodes respectively. + +Because mpi4py doesn't function on the Theta MOM nodes, libEnsemble is commonly +run in multiprocessing mode with all workers running on the MOM nodes. In these +circumstances, libEnsemble's job controller takes responsibility for ``aprun`` +submissions of jobs to compute nodes. + + + +Interactive Runs +^^^^^^^^^^^^^^^^ + +Users can run interactively with ``qsub`` by specifying the ``-I`` flag, similarly +to the following:: + + qsub -A [project] -n 128 -q default -t 120 -I + +This will place the user on a MOM node. To launch the job to the allocated compute nodes, +run:: + + aprun... + +.. note:: + You will need to re-activate your conda virtual environment and reload your + modules! Configuring this routine to occur automatically is recommended. + +Batch Runs +^^^^^^^^^^ + +Batch scripts specify run-settings using ``#COBALT`` statements. A simple example +for a libEnsemble use-case resembles the following: + +.. code-block:: bash + + + +With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble +on Theta becomes:: + + qsub --mode script myscript.sh + + +Running with Balsam +^^^^^^^^^^^^^^^^^^^ + + +Debugging Strategies +-------------------- + +Each of the two debug queues has sixteen nodes apiece. A user can use up to +eight nodes at a time for a maximum of one hour. Allocate nodes on the debug +queue interactively:: + + qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I + + +Additional Information +---------------------- + +See the ALCF guides_ on XC40 systems for much more information about Theta. + +Become more familiar with Balsam here_. + .. _ALCF: https://www.alcf.anl.gov/ .. _Theta: https://www.alcf.anl.gov/theta +.. _guides: https://www.alcf.anl.gov/user-guides/computational-systems +.. _here: https://balsam.readthedocs.io/en/latest/ From 88b04f6e40ef7ba4320d357a1bf4ed795309d0e0 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 22 Oct 2019 13:01:32 -0500 Subject: [PATCH 364/644] further formatting, external links, batch scripts, theta setup/balsam information --- docs/platforms/bebop.rst | 7 +- docs/platforms/theta.rst | 238 +++++++++++++++++++++++++++++++++------ 2 files changed, 209 insertions(+), 36 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 232e35f5e..5ff711620 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -6,7 +6,6 @@ Bebop_ is the newest addition to the computational power of LCRC at Argonne National Laboratory, featuring both Intel Broadwell and Knights Landing nodes. - Before Getting Started ---------------------- @@ -112,9 +111,9 @@ on Bebop becomes:: Debugging Strategies -------------------- -View the status of your submitted jobs with ``squeue``. +View the status of your submitted jobs with ``squeue`` and cancel jobs with ``scancel [jobID]``. -It's not recommended to debug compute-intensive tasks on the login nodes. Start +It's not recommended to debug compute-intensive tasks on the login nodes. Instead, start a bash session on a Knights Landing node for thirty minutes with:: srun --pty -A [username OR project] -p knl -t 00:30:00 /bin/bash @@ -127,7 +126,7 @@ a bash session on a Knights Landing node for thirty minutes with:: Additional Information ---------------------- -See the LCRC Bebop docs here_ for much more information about Bebop. +See the LCRC Bebop docs here_ for more information about Bebop. .. _Bebop: https://www.lcrc.anl.gov/systems/resources/bebop/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index e6a969e48..c87fd2280 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -3,53 +3,106 @@ Theta ===== Theta_ is a 11.69 petaflops system based on the second-generation Intel Xeon Phi -processor, available within ALCF at Argonne National Laboratory. +processor, available within ALCF_ at Argonne National Laboratory. - -Before getting started +Before Getting Started ---------------------- -An Argonne ALCF_ account is required to access Theta. Interested users will need +An Argonne ALCF account is required to access Theta. Interested users will need to apply for and be granted an account before continuing. To submit jobs to Theta, -users must charge their jobs to a project (likely specified while requesting an -account.) +users must charge their jobs to a project (specified while requesting an +account). -Theta consists of numerous types of nodes, but libEnsemble users will mostly interact +Theta contains numerous node-types, but libEnsemble users will mostly interact with login, Machine Oriented Mini-server (MOM) and compute nodes. MOM nodes execute -user batch scripts to run on the compute nodes. +user batch-scripts to run on the compute nodes. + +libEnsemble functions need to incorporate the libEnsemble +:ref:`job controller` to perform calculations on Theta. + Configuring Python ------------------ +Begin by loading the Python 3 Miniconda_ module:: + + $ module load miniconda-3.6/conda-4.5.12 + +Create a Conda_ virtual environment in which to install libEnsemble and all +dependencies:: + + $ conda config --add channels intel + $ conda create --name my_env intelpython3_core python=3 + $ source activate my_env + +More information_ on using Conda on Theta. Installing libEnsemble and Dependencies --------------------------------------- +libEnsemble and mpi4py +^^^^^^^^^^^^^^^^^^^^^^ + +There should be an indication that the virtual environment is activated. +Install mpi4py_ and libEnsemble in this environment, making sure to reference +the pre-installed Intel MPI Compiler. Your prompt should be similar to the +following block: + +.. code-block:: console + + (my_env) user@thetalogin6:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py + (my_env) user@thetalogin6:~$ pip install libensemble + + Balsam ^^^^^^ +Balsam_ is an ALCF Python utility for coordinating and executing workflows of +computations on systems like Theta. Balsam can stage in tasks from many sources, +including libEnsemble's job controller, and submit these tasks dynamically to the +compute nodes. If running with MPI, Balsam is necessary to launch libEnsemble on +Theta. + +Load the Balsam module with:: + + $ module load balsam/0.3.5.1 + +Balsam stages tasks in and out from a database. Initialize a new database similarly +to thefollowing: + +.. code-block:: bash + + $ balsam init ~/libe-workflow + $ source balsamactivate libe-workflow + $ balsam app --name libe-app --executable "calling.py" + $ balsam job --name libe-job --workflow test --application libe-app --args "hello!" + $ balsam submit-launch -A [project] -q default -t 5 -n 1 --job-mode=mpi + $ watch balsam ls # follow status in realtime from command-line + +See **Additional Information** for the Balsam docs. Job Submission -------------- -Theta features one default production queue, ``default``, and two debug queues, -``debug-cache-quad`` and ``debug-flat-quad``. +Theta uses Cobalt_ for job submission and management. For libEnsemble, the most +important command is ``qsub``, for submitting batch scripts from the login nodes. -.. note:: - For the default queue, the minimum number of nodes to allocate is 128! +On Theta, libEnsemble is commonly configured to run in one of two ways: -Theta uses Cobalt for job submission and management. The most important two -commands to run and manage jobs are ``qsub`` and ``aprun``, for submitting batch -scripts from the login nodes and launching jobs to the compute nodes respectively. + 1. Multiprocessing mode, with libEnsemble's MPI job controller taking + responsibility for direct submissions of jobs to compute nodes. -Because mpi4py doesn't function on the Theta MOM nodes, libEnsemble is commonly -run in multiprocessing mode with all workers running on the MOM nodes. In these -circumstances, libEnsemble's job controller takes responsibility for ``aprun`` -submissions of jobs to compute nodes. + 2. MPI mode, with libEnsemble's Balsam job controller interfacing with a Balsam + backend for dynamic job submission. + +Theta features one default production queue, ``default``, and two debug queues, +``debug-cache-quad`` and ``debug-flat-quad``. +.. note:: + For the default queue, the minimum number of nodes to allocate at once is 128 Interactive Runs @@ -58,12 +111,10 @@ Interactive Runs Users can run interactively with ``qsub`` by specifying the ``-I`` flag, similarly to the following:: - qsub -A [project] -n 128 -q default -t 120 -I + $ qsub -A [project] -n 128 -q default -t 120 -I -This will place the user on a MOM node. To launch the job to the allocated compute nodes, -run:: - - aprun... +This will place the user on a MOM node. If running in multiprocessing mode, launching +jobs to the compute nodes is as simple as ``python calling_script.py`` .. note:: You will need to re-activate your conda virtual environment and reload your @@ -73,42 +124,165 @@ Batch Runs ^^^^^^^^^^ Batch scripts specify run-settings using ``#COBALT`` statements. A simple example -for a libEnsemble use-case resembles the following: +for a libEnsemble use-case may resemble the following: .. code-block:: bash - + #!/bin/bash -x + #COBALT -t 02:00:00 + #COBALT -n 128 + #COBALT -q default + #COBALT -A [project] + #COBALT -O libE-project + + module load miniconda-3.6/conda-4.5.12 + + # Name of calling script + export EXE=calling_script.py + + # Communication Method + export COMMS='--comms local' + + # Number of workers. + export NWORKERS='--nworkers 128' + + # Name of Conda environment + export CONDA_ENV_NAME=my_env + + # Activate Conda environment + export PYTHONNOUSERSITE=1 + source activate $CONDA_ENV_NAME + + # Conda location - theta specific + export PATH=/home/user/path/to/packages/:$PATH + export LD_LIBRARY_PATH=/home/user/path/to/packages/:$LD_LIBRARY_PATH + export PYTHONPATH=/home/user/path/to/env/packages:$PYTHONPATH + + # Required for python kills on Theta + export PMI_NO_FORK=1 + + python $EXE $COMMS $NWORKERS > out.txt 2>&1 + With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble on Theta becomes:: - qsub --mode script myscript.sh + $ qsub --mode script myscript.sh + + +Balsam Runs +^^^^^^^^^^^ + +Balsam runs are Batch runs, except Balsam is responsible for submitting libEnsemble +for execution. This is an example Balsam submission script: + +.. code-block:: bash + + #!/bin/bash -x + #COBALT -t 60 + #COBALT -O libE_test + #COBALT -n 128 + #COBALT -q default + ##COBALT -A [project] + + # Name of calling script + export EXE=calling_script.py + + # Number of workers. + export NUM_WORKERS=128 + + # Wall-clock for libE job (supplied to Balsam) + export LIBE_WALLCLOCK=45 + + # Name of working directory where Balsam places running jobs/output + export WORKFLOW_NAME=libe_workflow + + #Tell libE manager to stop workers, dump timing.dat and exit after time. + export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3)) + # Name of Conda environment + export CONDA_ENV_NAME=my_env -Running with Balsam -^^^^^^^^^^^^^^^^^^^ + # Conda location - theta specific + export PATH=/path/to/python/bin:$PATH + export LD_LIBRARY_PATH=~/path/to/conda/env/lib:$LD_LIBRARY_PATH + + #Ensure environment isolated + export PYTHONNOUSERSITE=1 + + # Required for python kills on Theta + export PMI_NO_FORK=1 + + # Activate conda environment + . activate $CONDA_ENV_NAME + + # Activate Balsam database + . balsamactivate default + + + # Currently need at least one DB connection per worker (for postgres). + if [[ $NUM_WORKERS -gt 128 ]] + then + #Add a margin + echo -e "max_connections=$(($NUM_WORKERS+10)) #Appended by submission script" >> $BALSAM_DB_PATH/balsamdb/postgresql.conf + fi + wait + + # Make sure no existing apps/jobs + balsam rm apps --all --force + balsam rm jobs --all --force + wait + sleep 3 + + # Add calling script to Balsam database as app and job. + THIS_DIR=$PWD + SCRIPT_BASENAME=${EXE%.*} + + balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" + + # Running libE on one node - one manager and upto 63 workers + balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes + + #Run job + balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 + + . balsamdeactivate + +See **Additional Information** for the Balsam docs. Debugging Strategies -------------------- +View the status of your submitted jobs with ``qstat -fu [user]``. + +It's not recommended to debug compute-intensive tasks on the login or MOM nodes, +and if running in MPI mode, may be impossible. Allocate nodes on the debug queues +for the best results. + Each of the two debug queues has sixteen nodes apiece. A user can use up to eight nodes at a time for a maximum of one hour. Allocate nodes on the debug queue interactively:: - qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I + $ qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I Additional Information ---------------------- -See the ALCF guides_ on XC40 systems for much more information about Theta. +See the ALCF guides_ on XC40 systems for more information about Theta. -Become more familiar with Balsam here_. +Read the documentation for Balsam here_. .. _ALCF: https://www.alcf.anl.gov/ .. _Theta: https://www.alcf.anl.gov/theta +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _Cobalt: https://www.alcf.anl.gov/cobalt-scheduler .. _guides: https://www.alcf.anl.gov/user-guides/computational-systems .. _here: https://balsam.readthedocs.io/en/latest/ +.. _Miniconda: https://docs.conda.io/en/latest/miniconda.html +.. _Conda: https://conda.io/en/latest/ +.. _information: https://www.alcf.anl.gov/user-guides/conda +.. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ From 86a3937964db8f35f270dd71c7a113fb32f0d287 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 22 Oct 2019 13:43:30 -0500 Subject: [PATCH 365/644] reorganize FAQ into categories --- docs/FAQ.rst | 126 ++++++++++++++++++++++++--------------------------- 1 file changed, 60 insertions(+), 66 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 6110e180f..8c089aa46 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -7,31 +7,9 @@ If you have any further questions, feel free to contact us through Support_ .. _Support: https://libensemble.readthedocs.io/en/latest/quickstart.html#support -Local Parallel Debugging ------------------------- - -**How can I debug specific libEnsemble processes?** - -This is most easily addressed when running libEnsemble locally. Try - - ``mpiexec -np [num processes] xterm -e 'python [calling script].py'`` - -to launch an xterm terminal window specific to each process. Mac users will -need to install xQuartz_. - -If running in ``local`` mode try using one of the ``ForkablePdb`` -routines in ``libensemble/util/forkpdb.py`` to set breakpoints and debug similarly -to ``pdb``. How well this works varies by system:: - - from libensemble.util.forkpdb import ForkablePdb - ForkablePdb().set_trace() - -.. _xQuartz: https://www.xquartz.org/ - - -AssertionError - idle workers ------------------------------ +Common Errors +------------- **What does "AssertionError: Should not wait for workers when all workers are idle." mean?** @@ -44,9 +22,6 @@ This may also occur with two processes if you are using a persistent generator. The generator will occupy the one worker, leaving none to run simulation functions. -Not enough processors per worker to honor arguments ---------------------------------------------------- - **I keep getting: "Not enough processors per worker to honor arguments." when using the job controller. Can I launch jobs anyway?** @@ -60,9 +35,6 @@ Note that the job_controller ``.launch()`` method has a parameter``hyperthreads` which will attempt to use all hyperthreads/SMT threads available if set to ``True`` -FileExistsError ---------------- - **FileExistsError: [Errno 17] File exists: './sim_worker1'** This can happen when libEnsemble tries to create sim directories that already exist, @@ -73,8 +45,25 @@ To create differently named sim directories, you can use the ``sim_dir_suffix`` option in :ref:`sim_specs`. -libEnsemble hangs when using mpi4py ------------------------------------ +**PETSc and MPI errors with "[unset]: write_line error; fd=-1 buf=:cmd=abort exitcode=59"** + +with ``python [test with PETSc].py --comms local --nworkers 4`` + +This error occurs on some platforms, including Travis CI, when using PETSc with libEnsemble +in ``local`` (multiprocessing) mode. We believe this is due to PETSc initializing MPI +before libEnsemble forks processes using multiprocessing. The recommended solution +is running libEnsemble in MPI mode. An alternative solution may be using a serial +build of PETSc. + +.. note:: + This error may depend on how multiprocessing handles an existing MPI + communicator in a particular platform. + + + +HPC Errors and Questions +------------------------ + **Why does libEnsemble hang on certain systems when running with MPI?** @@ -91,62 +80,67 @@ Add these two lines BEFORE ``from mpi4py import MPI``:: Also see https://software.intel.com/en-us/articles/python-mpi4py-on-intel-true-scale-and-omni-path-clusters -Messages not received correctly when using mpi4py -------------------------------------------------- +**can't open hfi unit: -1 (err=23)** +**[13] MPI startup(): tmi fabric is not available and fallback fabric is not enabled** -This may manifest with the following error: +This may occur on TMI when libEnsemble Python processes have been launched to a node and these, +in turn, launch jobs on the node; creating too many processes for the available contexts. Note that +while processes can share contexts, the system is confused by the fact that there are two +phases, first libEnsemble processes and then sub-processes to run user jobs. The solution is to +either reduce the number processes running or to specify a fallback fabric through environment +variables:: + + unset I_MPI_FABRICS + export I_MPI_FABRICS_LIST=tmi,tcp + export I_MPI_FALLBACK=1 + +Another alternative is to run libEnsemble in central mode, in which libEnsemble runs on dedicated +nodes, while launching all sub-jobs to other nodes. -**_pickle.UnpicklingError: invalid load key, '\x00'.** -or some similar variation. This has been observed with the OFA fabric. The solution +**What does "_pickle.UnpicklingError: invalid load key, '\x00'." indicate?** + +This has been observed with the OFA fabric, and usually indicates MPI messages +aren't being received correctly when using mpi4py. The solution is to either switch fabric or turn off matching probes. See the answer for "Why does libEnsemble hang on certain systems when running with MPI?" For more information see: https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe -PETSc and MPI errors --------------------- -**PETSc and MPI errors with "[unset]: write_line error; fd=-1 buf=:cmd=abort exitcode=59"** +libEnsemble-theory +------------------ -with ``python [test with PETSc].py --comms local --nworkers 4`` +**How can I debug specific libEnsemble processes?** -This error occurs on some platforms, including Travis CI, when using PETSc with libEnsemble -in ``local`` (multiprocessing) mode. We believe this is due to PETSc initializing MPI -before libEnsemble forks processes using multiprocessing. The recommended solution -is running libEnsemble in MPI mode. An alternative solution may be using a serial -build of PETSc. +This is most easily addressed when running libEnsemble locally. Try -.. note:: - This error may depend on how multiprocessing handles an existing MPI - communicator in a particular platform. + ``mpiexec -np [num processes] xterm -e 'python [calling script].py'`` +to launch an xterm terminal window specific to each process. Mac users will +need to install xQuartz_. -Running out of contexts in distributed mode on TMI fabric ---------------------------------------------------------- +If running in ``local`` mode try using one of the ``ForkablePdb`` +routines in ``libensemble/util/forkpdb.py`` to set breakpoints and debug similarly +to ``pdb``. How well this works varies by system:: -The error message may be similar to below: + from libensemble.util.forkpdb import ForkablePdb + ForkablePdb().set_trace() -**can't open hfi unit: -1 (err=23)** -**[13] MPI startup(): tmi fabric is not available and fallback fabric is not enabled** -This may occur on TMI when libEnsemble Python processes have been launched to a node and these, -in turn, launch jobs on the node; creating too many processes for the available contexts. Note that -while processes can share contexts, the system is confused by the fact that there are two -phases, first libEnsemble processes and then sub-processes to run user jobs. The solution is to -either reduce the number processes running or to specify a fallback fabric through environment -variables:: +.. _xQuartz: https://www.xquartz.org/ - unset I_MPI_FABRICS - export I_MPI_FABRICS_LIST=tmi,tcp - export I_MPI_FALLBACK=1 -Another alternative is to run libEnsemble in central mode, in which libEnsemble runs on dedicated -nodes, while launching all sub-jobs to other nodes. +**Can I use the MPI Job Controller when running libEnsemble with multiprocessing?** + +Actually, yes! The job controller type only determines how launched jobs communicate +with libEnsemble, and is independent of ``comms`` chosen for manager-worker +communications. + -macOS - Common Issues +macOS-specific Errors --------------------- **"Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"** From f5e04652025240919d2b9779c88f541b4c563a37 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 22 Oct 2019 13:57:49 -0500 Subject: [PATCH 366/644] additional question --- docs/FAQ.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 8c089aa46..7a5d66d46 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -11,6 +11,13 @@ If you have any further questions, feel free to contact us through Support_ Common Errors ------------- +**I keep getting "Manager only - must be at least one worker (2 MPI tasks)" when +running with multiprocessing and multiple workers specified.** + +If your calling script code was switched from MPI to multiprocessing, make sure that +libE_specs is populated with ``comms: local`` and ``nworkers: [num]``. + + **What does "AssertionError: Should not wait for workers when all workers are idle." mean?** From 24e469a5e8bc83d842a15de626e6f12c1aecf290 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 22 Oct 2019 15:00:25 -0500 Subject: [PATCH 367/644] adjust toctree depth --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index f3eaac54d..ffeed0f44 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,7 @@ libEnsemble is a library for managing ensemble-like collections of computations. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Running on: platforms/bebop From c7b5298607a70d610e2f72c12b0c130fe20ac58a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Oct 2019 15:05:09 -0500 Subject: [PATCH 368/644] Edits for Jeff --- docs/data_structures/history_array.rst | 2 +- docs/data_structures/work_dict.rst | 2 +- libensemble/mpi_resources.py | 10 +++++----- libensemble/tests/regression_tests/.gitignore | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 78a5f8f3c..38ede0a0e 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -19,7 +19,7 @@ Below are the protected fields used in ``H`` .. literalinclude:: ../../libensemble/libE_fields.py :start-at: libE_fields -.. seealso:: +.. seealso:: See example :doc:`sim_specs<./sim_specs>`, :doc:`gen_specs<./gen_specs>`, and :doc:`alloc_specs<./alloc_specs>`. diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index a7dfaaef7..fcf9a3398 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -25,6 +25,6 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. -.. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py +.. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py .. _start_persistent_local_opt_gens.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_persistent_local_opt_gens.py .. _test_6-hump_camel_with_different_nodes_uniform_sample.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py diff --git a/libensemble/mpi_resources.py b/libensemble/mpi_resources.py index be88ff7a6..3bf4a68f4 100644 --- a/libensemble/mpi_resources.py +++ b/libensemble/mpi_resources.py @@ -58,7 +58,7 @@ def get_resources(self, num_procs=None, num_nodes=None, user supplied config is valid, and fills in any missing config information (ie. num_procs/num_nodes/ranks_per_node) - User supplied config options are honoured, and an exception is + User supplied config options are honored, and an exception is raised if these are infeasible. """ @@ -97,22 +97,22 @@ def get_resources(self, num_procs=None, num_nodes=None, # Could just downgrade to those available with warning - for now error rassert(num_nodes <= local_node_count, - "Not enough nodes to honour arguments. " + "Not enough nodes to honor arguments. " "Requested {}. Only {} available". format(num_nodes, local_node_count)) rassert(ranks_per_node <= cores_avail_per_node, - "Not enough processors on a node to honour arguments. " + "Not enough processors on a node to honor arguments. " "Requested {}. Only {} available". format(ranks_per_node, cores_avail_per_node)) rassert(ranks_per_node <= cores_avail_per_node_per_worker, - "Not enough processors per worker to honour arguments. " + "Not enough processors per worker to honor arguments. " "Requested {}. Only {} available". format(ranks_per_node, cores_avail_per_node_per_worker)) rassert(num_procs <= (cores_avail_per_node * local_node_count), - "Not enough procs to honour arguments. " + "Not enough procs to honor arguments. " "Requested {}. Only {} available". format(num_procs, cores_avail_per_node*local_node_count)) diff --git a/libensemble/tests/regression_tests/.gitignore b/libensemble/tests/regression_tests/.gitignore index ef2668a41..7791bddc5 100644 --- a/libensemble/tests/regression_tests/.gitignore +++ b/libensemble/tests/regression_tests/.gitignore @@ -3,4 +3,3 @@ output/ *.pickle ensemble.log libE_stats.txt - From 7318eca2f73b891a1c6dfb78fe7d5b0a01424e80 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Oct 2019 15:15:07 -0500 Subject: [PATCH 369/644] Changing on line --- docs/FAQ.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 7a5d66d46..be0592545 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -101,8 +101,8 @@ variables:: export I_MPI_FABRICS_LIST=tmi,tcp export I_MPI_FALLBACK=1 -Another alternative is to run libEnsemble in central mode, in which libEnsemble runs on dedicated -nodes, while launching all sub-jobs to other nodes. +Alternatively, libEnsemble can be run in central mode where all workers run on dedicated +nodes, while launching all sub-jobs onto other nodes. **What does "_pickle.UnpicklingError: invalid load key, '\x00'." indicate?** From fce674b67053ddc95615ea1d1fb8c3a3c8f047b5 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Oct 2019 16:16:12 -0500 Subject: [PATCH 370/644] one typo --- docs/data_structures/work_dict.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index fcf9a3398..2863fbd6a 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -12,7 +12,7 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` Required keys : 'persis_info' [dict]: Any persistent info to be sent to worker 'i' 'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' - 'tag' [int]: 'EVAL_SIM_TAG'/'EVAL_GEN_TAG') if worker 'i' is to call sim/gen_func + 'tag' [int]: 'EVAL_SIM_TAG'/'EVAL_GEN_TAG' if worker 'i' is to call sim/gen_func 'libE_info' [dict]: Info sent to/from worker to help manager update the 'H' Optional keys are: From 43b161d9648b93f8bf50afeb16e34f0a1f5fc11c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 13:10:01 -0500 Subject: [PATCH 371/644] Starting restructuring specs From 507fdea5086d3efa514e61be0c8b9c6c4c36d6e8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 13:14:37 -0500 Subject: [PATCH 372/644] Moving all user sim_specs references to use sim_specs['user'] --- libensemble/sim_funcs/branin/branin_obj.py | 4 ++-- libensemble/sim_funcs/chwirut1.py | 4 ++-- libensemble/sim_funcs/job_control_hworld.py | 2 +- libensemble/sim_funcs/six_hump_camel.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/sim_funcs/branin/branin_obj.py b/libensemble/sim_funcs/branin/branin_obj.py index 315189b11..0a21384b8 100644 --- a/libensemble/sim_funcs/branin/branin_obj.py +++ b/libensemble/sim_funcs/branin/branin_obj.py @@ -23,7 +23,7 @@ def call_branin(H, persis_info, sim_specs, _): O['f'][i] = branin(x[0], x[1]) - if 'uniform_random_pause_ub' in sim_specs: - time.sleep(sim_specs['uniform_random_pause_ub']*np.random.uniform()) + if 'uniform_random_pause_ub' in sim_specs['user']: + time.sleep(sim_specs['user']['uniform_random_pause_ub']*np.random.uniform()) return O, persis_info diff --git a/libensemble/sim_funcs/chwirut1.py b/libensemble/sim_funcs/chwirut1.py index c55db701f..279962c35 100644 --- a/libensemble/sim_funcs/chwirut1.py +++ b/libensemble/sim_funcs/chwirut1.py @@ -272,14 +272,14 @@ def chwirut_eval(H, persis_info, sim_specs, _): for i, x in enumerate(H['x']): if 'obj_component' in H.dtype.names: - if 'component_nan_frequency' in sim_specs and np.random.uniform(0, 1) < sim_specs['component_nan_frequency']: + if 'component_nan_frequency' in sim_specs['user'] and np.random.uniform(0, 1) < sim_specs['user']['component_nan_frequency']: O['f_i'][i] = np.nan else: O['f_i'][i] = EvaluateFunction(x, H['obj_component'][i]) else: O['fvec'][i] = EvaluateFunction(x) - O['f'][i] = sim_specs['combine_component_func'](O['fvec'][i]) + O['f'][i] = sim_specs['user']['combine_component_func'](O['fvec'][i]) return O, persis_info diff --git a/libensemble/sim_funcs/job_control_hworld.py b/libensemble/sim_funcs/job_control_hworld.py index 7b772fb0d..84351d93b 100644 --- a/libensemble/sim_funcs/job_control_hworld.py +++ b/libensemble/sim_funcs/job_control_hworld.py @@ -65,7 +65,7 @@ def polling_loop(comm, jobctl, job, timeout_sec=3.0, delay=0.3): def job_control_hworld(H, persis_info, sim_specs, libE_specs): """ Test of launching and polling job and exiting on job finish""" jobctl = MPIJobController.controller - cores = sim_specs['cores'] + cores = sim_specs['user']['cores'] comm = libE_specs['comm'] args_for_sim = 'sleep 1' diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index 3f452df9e..eab355806 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -34,7 +34,7 @@ def six_hump_camel_with_different_ranks_and_nodes(H, persis_info, sim_specs, lib with open(machinefilename, 'w') as f: for rank in ranks_involved: - b = sim_specs['nodelist'][rank] + '\n' + b = sim_specs['user']['nodelist'][rank] + '\n' f.write(b*H['ranks_per_node'][i]) outfile_name = "outfile_" + machinefilename + ".txt" @@ -74,7 +74,7 @@ def six_hump_camel(H, persis_info, sim_specs, _): O['grad'][i] = six_hump_camel_grad(x) if 'pause_time' in sim_specs: - time.sleep(sim_specs['pause_time']) + time.sleep(sim_specs['user']['pause_time']) return O, persis_info @@ -92,7 +92,7 @@ def six_hump_camel_simple(x, persis_info, sim_specs, _): O['f'] = six_hump_camel_func(x[0][0]) if 'pause_time' in sim_specs: - time.sleep(sim_specs['pause_time']) + time.sleep(sim_specs['user']['pause_time']) return O, persis_info From 642e6cc0367e51406bdae588aa64bb38444e6feb Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 13:30:02 -0500 Subject: [PATCH 373/644] Moving all user gen_specs references to use gen_specs['user'] or dereferencing to user_specs = gen_specs['user'] --- libensemble/gen_funcs/aposmm.py | 156 ++++++++-------- libensemble/gen_funcs/persistent_aposmm.py | 172 +++++++++--------- .../gen_funcs/persistent_inverse_bayes.py | 8 +- .../gen_funcs/persistent_uniform_sampling.py | 6 +- libensemble/gen_funcs/sampling.py | 46 ++--- libensemble/gen_funcs/uniform_or_localopt.py | 26 +-- 6 files changed, 208 insertions(+), 206 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 806bc1447..aea8a5d25 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -58,7 +58,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): objective components for a given ``'x'``) When using libEnsemble to do individual objective component evaluations, - APOSMM will return ``gen_specs['components']`` copies of each point, but + APOSMM will return ``gen_specs['user']['components']`` copies of each point, but the component=0 entry of each point will only be considered when - deciding where to start a run, @@ -67,7 +67,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): - storing the combined objective function value - etc - Necessary quantities in ``gen_specs`` are: + Necessary quantities in ``gen_specs['user']`` are: - ``'lb' [n floats]``: Lower bound on search domain - ``'ub' [n floats]``: Upper bound on search domain @@ -77,7 +77,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): - ``'localopt_method' [str]``: Name of an NLopt, PETSc/TAO, or SciPy method (see 'advance_local_run' below for supported methods) - Optional ``gen_specs`` entries are: + Optional ``gen_specs['user']`` entries are: - ``'sample_points' [numpy array]``: Points to be sampled (original domain) - ``'combine_component_func' [func]``: Function to combine obj components @@ -100,7 +100,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): - ``'rk_const' [float]``: Multiplier in front of the r_k value - ``'max_active_runs' [int]``: Bound on number of runs APOSMM is advancing - And ``gen_specs`` convergence tolerances for NLopt, PETSc/TAO, SciPy + And ``gen_specs['user']`` convergence tolerances for NLopt, PETSc/TAO, SciPy - ``'fatol' [float]``: - ``'ftol_abs' [float]``: @@ -126,7 +126,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): (breaking ties arbitrarily). :Note: - ``gen_specs['combine_component_func']`` must be defined when there are + ``gen_specs['user']['combine_component_func']`` must be defined when there are multiple objective components. :Note: @@ -180,10 +180,10 @@ def aposmm_logic(H, persis_info, gen_specs, _): """ - n, n_s, c_flag, O, r_k, mu, nu = initialize_APOSMM(H, gen_specs) + n, n_s, c_flag, O, r_k, mu, nu = initialize_APOSMM(H, gen_specs['user']) # np.savez('H'+str(len(H)),H=H,gen_specs=gen_specs,persis_info=persis_info) - if n_s < gen_specs['initial_sample_size']: + if n_s < gen_specs['user']['initial_sample_size']: updated_inds = set() else: @@ -203,7 +203,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): persis_info['total_runs'] += 1 num_runs = len(persis_info['run_order']) - if 'max_active_runs' in gen_specs and gen_specs['max_active_runs'] < num_runs: + if 'max_active_runs' in gen_specs['user'] and gen_specs['user']['max_active_runs'] < num_runs: # Store run number and sim_id of the best point in each run run_vals = np.zeros((num_runs, 2), dtype=int) for i, run in enumerate(persis_info['run_order'].keys()): @@ -221,8 +221,8 @@ def aposmm_logic(H, persis_info, gen_specs, _): dist_to_better[i] = np.min(P[i, better]) # Take max_active_runs largest - k_sorted = np.argpartition(-dist_to_better, kth=gen_specs['max_active_runs']-1) - active_runs = set(run_vals[k_sorted[:gen_specs['max_active_runs']], 0].astype(int)) + k_sorted = np.argpartition(-dist_to_better, kth=gen_specs['user']['max_active_runs']-1) + active_runs = set(run_vals[k_sorted[:gen_specs['user']['max_active_runs']], 0].astype(int)) else: active_runs = set(persis_info['run_order'].keys()) @@ -233,7 +233,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): if not np.all(H['returned'][persis_info['run_order'][run]]): continue # Can't advance a run if all points aren't returned. - x_opt, exit_code, persis_info, sorted_run_inds, x_new = advance_local_run(H, gen_specs, c_flag, run, persis_info) + x_opt, exit_code, persis_info, sorted_run_inds, x_new = advance_local_run(H, gen_specs['user'], c_flag, run, persis_info) if np.isinf(x_new).all(): if exit_code == 0: @@ -271,15 +271,15 @@ def aposmm_logic(H, persis_info, gen_specs, _): persis_info['old_runs'][i] = old_run if len(H) == 0: - samples_needed = gen_specs['initial_sample_size'] - elif 'min_batch_size' in gen_specs: - samples_needed = gen_specs['min_batch_size']-len(O) + samples_needed = gen_specs['user']['initial_sample_size'] + elif 'min_batch_size' in gen_specs['user']: + samples_needed = gen_specs['user']['min_batch_size']-len(O) else: samples_needed = int(not bool(len(O))) # 1 if len(O)==0, 0 otherwise - if samples_needed > 0 and 'sample_points' in gen_specs: + if samples_needed > 0 and 'sample_points' in gen_specs['user']: v = np.sum(~H['local_pt']) # Number of sample points so far - sampled_points = gen_specs['sample_points'][v:v+samples_needed] + sampled_points = gen_specs['user']['sample_points'][v:v+samples_needed] on_cube = False # Assume points are on original domain, not unit cube if len(sampled_points): persis_info = add_to_O(O, sampled_points, H, gen_specs, @@ -308,10 +308,10 @@ def add_to_O(O, pts, H, gen_specs, c_flag, persis_info, local_flag=0, original_len_O = len(O) len_H = len(H) - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] if c_flag: - m = gen_specs['components'] + m = gen_specs['user']['components'] assert len_H % m == 0, "Number of points in len_H not congruent to 0 mod 'components'" pt_ids = np.sort(np.tile(np.arange((len_H+original_len_O)/m, (len_H+original_len_O)/m+len(pts)), (1, m))) @@ -345,7 +345,7 @@ def add_to_O(O, pts, H, gen_specs, c_flag, persis_info, local_flag=0, O['num_active_runs'][-num_pts] += 1 # O['priority'][-num_pts:] = 1 # O['priority'][-num_pts:] = np.random.uniform(0,1,num_pts) - if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: + if 'high_priority_to_best_localopt_runs' in gen_specs['user'] and gen_specs['user']['high_priority_to_best_localopt_runs']: O['priority'][-num_pts:] = -min(H['f'][persis_info['run_order'][run]]) # Give highest priority to run with lowest function value else: O['priority'][-num_pts:] = persis_info['rand_stream'].uniform(0, 1, num_pts) @@ -358,7 +358,7 @@ def add_to_O(O, pts, H, gen_specs, c_flag, persis_info, local_flag=0, else: # p_tmp = np.random.uniform(0,1,num_pts) # persis_info['rand_stream'].uniform(lb,ub,(1,n)) - if 'high_priority_to_best_localopt_runs' in gen_specs and gen_specs['high_priority_to_best_localopt_runs']: + if 'high_priority_to_best_localopt_runs' in gen_specs['user'] and gen_specs['user']['high_priority_to_best_localopt_runs']: p_tmp = -np.inf*np.ones(num_pts) else: p_tmp = persis_info['rand_stream'].uniform(0, 1, num_pts) @@ -384,7 +384,7 @@ def update_history_dist(H, n, gen_specs, c_flag): for v in np.unique(H['pt_id'][new_inds]): inds = H['pt_id'] == v H['f'][inds] = np.inf - H['f'][np.where(inds)[0][0]] = gen_specs['combine_component_func'](H['f_i'][inds]) + H['f'][np.where(inds)[0][0]] = gen_specs['user']['combine_component_func'](H['f_i'][inds]) p = np.logical_and.reduce((H['returned'], H['obj_component'] == 0, ~np.isnan(H['f']))) else: @@ -478,7 +478,7 @@ def update_history_optimal(x_opt, H, run_inds): H['num_active_runs'][run_inds] -= 1 -def advance_local_run(H, gen_specs, c_flag, run, persis_info): +def advance_local_run(H, user_specs, c_flag, run, persis_info): """ Moves a local optimization method one iteration forward. We currently do this by feeding all past evaluations from a run to the method and then @@ -487,52 +487,52 @@ def advance_local_run(H, gen_specs, c_flag, run, persis_info): while 1: sorted_run_inds = persis_info['run_order'][run] - advance_local_run.x_new = np.ones((1, len(gen_specs['ub'])))*np.inf + advance_local_run.x_new = np.ones((1, len(user_specs['ub'])))*np.inf advance_local_run.pt_in_run = 0 - if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', + if user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: - if gen_specs['localopt_method'] in ['LD_MMA']: + if user_specs['localopt_method'] in ['LD_MMA']: fields_to_pass = ['x_on_cube', 'f', 'grad'] else: fields_to_pass = ['x_on_cube', 'f'] try: - x_opt, exit_code = set_up_and_run_nlopt(H[fields_to_pass][sorted_run_inds], gen_specs) + x_opt, exit_code = set_up_and_run_nlopt(H[fields_to_pass][sorted_run_inds], user_specs) except Exception as e: x_opt = 0 exit_code = 0 display_exception(e) - elif gen_specs['localopt_method'] in ['pounders', 'blmvm']: + elif user_specs['localopt_method'] in ['pounders', 'blmvm']: if c_flag: - Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, gen_specs['components'])]) + Run_H_F = np.zeros(len(sorted_run_inds), dtype=[('fvec', float, user_specs['components'])]) for i, ind in enumerate(sorted_run_inds): a1 = H['pt_id'] == H['pt_id'][ind] Run_H_F['fvec'][i, :] = H['f_i'][a1] Run_H = merge_arrays([H[['x_on_cube']][sorted_run_inds], Run_H_F], flatten=True) else: - if gen_specs['localopt_method'] == 'pounders': + if user_specs['localopt_method'] == 'pounders': Run_H = H[['x_on_cube', 'fvec']][sorted_run_inds] else: Run_H = H[['x_on_cube', 'f', 'grad']][sorted_run_inds] try: - x_opt, exit_code = set_up_and_run_tao(Run_H, gen_specs) + x_opt, exit_code = set_up_and_run_tao(Run_H, user_specs) except Exception as e: x_opt = 0 exit_code = 0 display_exception(e) - elif gen_specs['localopt_method'] == 'scipy_COBYLA': + elif user_specs['localopt_method'] == 'scipy_COBYLA': fields_to_pass = ['x_on_cube', 'f'] try: - x_opt, exit_code = set_up_and_run_scipy_minimize(H[fields_to_pass][sorted_run_inds], gen_specs) + x_opt, exit_code = set_up_and_run_scipy_minimize(H[fields_to_pass][sorted_run_inds], user_specs) except Exception as e: x_opt = 0 exit_code = 0 @@ -552,7 +552,7 @@ def advance_local_run(H, gen_specs, c_flag, run, persis_info): return x_opt, exit_code, persis_info, sorted_run_inds, advance_local_run.x_new -def set_up_and_run_scipy_minimize(Run_H, gen_specs): +def set_up_and_run_scipy_minimize(Run_H, user_specs): """ Set up objective and runs scipy Declares the appropriate syntax for our special objective function to read @@ -571,14 +571,14 @@ def scipy_obj_fun(x, Run_H): cons = [] for factor in range(len(x0)): lo = {'type': 'ineq', - 'fun': lambda x, lb=gen_specs['lb'][factor], i=factor: x[i]-lb} + 'fun': lambda x, lb=user_specs['lb'][factor], i=factor: x[i]-lb} up = {'type': 'ineq', - 'fun': lambda x, ub=gen_specs['ub'][factor], i=factor: ub-x[i]} + 'fun': lambda x, ub=user_specs['ub'][factor], i=factor: ub-x[i]} cons.append(lo) cons.append(up) - method = gen_specs['localopt_method'][6:] - res = scipy_optimize.minimize(obj, x0, method=method, options={'maxiter': len(Run_H['x_on_cube'])+1, 'tol': gen_specs['tol']}) + method = user_specs['localopt_method'][6:] + res = scipy_optimize.minimize(obj, x0, method=method, options={'maxiter': len(Run_H['x_on_cube'])+1, 'tol': user_specs['tol']}) if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum exit_code = 0 @@ -591,27 +591,27 @@ def scipy_obj_fun(x, Run_H): return x_opt, exit_code -def set_up_and_run_nlopt(Run_H, gen_specs): +def set_up_and_run_nlopt(Run_H, user_specs): """ Set up objective and runs nlopt Declares the appropriate syntax for our special objective function to read through Run_H, sets the parameters and starting points for the run. """ - assert 'xtol_rel' or 'xtol_abs' or 'ftol_rel' or 'ftol_abs' in gen_specs, "NLopt can cycle if xtol_rel, xtol_abs, ftol_rel, or ftol_abs are not set" + assert 'xtol_rel' or 'xtol_abs' or 'ftol_rel' or 'ftol_abs' in user_specs, "NLopt can cycle if xtol_rel, xtol_abs, ftol_rel, or ftol_abs are not set" def nlopt_obj_fun(x, grad, Run_H): out = look_in_history(x, Run_H) - if gen_specs['localopt_method'] in ['LD_MMA']: + if user_specs['localopt_method'] in ['LD_MMA']: grad[:] = out[1] out = out[0] return out - n = len(gen_specs['ub']) + n = len(user_specs['ub']) - opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) + opt = nlopt.opt(getattr(nlopt, user_specs['localopt_method']), n) lb = np.zeros(n) ub = np.ones(n) @@ -623,21 +623,21 @@ def nlopt_obj_fun(x, grad, Run_H): dist_to_bound = min(min(ub-x0), min(x0-lb)) assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" - if 'dist_to_bound_multiple' in gen_specs: - opt.set_initial_step(dist_to_bound*gen_specs['dist_to_bound_multiple']) + if 'dist_to_bound_multiple' in user_specs: + opt.set_initial_step(dist_to_bound*user_specs['dist_to_bound_multiple']) else: opt.set_initial_step(dist_to_bound) opt.set_maxeval(len(Run_H)+1) # evaluate one more point opt.set_min_objective(lambda x, grad: nlopt_obj_fun(x, grad, Run_H)) - if 'xtol_rel' in gen_specs: - opt.set_xtol_rel(gen_specs['xtol_rel']) - if 'ftol_rel' in gen_specs: - opt.set_ftol_rel(gen_specs['ftol_rel']) - if 'xtol_abs' in gen_specs: - opt.set_xtol_abs(gen_specs['xtol_abs']) - if 'ftol_abs' in gen_specs: - opt.set_ftol_abs(gen_specs['ftol_abs']) + if 'xtol_rel' in user_specs: + opt.set_xtol_rel(user_specs['xtol_rel']) + if 'ftol_rel' in user_specs: + opt.set_ftol_rel(user_specs['ftol_rel']) + if 'xtol_abs' in user_specs: + opt.set_xtol_abs(user_specs['xtol_abs']) + if 'ftol_abs' in user_specs: + opt.set_ftol_abs(user_specs['ftol_abs']) x_opt = opt.optimize(x0) exit_code = opt.last_optimize_result() @@ -648,14 +648,14 @@ def nlopt_obj_fun(x, grad, Run_H): return x_opt, exit_code -def set_up_and_run_tao(Run_H, gen_specs): +def set_up_and_run_tao(Run_H, user_specs): """ Set up objective and runs PETSc on the comm_self communicator Declares the appropriate syntax for our special objective function to read through Run_H, sets the parameters and starting points for the run. """ tao_comm = MPI.COMM_SELF - n = len(gen_specs['ub']) + n = len(user_specs['ub']) def pounders_obj_func(tao, X, F, Run_H): F.array = look_in_history(X.array_r, Run_H, vector_return=True) @@ -676,14 +676,14 @@ def blmvm_obj_func(tao, X, G, Run_H): lb.array = 0*np.ones(n) ub.array = 1*np.ones(n) tao = PETSc.TAO().create(tao_comm) - tao.setType(gen_specs['localopt_method']) + tao.setType(user_specs['localopt_method']) - if gen_specs['localopt_method'] == 'pounders': + if user_specs['localopt_method'] == 'pounders': f = PETSc.Vec().create(tao_comm) f.setSizes(len(Run_H['fvec'][0])) f.setFromOptions() - delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + delta_0 = user_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) @@ -693,7 +693,7 @@ def blmvm_obj_func(tao, X, G, Run_H): else: tao.setSeparableObjective(lambda tao, x, f: pounders_obj_func(tao, x, f, Run_H), f) - elif gen_specs['localopt_method'] == 'blmvm': + elif user_specs['localopt_method'] == 'blmvm': g = PETSc.Vec().create(tao_comm) g.setSizes(n) g.setFromOptions() @@ -703,9 +703,9 @@ def blmvm_obj_func(tao, X, G, Run_H): PETSc.Options().setValue('-tao_max_funcs', str(len(Run_H)+1)) tao.setFromOptions() tao.setVariableBounds((lb, ub)) - # tao.setObjectiveTolerances(fatol=gen_specs['fatol'], frtol=gen_specs['frtol']) - # tao.setGradientTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) - tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + # tao.setObjectiveTolerances(fatol=user_specs['fatol'], frtol=user_specs['frtol']) + # tao.setGradientTolerances(grtol=user_specs['grtol'], gatol=user_specs['gatol']) + tao.setTolerances(grtol=user_specs['grtol'], gatol=user_specs['gatol']) tao.setInitial(x) tao.solve(x) @@ -716,9 +716,9 @@ def blmvm_obj_func(tao, X, G, Run_H): # print(tao.view()) # print(x_opt) - if gen_specs['localopt_method'] == 'pounders': + if user_specs['localopt_method'] == 'pounders': f.destroy() - if gen_specs['localopt_method'] == 'blmvm': + if user_specs['localopt_method'] == 'blmvm': g.destroy() lb.destroy() @@ -887,7 +887,7 @@ def calc_rk(n, n_s, rk_const, lhs_divisions=0): return r_k -def initialize_APOSMM(H, gen_specs): +def initialize_APOSMM(H, user_specs): """ Computes common values every time that APOSMM is reinvoked @@ -895,10 +895,10 @@ def initialize_APOSMM(H, gen_specs): ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ - n = len(gen_specs['ub']) + n = len(user_specs['ub']) - if 'single_component_at_a_time' in gen_specs and gen_specs['single_component_at_a_time']: - assert gen_specs['batch_mode'], ("Must be in batch mode when using " + if 'single_component_at_a_time' in user_specs and user_specs['single_component_at_a_time']: + assert user_specs['batch_mode'], ("Must be in batch mode when using " "'single_component_at_a_time'") c_flag = True else: @@ -908,32 +908,32 @@ def initialize_APOSMM(H, gen_specs): # Get the pt_id for non-nan, returned points pt_ids = H['pt_id'][np.logical_and(H['returned'], ~np.isnan(H['f_i']))] _, counts = np.unique(pt_ids, return_counts=True) - n_s = np.sum(counts == gen_specs['components']) + n_s = np.sum(counts == user_specs['components']) else: # Number of returned sampled points (excluding nans) n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['returned']))) # Rather than build up a large output, we will just make changes in the # given H, and then send back the rows corresponding to updated H entries. - Out = np.empty(0, dtype=gen_specs['out']) + Out = np.empty(0, dtype=user_specs['out']) - if 'rk_const' in gen_specs: - rk_c = gen_specs['rk_const'] + if 'rk_const' in user_specs: + rk_c = user_specs['rk_const'] else: rk_c = ((gamma(1+(n/2.0))*5.0)**(1.0/n))/sqrt(pi) - if 'lhs_divisions' in gen_specs: - ld = gen_specs['lhs_divisions'] + if 'lhs_divisions' in user_specs: + ld = user_specs['lhs_divisions'] else: ld = 0 - if 'mu' in gen_specs: - mu = gen_specs['mu'] + if 'mu' in user_specs: + mu = user_specs['mu'] else: mu = 1e-4 - if 'nu' in gen_specs: - nu = gen_specs['nu'] + if 'nu' in user_specs: + nu = user_specs['nu'] else: nu = 0 diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 9914ce761..13f9d8f40 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -67,7 +67,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): objective components for a given ``'x'``) When using libEnsemble to do individual objective component evaluations, - APOSMM will return ``gen_specs['components']`` copies of each point, but + APOSMM will return ``gen_specs['user']['components']`` copies of each point, but the component=0 entry of each point will only be considered when - deciding where to start a run, @@ -76,7 +76,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): - storing the combined objective function value - etc - Necessary quantities in ``gen_specs`` are: + Necessary quantities in ``gen_specs['user']`` are: - ``'lb' [n floats]``: Lower bound on search domain - ``'ub' [n floats]``: Upper bound on search domain @@ -86,7 +86,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): - ``'localopt_method' [str]``: Name of an NLopt, PETSc/TAO, or SciPy method (see 'advance_local_run' below for supported methods) - Optional ``gen_specs`` entries are: + Optional ``gen_specs['user']`` entries are: - ``'sample_points' [numpy array]``: Points to be sampled (original domain) - ``'combine_component_func' [func]``: Function to combine obj components @@ -107,7 +107,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): - ``'rk_const' [float]``: Multiplier in front of the r_k value - ``'max_active_runs' [int]``: Bound on number of runs APOSMM is advancing - And ``gen_specs`` convergence tolerances for NLopt, PETSc/TAO, SciPy + And ``gen_specs['user']`` convergence tolerances for NLopt, PETSc/TAO, SciPy - ``'fatol' [float]``: - ``'ftol_abs' [float]``: @@ -133,7 +133,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): (breaking ties arbitrarily). :Note: - ``gen_specs['combine_component_func']`` must be defined when there are + ``gen_specs['user']['combine_component_func']`` must be defined when there are multiple objective components. :Note: @@ -183,29 +183,31 @@ def aposmm(H, persis_info, gen_specs, libE_info): persis_info['old_runs']: Sequence of indices of points in finished runs """ - n, n_s, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, gen_specs, libE_info) + user_specs = gen_specs['user'] + + n, n_s, rk_const, ld, mu, nu, comm, local_H = initialize_APOSMM(H, user_specs, libE_info) # Initialize stuff for localopt children local_opters = {} sim_id_to_child_indices = {} run_order = {} total_runs = 0 - if gen_specs['localopt_method'] in ['LD_MMA', 'blmvm']: + if user_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['x_on_cube', 'f', 'grad'] - elif gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', + elif user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'scipy_Nelder-Mead']: fields_to_pass = ['x_on_cube', 'f'] - elif gen_specs['localopt_method'] in ['pounders']: + elif user_specs['localopt_method'] in ['pounders']: fields_to_pass = ['x_on_cube', 'fvec'] else: - raise NotImplementedError("Unknown local optimization method " "'{}'.".format(gen_specs['localopt_method'])) + raise NotImplementedError("Unknown local optimization method " "'{}'.".format(user_specs['localopt_method'])) # Send our initial sample. We don't need to check that n_s is large enough: # the alloc_func only returns when the initial sample has function values. - persis_info = add_k_sample_points_to_local_H(gen_specs['initial_sample_size'], gen_specs, + persis_info = add_k_sample_points_to_local_H(user_specs['initial_sample_size'], user_specs, persis_info, n, comm, local_H, sim_id_to_child_indices) - send_mgr_worker_msg(comm, local_H[:gen_specs['initial_sample_size']][[i[0] for i in gen_specs['out']]]) + send_mgr_worker_msg(comm, local_H[:user_specs['initial_sample_size']][[i[0] for i in gen_specs['out']]]) tag = None while 1: @@ -215,7 +217,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): clean_up_and_stop(local_H, local_opters, run_order) break - n_s = update_local_H_after_receiving(local_H, n, n_s, gen_specs, Work, calc_in) + n_s = update_local_H_after_receiving(local_H, n, n_s, user_specs, Work, calc_in) new_opt_inds_to_send_mgr = [] new_inds_to_send_mgr = [] @@ -230,7 +232,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): new_opt_inds_to_send_mgr.append(opt_ind) local_opters.pop(child_idx) else: - add_to_local_H(local_H, x_new, gen_specs, local_flag=1, on_cube=True) + add_to_local_H(local_H, x_new, user_specs, local_flag=1, on_cube=True) new_inds_to_send_mgr.append(len(local_H)-1) run_order[child_idx].append(local_H[-1]['sim_id']) @@ -242,11 +244,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): starting_inds = decide_where_to_start_localopt(local_H, n, n_s, rk_const, ld, mu, nu) for ind in starting_inds: - if len([p for p in local_opters.values() if p.is_running]) < gen_specs.get('max_active_runs', np.inf): + if len([p for p in local_opters.values() if p.is_running]) < user_specs.get('max_active_runs', np.inf): local_H['started_run'][ind] = 1 # Initialize a local opt run - local_opter = LocalOptInterfacer(gen_specs, local_H[ind]['x_on_cube'], + local_opter = LocalOptInterfacer(user_specs, local_H[ind]['x_on_cube'], local_H[ind]['f'] if 'f' in fields_to_pass else local_H[ind]['fvec'], local_H[ind]['grad'] if 'grad' in fields_to_pass else None) @@ -254,7 +256,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): x_new = local_opter.iterate(local_H[ind][fields_to_pass]) # Assuming the second point can't be ruled optimal - add_to_local_H(local_H, x_new, gen_specs, local_flag=1, on_cube=True) + add_to_local_H(local_H, x_new, user_specs, local_flag=1, on_cube=True) new_inds_to_send_mgr.append(len(local_H)-1) run_order[total_runs] = [ind, local_H[-1]['sim_id']] @@ -267,7 +269,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): total_runs += 1 if len(new_inds_to_send_mgr) == 0: - persis_info = add_k_sample_points_to_local_H(1, gen_specs, persis_info, n, + persis_info = add_k_sample_points_to_local_H(1, user_specs, persis_info, n, comm, local_H, sim_id_to_child_indices) new_inds_to_send_mgr.append(len(local_H)-1) @@ -277,7 +279,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): class LocalOptInterfacer(object): - def __init__(self, gen_specs, x0, f0, grad0=None): + def __init__(self, user_specs, x0, f0, grad0=None): """ :param x0: A numpy array of the initial guess solution. This guess should be scaled to a unit cube. @@ -303,17 +305,17 @@ def __init__(self, gen_specs, x0, f0, grad0=None): # {{{ setting the local optimization method - if gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: + if user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA']: run_local_opt = run_local_nlopt - elif gen_specs['localopt_method'] in ['pounders', 'blmvm']: + elif user_specs['localopt_method'] in ['pounders', 'blmvm']: run_local_opt = run_local_tao - elif gen_specs['localopt_method'] in ['scipy_Nelder-Mead']: + elif user_specs['localopt_method'] in ['scipy_Nelder-Mead']: run_local_opt = run_local_scipy_opt # }}} self.parent_can_read.clear() - self.process = Process(target=run_local_opt, args=(gen_specs, + self.process = Process(target=run_local_opt, args=(user_specs, self.comm_queue, x0, f0, self.child_can_read, self.parent_can_read)) @@ -375,15 +377,15 @@ def destroy(self, previous_x): # {{{ NLOPT for local opt -def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, gen_specs): +def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, user_specs): comm_queue.put(x) parent_can_read.set() child_can_read.wait() - if gen_specs['localopt_method'] in ['LD_MMA']: + if user_specs['localopt_method'] in ['LD_MMA']: x_recv, f_recv, grad_recv = comm_queue.get() grad[:] = grad_recv else: - assert gen_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', + assert user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] x_recv, f_recv = comm_queue.get() @@ -394,11 +396,11 @@ def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, gen return f_recv -def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): +def run_local_nlopt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read): # print('[Child]: Started local opt at {}.'.format(x0), flush=True) - n = len(gen_specs['ub']) + n = len(user_specs['ub']) - opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) + opt = nlopt.opt(getattr(nlopt, user_specs['localopt_method']), n) lb = np.zeros(n) ub = np.ones(n) @@ -409,8 +411,8 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_re dist_to_bound = min(min(ub-x0), min(x0-lb)) assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" - if 'dist_to_bound_multiple' in gen_specs: - opt.set_initial_step(dist_to_bound*gen_specs['dist_to_bound_multiple']) + if 'dist_to_bound_multiple' in user_specs: + opt.set_initial_step(dist_to_bound*user_specs['dist_to_bound_multiple']) else: opt.set_initial_step(dist_to_bound) @@ -419,16 +421,16 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_re opt.set_min_objective(lambda x, grad: nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, - gen_specs)) + user_specs)) - if 'xtol_rel' in gen_specs: - opt.set_xtol_rel(gen_specs['xtol_rel']) - if 'ftol_rel' in gen_specs: - opt.set_ftol_rel(gen_specs['ftol_rel']) - if 'xtol_abs' in gen_specs: - opt.set_xtol_abs(gen_specs['xtol_abs']) - if 'ftol_abs' in gen_specs: - opt.set_ftol_abs(gen_specs['ftol_abs']) + if 'xtol_rel' in user_specs: + opt.set_xtol_rel(user_specs['xtol_rel']) + if 'ftol_rel' in user_specs: + opt.set_ftol_rel(user_specs['ftol_rel']) + if 'xtol_abs' in user_specs: + opt.set_xtol_abs(user_specs['xtol_abs']) + if 'ftol_abs' in user_specs: + opt.set_ftol_abs(user_specs['ftol_abs']) # FIXME: Do we need to do something of the final 'x_opt'? # print('[Child]: Started my optimization', flush=True) @@ -442,7 +444,7 @@ def run_local_nlopt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_re # {{{ SciPy optimization -def scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, gen_specs): +def scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, user_specs): comm_queue.put(x) # print('[Child]: Parent should no longer wait.', flush=True) parent_can_read.set() @@ -456,23 +458,23 @@ def scipy_callback_fun(x, comm_queue, child_can_read, parent_can_read, gen_specs return f_x_recv -def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): +def run_local_scipy_opt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read): # Construct the bounds in the form of constraints cons = [] for factor in range(len(x0)): lo = {'type': 'ineq', - 'fun': lambda x, lb=gen_specs['lb'][factor], i=factor: x[i]-lb} + 'fun': lambda x, lb=user_specs['lb'][factor], i=factor: x[i]-lb} up = {'type': 'ineq', - 'fun': lambda x, ub=gen_specs['ub'][factor], i=factor: ub-x[i]} + 'fun': lambda x, ub=user_specs['ub'][factor], i=factor: ub-x[i]} cons.append(lo) cons.append(up) - method = gen_specs['localopt_method'][6:] + method = user_specs['localopt_method'][6:] # print('[Child]: Started my optimization', flush=True) res = sp_opt.minimize(lambda x: scipy_callback_fun(x, comm_queue, - child_can_read, parent_can_read, gen_specs), x0, - method=method, options={'maxiter': 10, 'fatol': gen_specs['fatol'], 'xatol': gen_specs['xatol']}) + child_can_read, parent_can_read, user_specs), x0, + method=method, options={'maxiter': 10, 'fatol': user_specs['fatol'], 'xatol': user_specs['xatol']}) # if res['status'] == 2: # SciPy code for exhausting budget of evaluations, so not at a minimum # exit_code = 0 @@ -495,7 +497,7 @@ def run_local_scipy_opt(gen_specs, comm_queue, x0, f0, child_can_read, parent_ca # {{{ TAO routines for local opt -def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs): +def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, user_specs): comm_queue.put(x.array_r) # print('[Child]: I just put x_on_cube =', x.array, flush=True) # print('[Child]: Parent should no longer wait.', flush=True) @@ -512,7 +514,7 @@ def tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen return f -def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, gen_specs): +def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, user_specs): comm_queue.put(x.array_r) # print('[Child]: I just put x_on_cube =', x.array, flush=True) @@ -530,7 +532,7 @@ def tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read return f_recv -def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read): +def run_local_tao(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read): assert isinstance(x0, np.ndarray) @@ -551,25 +553,25 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read lb.array = 0*np.ones(n) ub.array = 1*np.ones(n) tao = PETSc.TAO().create(tao_comm) - tao.setType(gen_specs['localopt_method']) + tao.setType(user_specs['localopt_method']) - if gen_specs['localopt_method'] == 'pounders': + if user_specs['localopt_method'] == 'pounders': f = PETSc.Vec().create(tao_comm) f.setSizes(m) f.setFromOptions() if hasattr(tao, 'setResidual'): - tao.setResidual(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) + tao.setResidual(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, user_specs), f) else: - tao.setSeparableObjective(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, gen_specs), f) + tao.setSeparableObjective(lambda tao, x, f: tao_callback_fun(tao, x, f, comm_queue, child_can_read, parent_can_read, user_specs), f) - elif gen_specs['localopt_method'] == 'blmvm': + elif user_specs['localopt_method'] == 'blmvm': g = PETSc.Vec().create(tao_comm) g.setSizes(n) g.setFromOptions() - tao.setObjectiveGradient(lambda tao, x, g: tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, gen_specs)) + tao.setObjectiveGradient(lambda tao, x, g: tao_callback_fun_grad(tao, x, g, comm_queue, child_can_read, parent_can_read, user_specs)) - delta_0 = gen_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) + delta_0 = user_specs['dist_to_bound_multiple']*np.min([np.min(ub.array-x.array), np.min(x.array-lb.array)]) PETSc.Options().setValue('-tao_pounders_delta', str(delta_0)) # Set everything for tao before solving @@ -578,9 +580,9 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read PETSc.Options().setValue('-tao_max_funcs', '100') tao.setFromOptions() tao.setVariableBounds((lb, ub)) - # tao.setObjectiveTolerances(fatol=gen_specs['fatol'], frtol=gen_specs['frtol']) - # tao.setGradientTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) - tao.setTolerances(grtol=gen_specs['grtol'], gatol=gen_specs['gatol']) + # tao.setObjectiveTolerances(fatol=user_specs['fatol'], frtol=user_specs['frtol']) + # tao.setGradientTolerances(grtol=user_specs['grtol'], gatol=user_specs['gatol']) + tao.setTolerances(grtol=user_specs['grtol'], gatol=user_specs['gatol']) tao.setInitial(x) # print('[Child]: Started my optimization', flush=True) @@ -594,9 +596,9 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read # print(tao.view()) # print(x_opt) - if gen_specs['localopt_method'] == 'pounders': + if user_specs['localopt_method'] == 'pounders': f.destroy() - elif gen_specs['localopt_method'] == 'blmvm': + elif user_specs['localopt_method'] == 'blmvm': g.destroy() lb.destroy() @@ -614,7 +616,7 @@ def run_local_tao(gen_specs, comm_queue, x0, f0, child_can_read, parent_can_read # }}} -def update_local_H_after_receiving(local_H, n, n_s, gen_specs, Work, calc_in): +def update_local_H_after_receiving(local_H, n, n_s, user_specs, Work, calc_in): for name in calc_in.dtype.names: local_H[name][Work['libE_info']['H_rows']] = calc_in[name] @@ -623,12 +625,12 @@ def update_local_H_after_receiving(local_H, n, n_s, gen_specs, Work, calc_in): n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) # dist -> distance - update_history_dist(local_H, n, gen_specs) + update_history_dist(local_H, n) return n_s -def add_to_local_H(local_H, pts, gen_specs, local_flag=0, sorted_run_inds=[], run=[], on_cube=True): +def add_to_local_H(local_H, pts, user_specs, local_flag=0, sorted_run_inds=[], run=[], on_cube=True): """ Adds points to O, the numpy structured array to be sent back to the manager """ @@ -636,8 +638,8 @@ def add_to_local_H(local_H, pts, gen_specs, local_flag=0, sorted_run_inds=[], ru len_local_H = len(local_H) - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = user_specs['ub'] + lb = user_specs['lb'] num_pts = len(pts) @@ -665,7 +667,7 @@ def add_to_local_H(local_H, pts, gen_specs, local_flag=0, sorted_run_inds=[], ru local_H['priority'][-num_pts:] = 1 -def update_history_dist(H, n, gen_specs): +def update_history_dist(H, n): """ Updates distances/indices after new points that have been evaluated. @@ -889,34 +891,34 @@ def calc_rk(n, n_s, rk_const, lhs_divisions=0): return r_k -def initialize_APOSMM(H, gen_specs, libE_info): +def initialize_APOSMM(H, user_specs, libE_info): """ Computes common values every time that APOSMM is reinvoked :See: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ - n = len(gen_specs['ub']) + n = len(user_specs['ub']) n_s = 0 - if 'rk_const' in gen_specs: - rk_c = gen_specs['rk_const'] + if 'rk_const' in user_specs: + rk_c = user_specs['rk_const'] else: rk_c = ((gamma(1+(n/2.0))*5.0)**(1.0/n))/sqrt(pi) - if 'lhs_divisions' in gen_specs: - ld = gen_specs['lhs_divisions'] + if 'lhs_divisions' in user_specs: + ld = user_specs['lhs_divisions'] else: ld = 0 - if 'mu' in gen_specs: - mu = gen_specs['mu'] + if 'mu' in user_specs: + mu = user_specs['mu'] else: mu = 1e-4 - if 'nu' in gen_specs: - nu = gen_specs['nu'] + if 'nu' in user_specs: + nu = user_specs['nu'] else: nu = 0 @@ -942,27 +944,27 @@ def initialize_APOSMM(H, gen_specs, libE_info): ('returned', bool), ] - if 'components' in gen_specs: - local_H_fields += [('fvec', float, gen_specs['components'])] + if 'components' in user_specs: + local_H_fields += [('fvec', float, user_specs['components'])] local_H = np.empty(0, dtype=local_H_fields) return n, n_s, rk_c, ld, mu, nu, comm, local_H -def add_k_sample_points_to_local_H(k, gen_specs, persis_info, n, comm, local_H, sim_id_to_child_indices): +def add_k_sample_points_to_local_H(k, user_specs, persis_info, n, comm, local_H, sim_id_to_child_indices): - if 'sample_points' in gen_specs: + if 'sample_points' in user_specs: v = np.sum(~local_H['local_pt']) # Number of sample points so far - sampled_points = gen_specs['sample_points'][v:v+k] + sampled_points = user_specs['sample_points'][v:v+k] on_cube = False # Assume points are on original domain, not unit cube if len(sampled_points): - add_to_local_H(local_H, sampled_points, gen_specs, on_cube=on_cube) + add_to_local_H(local_H, sampled_points, user_specs, on_cube=on_cube) k = k-len(sampled_points) if k > 0: sampled_points = persis_info['rand_stream'].uniform(0, 1, (k, n)) - add_to_local_H(local_H, sampled_points, gen_specs, on_cube=True) + add_to_local_H(local_H, sampled_points, user_specs, on_cube=True) return persis_info diff --git a/libensemble/gen_funcs/persistent_inverse_bayes.py b/libensemble/gen_funcs/persistent_inverse_bayes.py index fafb67079..1e8e3096e 100644 --- a/libensemble/gen_funcs/persistent_inverse_bayes.py +++ b/libensemble/gen_funcs/persistent_inverse_bayes.py @@ -7,12 +7,12 @@ def persistent_updater_after_likelihood(H, persis_info, gen_specs, libE_info): """ """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) comm = libE_info['comm'] - subbatch_size = gen_specs['subbatch_size'] - num_subbatches = gen_specs['num_subbatches'] + subbatch_size = gen_specs['user']['subbatch_size'] + num_subbatches = gen_specs['user']['num_subbatches'] # Receive information from the manager (or a STOP_TAG) batch = -1 diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index 3d13916a9..ed51ec599 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -12,10 +12,10 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): :See: ``libensemble/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) - b = gen_specs['gen_batch_size'] + b = gen_specs['user']['gen_batch_size'] comm = libE_info['comm'] # Send batches until manager sends stop tag diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index 74431fc94..bc045edbe 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -8,19 +8,19 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_specs, _): """ - Generates points uniformly over the domain defined by ``gen_specs['ub']`` and - ``gen_specs['lb']``. Also randomly requests a different ``number_of_nodes`` + Generates points uniformly over the domain defined by ``gen_specs['user']['ub']`` and + ``gen_specs['user']['lb']``. Also randomly requests a different ``number_of_nodes`` and ``ranks_per_node`` to be used in the evaluation of the generated point. :See: ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) if len(H) == 0: - b = gen_specs['initial_batch_size'] + b = gen_specs['user']['initial_batch_size'] O = np.zeros(b, dtype=gen_specs['out']) for i in range(0, b): @@ -33,8 +33,8 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe else: O = np.zeros(1, dtype=gen_specs['out']) O['x'] = len(H)*np.ones(n) - O['num_nodes'] = np.random.randint(1, gen_specs['max_num_nodes']+1) - O['ranks_per_node'] = np.random.randint(1, gen_specs['max_ranks_per_node']+1) + O['num_nodes'] = np.random.randint(1, gen_specs['user']['max_num_nodes']+1) + O['ranks_per_node'] = np.random.randint(1, gen_specs['user']['max_ranks_per_node']+1) O['priority'] = 10*O['num_nodes'] return O, persis_info @@ -42,19 +42,19 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe def uniform_random_sample_obj_components(H, persis_info, gen_specs, _): """ - Generates points uniformly over the domain defined by ``gen_specs['ub']`` - and ``gen_specs['lb']`` but requests each ``obj_component`` be evaluated + Generates points uniformly over the domain defined by ``gen_specs['user']['ub']`` + and ``gen_specs['user']['lb']`` but requests each ``obj_component`` be evaluated separately. :See: ``libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py`` """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) - m = gen_specs['components'] - b = gen_specs['gen_batch_size'] + m = gen_specs['user']['components'] + b = gen_specs['user']['gen_batch_size'] O = np.zeros(b*m, dtype=gen_specs['out']) for i in range(0, b): @@ -71,17 +71,17 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs, _): def uniform_random_sample(H, persis_info, gen_specs, _): """ - Generates ``gen_specs['gen_batch_size']`` points uniformly over the domain - defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. + Generates ``gen_specs['user']['gen_batch_size']`` points uniformly over the domain + defined by ``gen_specs['user']['ub']`` and ``gen_specs['user']['lb']``. :See: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) - b = gen_specs['gen_batch_size'] + b = gen_specs['user']['gen_batch_size'] O = np.zeros(b, dtype=gen_specs['out']) @@ -92,15 +92,15 @@ def uniform_random_sample(H, persis_info, gen_specs, _): def latin_hypercube_sample(H, persis_info, gen_specs, _): """ - Generates ``gen_specs['gen_batch_size']`` in a Latin hypercube sample over - the domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. + Generates ``gen_specs['user']['gen_batch_size']`` in a Latin hypercube sample over + the domain defined by ``gen_specs['user']['ub']`` and ``gen_specs['user']['lb']``. """ - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) - b = gen_specs['gen_batch_size'] + b = gen_specs['user']['gen_batch_size'] O = np.zeros(b, dtype=gen_specs['out']) diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index 7c6950634..53342f15c 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -10,7 +10,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): """ - This generation function returns ``gen_specs['gen_batch_size']`` uniformly + This generation function returns ``gen_specs['user']['gen_batch_size']`` uniformly sampled points when called in nonpersistent mode (i.e., when ``libE_info['persistent']`` isn't ``True``). Otherwise, the generation function a persistent nlopt local optimization run. @@ -24,10 +24,10 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): O = [] return O, persis_info_updates, tag_out else: - ub = gen_specs['ub'] - lb = gen_specs['lb'] + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] n = len(lb) - b = gen_specs['gen_batch_size'] + b = gen_specs['user']['gen_batch_size'] O = np.zeros(b, dtype=gen_specs['out']) for i in range(0, b): @@ -50,43 +50,43 @@ def nlopt_obj_fun(x, grad): # Check if we can do an early return if np.array_equiv(x, H['x']): - if gen_specs['localopt_method'] in ['LD_MMA']: + if gen_specs['user']['localopt_method'] in ['LD_MMA']: grad[:] = H['grad'] return np.float(H['f']) # Send back x to the manager, then receive info or stop tag O = add_to_O(np.zeros(1, dtype=gen_specs['out']), x, 0, - gen_specs['ub'], gen_specs['lb'], local=True, active=True) + gen_specs['user']['ub'], gen_specs['user']['lb'], local=True, active=True) tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, O) if tag in [STOP_TAG, PERSIS_STOP]: nlopt.forced_stop.message = 'tag=' + str(tag) raise nlopt.forced_stop # Return function value (and maybe gradient) - if gen_specs['localopt_method'] in ['LD_MMA']: + if gen_specs['user']['localopt_method'] in ['LD_MMA']: grad[:] = calc_in['grad'] return float(calc_in['f']) # --------------------------------------------------------------------- x0 = H['x'].flatten() - lb = gen_specs['lb'] - ub = gen_specs['ub'] + lb = gen_specs['user']['lb'] + ub = gen_specs['user']['ub'] n = len(ub) - opt = nlopt.opt(getattr(nlopt, gen_specs['localopt_method']), n) + opt = nlopt.opt(getattr(nlopt, gen_specs['user']['localopt_method']), n) opt.set_lower_bounds(lb) opt.set_upper_bounds(ub) # Care must be taken with NLopt because a too-large initial step causes # nlopt to move the starting point! dist_to_bound = min(min(ub-x0), min(x0-lb)) - init_step = dist_to_bound*gen_specs.get('dist_to_bound_multiple', 1) + init_step = dist_to_bound*gen_specs['user'].get('dist_to_bound_multiple', 1) opt.set_initial_step(init_step) - opt.set_maxeval(gen_specs.get('localopt_maxeval', 100*n)) + opt.set_maxeval(gen_specs['user'].get('localopt_maxeval', 100*n)) opt.set_min_objective(nlopt_obj_fun) - opt.set_xtol_rel(gen_specs['xtol_rel']) + opt.set_xtol_rel(gen_specs['user']['xtol_rel']) # Run local optimization. Only send persis_info_updates back so new # information added to persis_info since this persistent instance started From b0d9a9944ed2c72ba94729ab23a80148790289cc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 14:20:45 -0500 Subject: [PATCH 374/644] Moving all user references in alloc_funcs to gen_specs['user'] or alloc_specs['user'] --- libensemble/alloc_funcs/fast_alloc.py | 4 ++-- .../alloc_funcs/fast_alloc_and_pausing.py | 20 +++++++++---------- .../alloc_funcs/fast_alloc_to_aposmm.py | 6 +++--- .../alloc_funcs/give_sim_work_first.py | 10 +++++----- .../alloc_funcs/inverse_bayes_allocf.py | 2 +- .../alloc_funcs/persistent_aposmm_alloc.py | 2 +- .../start_persistent_local_opt_gens.py | 4 ++-- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index 4919e9f98..dd7322a57 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -7,7 +7,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): to evaluate in the simulation function. The fields in ``sim_specs['in']`` are given. If all entries in `H` have been given a be evaluated, a worker is told to call the generator function, provided this wouldn't result in - more than ``gen_specs['num_active_gen']`` active generators. + more than ``gen_specs['user']['num_active_gen']`` active generators. :See: ``/libensemble/tests/regression_tests/test_fast_alloc.py`` @@ -23,7 +23,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): sim_work(Work, i, sim_specs['in'], [persis_info['next_to_give']], []) persis_info['next_to_give'] += 1 - elif gen_count < gen_specs.get('num_active_gens', gen_count+1): + elif gen_count < gen_specs['user'].get('num_active_gens', gen_count+1): # Give gen work persis_info['total_gen_calls'] += 1 diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 9b116fc61..f1cfea0d9 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -9,16 +9,16 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): to evaluate in the simulation function. The fields in ``sim_specs['in']`` are given. If all entries in `H` have been given a be evaluated, a worker is told to call the generator function, provided this wouldn't result in - more than ``gen_specs['num_active_gen']`` active generators. Also allows + more than ``gen_specs['user']['num_active_gen']`` active generators. Also allows for a 'batch_mode'. When there are multiple objective components, this allocation function does not evaluate further components for some point in the following scenarios: - alloc_specs['stop_on_NaNs']: True --- after a NaN has been found in returned in some + alloc_specs['user']['stop_on_NaNs']: True --- after a NaN has been found in returned in some objective component - allocated['stop_partial_fvec_eval']: True --- after the value returned from + alloc_specs['user']['stop_partial_fvec_eval']: True --- after the value returned from combine_component_func is larger than a known upper bound on the objective. :See: @@ -46,14 +46,14 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if len(persis_info['need_to_give']): # If 'stop_on_NaN' is true and any f_i is a NaN, then pause # evaluations of other f_i, corresponding to the same pt_id - if 'stop_on_NaNs' in alloc_specs and alloc_specs['stop_on_NaNs']: + if alloc_specs['user'].get('stop_on_NaNs'): pt_ids_to_pause.update(H['pt_id'][np.isnan(H['f_i'])]) # If 'stop_partial_fvec_eval' is true, pause entries in H if a # partial combine_component_func evaluation is # worse than the # best, known, complete evaluation (and the point is not a # local_pt). - if 'stop_partial_fvec_eval' in alloc_specs and alloc_specs['stop_partial_fvec_eval']: + if alloc_specs['user'].get(['stop_partial_fvec_eval']): pt_ids = set(persis_info['pt_ids']) - persis_info['has_nan'] - persis_info['complete'] pt_ids = np.array(list(pt_ids)) partial_fvals = np.zeros(len(pt_ids)) @@ -71,11 +71,11 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if np.all(H['returned'][a1]): persis_info['complete'].add(pt_id) - persis_info['best_complete_val'] = min(persis_info['best_complete_val'], gen_specs['combine_component_func'](H['f_i'][a1])) + persis_info['best_complete_val'] = min(persis_info['best_complete_val'], gen_specs['user']['combine_component_func'](H['f_i'][a1])) else: # Ensure combine_component_func calculates partial fevals correctly # with H['f_i'] = 0 for non-returned point - partial_fvals[j] = gen_specs['combine_component_func'](H['f_i'][a1]) + partial_fvals[j] = gen_specs['user']['combine_component_func'](H['f_i'][a1]) if len(persis_info['complete']) and len(pt_ids) > 1: @@ -102,13 +102,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): i, idle_workers = idle_workers[0], idle_workers[1:] sim_work(Work, i, sim_specs['in'], [next_row], []) - elif gen_count < gen_specs.get('num_active_gens', gen_count+1): + elif gen_count < gen_specs['user'].get('num_active_gens', gen_count+1): lw = persis_info['last_worker'] last_size = persis_info.get('last_size') if len(H): # Don't give gen instances in batch mode if points are unfinished - if (gen_specs.get('batch_mode') + if (gen_specs['user'].get('batch_mode') and not all(np.logical_or(H['returned'][last_size:], H['paused'][last_size:]))): break @@ -131,7 +131,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): persis_info['last_worker'] = i - elif gen_count >= gen_specs.get('num_active_gens', gen_count+1): + elif gen_count >= gen_specs['user'].get('num_active_gens', gen_count+1): idle_workers = [] return Work, persis_info diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index 47119e73d..8ee694c62 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -9,7 +9,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): to evaluate in the simulation function. The fields in ``sim_specs['in']`` are given. If all entries in `H` have been given a be evaluated, a worker is told to call the generator function, provided this wouldn't result in - more than ``gen_specs['num_active_gen']`` active generators. Also allows + more than ``gen_specs['user']['num_active_gen']`` active generators. Also allows for a 'batch_mode'. :See: @@ -27,13 +27,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): sim_work(Work, i, sim_specs['in'], [persis_info['next_to_give']], []) persis_info['next_to_give'] += 1 - elif gen_count < gen_specs.get('num_active_gens', gen_count+1): + elif gen_count < gen_specs['user'].get('num_active_gens', gen_count+1): lw = persis_info['last_worker'] last_size = persis_info.get('last_size') if len(H): # Don't give gen instances in batch mode if points are unfinished - if (gen_specs.get('batch_mode') + if (gen_specs['user'].get('batch_mode') and not all(np.logical_or(H['returned'][last_size:], H['paused'][last_size:]))): break diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 6b2f9d78b..90bbdd6d8 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -7,10 +7,10 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): """ Decide what should be given to workers. This allocation function gives any available simulation work first, and only when all simulations are - completed or running does it start (at most ``gen_specs['num_active_gens']``) + completed or running does it start (at most ``gen_specs['user']['num_active_gens']``) generator instances. - Allows for a ``gen_specs['batch_mode']`` where no generation + Allows for a ``gen_specs['user']['batch_mode']`` where no generation work is given out unless all entries in ``H`` are returned. Allows for ``blocking`` of workers that are not active, for example, so @@ -39,7 +39,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # Pick all high priority, oldest high priority, or just oldest point if 'priority' in H.dtype.fields: priorities = H['priority'][~H['allocated']] - if gen_specs.get('give_all_with_same_priority'): + if gen_specs['user'].get('give_all_with_same_priority'): q_inds = (priorities == np.max(priorities)) else: q_inds = np.argmax(priorities) @@ -68,12 +68,12 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): else: # Allow at most num_active_gens active generator instances - if gen_count >= gen_specs.get('num_active_gens', gen_count+1): + if gen_count >= gen_specs['user'].get('num_active_gens', gen_count+1): break # No gen instances in batch mode if workers still working still_working = ~H['returned'] - if gen_specs.get('batch_mode') and np.any(still_working): + if gen_specs['user'].get('batch_mode') and np.any(still_working): break # Give gen work diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index 668cc19a9..05b0133a4 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -32,7 +32,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe inds_to_send_back = np.where(np.logical_and(inds_generated_by_i, last_batch_inds))[0] if H['batch'][-1] > 0: - n = gen_specs['subbatch_size']*gen_specs['num_subbatches'] + n = gen_specs['user']['subbatch_size']*gen_specs['user']['num_subbatches'] k = H['batch'][-1] H['weight'][(n*(k-1)):(n*k)] = H['weight'][(n*k):(n*(k+1))] diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 3a3035e2d..df0586274 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -23,7 +23,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # If any persistent worker's calculated values have returned, give them back. for i in avail_worker_ids(W, persistent=True): - if persis_info.get('sample_done') or sum(H['returned']) >= gen_specs['initial_sample_size']: + if persis_info.get('sample_done') or sum(H['returned']) >= gen_specs['user']['initial_sample_size']: # Don't return if the initial sample is not complete persis_info['sample_done'] = True diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 3a0bbe0a3..13ef1931d 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -54,8 +54,8 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for i in avail_worker_ids(W, persistent=False): # Find candidates to start local opt runs if a sample has been evaluated if np.any(np.logical_and(~H['local_pt'], H['returned'])): - n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs) - update_history_dist(H, n, gen_specs, c_flag=False) + n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs['user']) + update_history_dist(H, n, gen_specs['user'], c_flag=False) starting_inds = decide_where_to_start_localopt(H, r_k, mu, nu) else: starting_inds = [] From 40abfb8fe72cdaea44109f1175ef3a8bf5d20919 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 16:12:02 -0500 Subject: [PATCH 375/644] Starting list of allowable keys to gen_, sim_, and alloc_specs --- libensemble/libE.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libensemble/libE.py b/libensemble/libE.py index 9b5cb0e3c..6a78051bd 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,6 +29,9 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) +allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'save_every_k', 'user'] +allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] +allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] def report_manager_exception(hist, persis_info, mgr_exc=None): "Write out exception manager exception to log." @@ -477,6 +480,9 @@ def check_alloc_specs(alloc_specs): assert isinstance(alloc_specs, dict), "alloc_specs must be a dictionary" assert alloc_specs['alloc_f'], "Allocation function must be specified" + for k in alloc_specs.keys(): + assert k in allowed_alloc_spec_keys, "Key %s is not allowed in alloc_specs. Supported keys are: %s " % (k,allowed_alloc_spec_keys) + def check_sim_specs(sim_specs): assert isinstance(sim_specs, dict), "sim_specs must be a dictionary" @@ -486,11 +492,17 @@ def check_sim_specs(sim_specs): assert len(sim_specs['out']), "sim_specs must have 'out' entries" assert isinstance(sim_specs['in'], list), "'in' field must exist and be a list of field names" + for k in sim_specs.keys(): + assert k in allowed_sim_spec_keys, "Key %s is not allowed in sim_specs. Supported keys are: %s " % (k,allowed_sim_spec_keys) + def check_gen_specs(gen_specs): assert isinstance(gen_specs, dict), "gen_specs must be a dictionary" assert not bool(gen_specs) or len(gen_specs['out']), "gen_specs must have 'out' entries" + for k in alloc_specs.keys(): + assert k in allowed_gen_spec_keys, "Key %s in gen_specs is not an allowable key" % (k,allowed_gen_spec_keys) + def check_exit_criteria(exit_criteria, sim_specs, gen_specs): assert isinstance(exit_criteria, dict), "exit_criteria must be a dictionary" From 8e105178471ca3897d0466b1a060737342ffd512 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 16:15:12 -0500 Subject: [PATCH 376/644] Fixing test of gen_spec keys --- libensemble/libE.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 6a78051bd..9abf50f65 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -500,8 +500,8 @@ def check_gen_specs(gen_specs): assert isinstance(gen_specs, dict), "gen_specs must be a dictionary" assert not bool(gen_specs) or len(gen_specs['out']), "gen_specs must have 'out' entries" - for k in alloc_specs.keys(): - assert k in allowed_gen_spec_keys, "Key %s in gen_specs is not an allowable key" % (k,allowed_gen_spec_keys) + for k in gen_specs.keys(): + assert k in allowed_gen_spec_keys, "Key %s is not allowed in gen_specs. Supported keys are: %s " % (k,allowed_gen_spec_keys) def check_exit_criteria(exit_criteria, sim_specs, gen_specs): From 9b3f9c310989c448b66892f5f5fdc53ef2fcf63b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 16:42:49 -0500 Subject: [PATCH 377/644] New calling scripts with 'user' fields --- .../alloc_funcs/fast_alloc_and_pausing.py | 2 +- .../start_persistent_local_opt_gens.py | 2 +- libensemble/gen_funcs/aposmm.py | 7 +-- libensemble/libE.py | 2 +- libensemble/sim_funcs/branin/branin_obj.py | 2 +- libensemble/sim_funcs/chwirut1.py | 2 +- libensemble/sim_funcs/six_hump_camel.py | 4 +- .../script_test_balsam_hworld.py | 8 +-- .../regression_tests/test_1d_sampling.py | 10 ++-- .../test_1d_uniform_sampling_with_comm_dup.py | 10 ++-- ...mp_camel_active_persistent_worker_abort.py | 20 ++++---- .../test_6-hump_camel_aposmm_LD_MMA.py | 50 ++++++++++--------- .../test_6-hump_camel_elapsed_time_abort.py | 14 +++--- .../test_6-hump_camel_persistent_aposmm_1.py | 23 +++++---- .../test_6-hump_camel_persistent_aposmm_2.py | 25 +++++----- .../test_6-hump_camel_persistent_aposmm_3.py | 25 +++++----- .../test_6-hump_camel_persistent_aposmm_4.py | 27 +++++----- ...-hump_camel_persistent_uniform_sampling.py | 7 +-- .../test_6-hump_camel_uniform_sampling.py | 7 +-- ..._sampling_with_persistent_localopt_gens.py | 17 ++++--- ...mel_with_different_nodes_uniform_sample.py | 20 ++++---- ...test_branin_aposmm_nlopt_and_then_scipy.py | 33 ++++++------ .../regression_tests/test_calc_exception.py | 8 +-- ...t_chwirut_aposmm_one_residual_at_a_time.py | 30 +++++------ .../regression_tests/test_chwirut_pounders.py | 22 ++++---- .../test_chwirut_pounders_persistent.py | 34 +++++++------ .../test_chwirut_pounders_splitcomm.py | 22 ++++---- .../test_chwirut_pounders_subcomm.py | 24 +++++---- ...uniform_sampling_one_residual_at_a_time.py | 29 ++++++----- .../tests/regression_tests/test_comms.py | 13 ++--- .../tests/regression_tests/test_fast_alloc.py | 17 ++++--- .../test_inverse_bayes_example.py | 15 +++--- .../test_jobcontroller_hworld.py | 17 ++++--- .../regression_tests/test_nan_func_aposmm.py | 17 ++++--- .../test_worker_exceptions.py | 11 ++-- 35 files changed, 311 insertions(+), 265 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index f1cfea0d9..d8e96d4cd 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -53,7 +53,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # partial combine_component_func evaluation is # worse than the # best, known, complete evaluation (and the point is not a # local_pt). - if alloc_specs['user'].get(['stop_partial_fvec_eval']): + if alloc_specs['user'].get('stop_partial_fvec_eval'): pt_ids = set(persis_info['pt_ids']) - persis_info['has_nan'] - persis_info['complete'] pt_ids = np.array(list(pt_ids)) partial_fvals = np.zeros(len(pt_ids)) diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 13ef1931d..a56b49906 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -54,7 +54,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for i in avail_worker_ids(W, persistent=False): # Find candidates to start local opt runs if a sample has been evaluated if np.any(np.logical_and(~H['local_pt'], H['returned'])): - n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs['user']) + n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs) update_history_dist(H, n, gen_specs['user'], c_flag=False) starting_inds = decide_where_to_start_localopt(H, r_k, mu, nu) else: diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index aea8a5d25..3fab1459b 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -180,7 +180,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): """ - n, n_s, c_flag, O, r_k, mu, nu = initialize_APOSMM(H, gen_specs['user']) + n, n_s, c_flag, O, r_k, mu, nu = initialize_APOSMM(H, gen_specs) # np.savez('H'+str(len(H)),H=H,gen_specs=gen_specs,persis_info=persis_info) if n_s < gen_specs['user']['initial_sample_size']: @@ -887,7 +887,7 @@ def calc_rk(n, n_s, rk_const, lhs_divisions=0): return r_k -def initialize_APOSMM(H, user_specs): +def initialize_APOSMM(H, gen_specs): """ Computes common values every time that APOSMM is reinvoked @@ -895,6 +895,7 @@ def initialize_APOSMM(H, user_specs): ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ + user_specs = gen_specs['user'] n = len(user_specs['ub']) if 'single_component_at_a_time' in user_specs and user_specs['single_component_at_a_time']: @@ -915,7 +916,7 @@ def initialize_APOSMM(H, user_specs): # Rather than build up a large output, we will just make changes in the # given H, and then send back the rows corresponding to updated H entries. - Out = np.empty(0, dtype=user_specs['out']) + Out = np.empty(0, dtype=gen_specs['out']) if 'rk_const' in user_specs: rk_c = user_specs['rk_const'] diff --git a/libensemble/libE.py b/libensemble/libE.py index 9abf50f65..9e9bc7989 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,7 +29,7 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'save_every_k', 'user'] +allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'clean_jobs', 'save_every_k', 'user'] allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] diff --git a/libensemble/sim_funcs/branin/branin_obj.py b/libensemble/sim_funcs/branin/branin_obj.py index 0a21384b8..00a801858 100644 --- a/libensemble/sim_funcs/branin/branin_obj.py +++ b/libensemble/sim_funcs/branin/branin_obj.py @@ -23,7 +23,7 @@ def call_branin(H, persis_info, sim_specs, _): O['f'][i] = branin(x[0], x[1]) - if 'uniform_random_pause_ub' in sim_specs['user']: + if 'user' in sim_specs and 'uniform_random_pause_ub' in sim_specs['user']: time.sleep(sim_specs['user']['uniform_random_pause_ub']*np.random.uniform()) return O, persis_info diff --git a/libensemble/sim_funcs/chwirut1.py b/libensemble/sim_funcs/chwirut1.py index 279962c35..59c205b33 100644 --- a/libensemble/sim_funcs/chwirut1.py +++ b/libensemble/sim_funcs/chwirut1.py @@ -272,7 +272,7 @@ def chwirut_eval(H, persis_info, sim_specs, _): for i, x in enumerate(H['x']): if 'obj_component' in H.dtype.names: - if 'component_nan_frequency' in sim_specs['user'] and np.random.uniform(0, 1) < sim_specs['user']['component_nan_frequency']: + if 'user' in sim_specs and 'component_nan_frequency' in sim_specs['user'] and np.random.uniform(0, 1) < sim_specs['user']['component_nan_frequency']: O['f_i'][i] = np.nan else: O['f_i'][i] = EvaluateFunction(x, H['obj_component'][i]) diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index eab355806..8848c638b 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -57,7 +57,7 @@ def six_hump_camel(H, persis_info, sim_specs, _): """ Evaluates the six hump camel function for a collection of points given in ``H['x']``. Additionally evaluates the gradient if ``'grad'`` is a field in - ``sim_specs['out']`` and pauses for ``sim_specs['pause_time']]`` if + ``sim_specs['out']`` and pauses for ``sim_specs['user']['pause_time']]`` if defined. :See: @@ -73,7 +73,7 @@ def six_hump_camel(H, persis_info, sim_specs, _): if 'grad' in O.dtype.names: O['grad'][i] = six_hump_camel_grad(x) - if 'pause_time' in sim_specs: + if 'user' in sim_specs and 'pause_time' in sim_specs['user']: time.sleep(sim_specs['user']['pause_time']) return O, persis_info diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index e38c51b93..02e201b00 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -42,17 +42,19 @@ def build_simfunc(): 'in': ['x'], 'out': [('f', float), ('cstat', int)], 'save_every_k': 400, - 'cores': cores_per_job} + 'user': {'cores': cores_per_job}} gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], + 'save_every_k': 20, + 'user': { 'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'gen_batch_size': nworkers, 'batch_mode': True, - 'num_active_gens': 1, - 'save_every_k': 20} + 'num_active_gens': 1} + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 5e7990f02..ea55ad0e3 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -27,10 +27,12 @@ gen_specs = {'gen_f': gen_f, 'out': [('x', float, (1,))], - 'lb': np.array([-3]), - 'ub': np.array([3]), - 'gen_batch_size': 500, - 'save_every_k': 300} + 'save_every_k': 300, + 'user': {'gen_batch_size': 500, + 'lb': np.array([-3]), + 'ub': np.array([3]), + } + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index 089b2d44c..dd5cc05a7 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -45,10 +45,12 @@ gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (1,))], - 'lb': np.array([-3]), - 'ub': np.array([3]), - 'gen_batch_size': 500, - 'save_every_k': 300} + 'save_every_k': 300, + 'user': {'lb': np.array([-3]), + 'ub': np.array([3]), + 'gen_batch_size': 500 + } + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index c8d9d5a33..40c9d639a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -32,16 +32,18 @@ gen_out += [('x', float, 2), ('x_on_cube', float, 2)] gen_specs = {'gen_f': gen_f, 'in': [], - 'localopt_method': 'LN_BOBYQA', - 'xtol_rel': 1e-4, 'out': gen_out, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': 2, - 'batch_mode': True, - 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.5, - 'localopt_maxeval': 4} + 'user': {'localopt_method': 'LN_BOBYQA', + 'xtol_rel': 1e-4, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'gen_batch_size': 2, + 'batch_mode': True, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.5, + 'localopt_maxeval': 4 + } + } alloc_specs = {'alloc_f': alloc_f, 'out': gen_out} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 4a88c1473..93c7ae864 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -36,17 +36,19 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f', 'grad', 'returned'], 'out': gen_out, - 'num_active_gens': 1, - 'batch_mode': True, - 'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), - 'localopt_method': 'LD_MMA', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'xtol_rel': 1e-3, - 'num_active_gens': 1, - 'max_active_runs': 6, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'num_active_gens': 1, + 'batch_mode': True, + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'LD_MMA', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'xtol_rel': 1e-3, + 'num_active_gens': 1, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]) + } + } alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)]} @@ -77,27 +79,27 @@ def libE_mpi_abort(): libE_specs['worker_cmd'].append(str(run)) if run == 1: - gen_specs['localopt_method'] = 'blmvm' - gen_specs['grtol'] = 1e-5 - gen_specs['gatol'] = 1e-5 + gen_specs['user']['localopt_method'] = 'blmvm' + gen_specs['user']['grtol'] = 1e-5 + gen_specs['user']['gatol'] = 1e-5 persis_info = deepcopy(persis_info_safe) if run == 2: - gen_specs['localopt_method'] = 'LD_MMA' + gen_specs['user']['localopt_method'] = 'LD_MMA' # Change the bounds to put a local min at a corner point (to test that # APOSMM handles the same point being in multiple runs) ability to # give back a previously evaluated point) - gen_specs['ub'] = np.array([-2.9, -1.9]) - gen_specs['mu'] = 1e-4 - gen_specs['rk_const'] = 0.01*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi) - gen_specs['lhs_divisions'] = 2 + gen_specs['user']['ub'] = np.array([-2.9, -1.9]) + gen_specs['user']['mu'] = 1e-4 + gen_specs['user']['rk_const'] = 0.01*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi) + gen_specs['user']['lhs_divisions'] = 2 # APOSMM can be called when some run is incomplete - gen_specs.pop('batch_mode') + gen_specs['user'].pop('batch_mode') - gen_specs.pop('xtol_rel') - gen_specs['ftol_rel'] = 1e-2 - gen_specs['xtol_abs'] = 1e-3 - gen_specs['ftol_abs'] = 1e-8 + gen_specs['user'].pop('xtol_rel') + gen_specs['user']['ftol_rel'] = 1e-2 + gen_specs['user']['xtol_abs'] = 1e-3 + gen_specs['user']['ftol_abs'] = 1e-8 exit_criteria = {'sim_max': 200, 'elapsed_wallclock_time': 300} minima = np.array([[-2.9, -1.9]]) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index 1e3cb393b..bf1b1cb52 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -27,16 +27,18 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)], - 'pause_time': 2} + 'user': {'pause_time': 2} + } gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], - 'gen_batch_size': 5, - 'num_active_gens': 1, - 'batch_mode': False, 'out': [('x', float, (2,))], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'gen_batch_size': 5, + 'num_active_gens': 1, + 'batch_mode': False, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 94a145662..e112e8688 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -46,17 +46,18 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, - 'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), - 'localopt_method': 'LD_MMA', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'xtol_rel': 1e-6, - 'ftol_rel': 1e-6, - 'num_active_gens': 1, - 'max_active_runs': 6, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'batch_mode': True, + 'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'LD_MMA', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'xtol_rel': 1e-6, + 'ftol_rel': 1e-6, + 'num_active_gens': 1, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index abe9d1dca..102c7782c 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -46,18 +46,19 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, - 'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), - 'localopt_method': 'LN_BOBYQA', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'xtol_abs': 1e-6, - 'ftol_abs': 1e-6, - 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.5, - 'max_active_runs': 6, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'LN_BOBYQA', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'xtol_abs': 1e-6, + 'ftol_abs': 1e-6, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, + 'batch_mode': True, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index a8f566eca..7fe9264c5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -46,18 +46,19 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, - 'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), - 'localopt_method': 'blmvm', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'grtol': 1e-4, - 'gatol': 1e-4, - 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.5, - 'max_active_runs': 6, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'initial_sample_size': 100, + 'sample_points': np.round(minima, 1), + 'batch_mode': True, + 'localopt_method': 'blmvm', + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'grtol': 1e-4, + 'gatol': 1e-4, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.5, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index f6e9bf0b3..288ce5980 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -45,19 +45,20 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, - 'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), - 'localopt_method': 'scipy_Nelder-Mead', - 'fatol': 1e-5, - 'xatol': 1e-5, - 'nu': 1e-8, - 'mu': 1e-8, - 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.01, - 'max_active_runs': 6, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'initial_sample_size': 100, + 'batch_mode': True, + 'sample_points': np.round(minima, 1), + 'localopt_method': 'scipy_Nelder-Mead', + 'fatol': 1e-5, + 'xatol': 1e-5, + 'nu': 1e-8, + 'mu': 1e-8, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.01, + 'max_active_runs': 6, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py index c0eda0790..769d0e10a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py @@ -36,10 +36,11 @@ gen_specs = {'gen_f': gen_f, 'in': [], - 'gen_batch_size': 20, 'out': [('x', float, (n,))], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index bd4e96352..e1c74b961 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -35,9 +35,10 @@ gen_specs = {'gen_f': gen_f, # Tell libE function generating sim_f input 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size 'save_every_k': 300, # Tell libE to save every 300 gen entries - 'gen_batch_size': 500, # Tell gen_f how much to generate per call - 'lb': np.array([-3, -2]), # Tell gen_f lower bounds - 'ub': np.array([3, 2]), # Tell gen_f upper bounds + 'user': {'gen_batch_size': 500, # Tell gen_f how much to generate per call + 'lb': np.array([-3, -2]), # Tell gen_f lower bounds + 'ub': np.array([3, 2]), # Tell gen_f upper bounds + } } # end_gen_specs_rst_tag diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index bcaa94c91..a9f8eea70 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -39,15 +39,16 @@ gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, 'in': [], - 'xtol_rel': 1e-4, 'out': gen_out, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': 2, - 'batch_mode': True, - 'num_active_gens': 1, - 'localopt_method': 'LD_MMA', - 'xtol_rel': 1e-4} + 'user': {'xtol_rel': 1e-4, + 'batch_mode': True, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'gen_batch_size': 2, + 'num_active_gens': 1, + 'localopt_method': 'LD_MMA', + 'xtol_rel': 1e-4} + } alloc_specs = {'alloc_f': alloc_f, 'out': gen_out} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index ac319ea8a..b787596d8 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -46,7 +46,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x', 'num_nodes', 'ranks_per_node'], 'out': [('f', float)], - 'nodelist': libE_machinefile} + 'user': {'nodelist': libE_machinefile} + } gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], @@ -55,14 +56,15 @@ ('ranks_per_node', int), ('x', float, n), ('x_on_cube', float, n)], - 'initial_batch_size': 5, - 'max_ranks_per_node': 8, - 'num_active_gens': 1, - 'batch_mode': False, - 'give_all_with_same_priority': True, - 'max_num_nodes': nworkers, # Used in uniform_random_sample_with_different_nodes_and_ranks, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'initial_batch_size': 5, + 'batch_mode': False, + 'max_ranks_per_node': 8, + 'num_active_gens': 1, + 'give_all_with_same_priority': True, + 'max_num_nodes': nworkers, # Used in uniform_random_sample_with_different_nodes_and_ranks, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index ef22e533e..dc6089255 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -40,24 +40,25 @@ # Otherwise, will just copy in same directory as sim_dir sim_specs['sim_dir_prefix'] = '~' elif nworkers == 3: - sim_specs['uniform_random_pause_ub'] = 0.05 + sim_specs['user'] = {'uniform_random_pause_ub': 0.05} n = 2 gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f', 'returned'], 'out': gen_out, - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': np.array([-5, 0]), - 'ub': np.array([10, 15]), - 'initial_sample_size': 20, - 'localopt_method': 'LN_BOBYQA', - 'dist_to_bound_multiple': 0.99, - 'xtol_rel': 1e-3, - 'min_batch_size': nworkers, - 'high_priority_to_best_localopt_runs': True, - 'max_active_runs': 3} + 'user': {'num_active_gens': 1, + 'batch_mode': True, + 'lb': np.array([-5, 0]), + 'ub': np.array([10, 15]), + 'initial_sample_size': 20, + 'localopt_method': 'LN_BOBYQA', + 'dist_to_bound_multiple': 0.99, + 'xtol_rel': 1e-3, + 'min_batch_size': nworkers, + 'high_priority_to_best_localopt_runs': True, + 'max_active_runs': 3} + } persis_info = per_worker_stream(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) @@ -71,9 +72,9 @@ # Perform the run for run in range(2): if run == 1: - gen_specs['localopt_method'] = 'scipy_COBYLA' - gen_specs.pop('xtol_rel') - gen_specs['tol'] = 1e-5 + gen_specs['user']['localopt_method'] = 'scipy_COBYLA' + gen_specs['user'].pop('xtol_rel') + gen_specs['user']['tol'] = 1e-5 exit_criteria['sim_max'] = 500 persis_info = deepcopy(persis_info_safe) @@ -89,5 +90,5 @@ print(dist) assert dist < tol - print("\nAPOSMM + " + gen_specs['localopt_method'] + " found " + str(k) + " minima to tolerance " + str(tol)) + print("\nAPOSMM + " + gen_specs['user']['localopt_method'] + " found " + str(k) + " minima to tolerance " + str(tol)) save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index 604779a63..58247bb9c 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -29,12 +29,14 @@ def six_hump_camel_err(H, persis_info, sim_specs, _): sim_specs = {'sim_f': six_hump_camel_err, 'in': ['x'], 'out': [('f', float)]} + gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, 2)], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': 10} + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'gen_batch_size': 10} + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py index 55bb6994c..3d0108126 100644 --- a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py @@ -42,25 +42,27 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f_i', 'returned'], 'out': gen_out, - 'initial_sample_size': 5, - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': LB, - 'ub': UB, - 'localopt_method': 'pounders', - 'dist_to_bound_multiple': 0.5, - 'single_component_at_a_time': True, - 'components': m, - 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + 'user': {'initial_sample_size': 5, + 'batch_mode': True, + 'num_active_gens': 1, + 'lb': LB, + 'ub': UB, + 'localopt_method': 'pounders', + 'dist_to_bound_multiple': 0.5, + 'single_component_at_a_time': True, + 'components': m, + 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + } -gen_specs.update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) +gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) np.random.RandomState(0) -gen_specs['sample_points'] = np.random.uniform(0, 1, (budget, n))*(UB-LB)+LB +gen_specs['user']['sample_points'] = np.random.uniform(0, 1, (budget, n))*(UB-LB)+LB alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)], - 'stop_on_NaNs': True, - 'stop_partial_fvec_eval': True} + 'user': {'stop_on_NaNs': True, + 'stop_partial_fvec_eval': True} + } persis_info = per_worker_stream(persis_info, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders.py b/libensemble/tests/regression_tests/test_chwirut_pounders.py index e65adba26..0cf677c9d 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders.py @@ -32,7 +32,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('fvec', float, m)], - 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + 'user': {'combine_component_func': lambda x: np.sum(np.power(x, 2))} + } gen_out += [('x', float, n), ('x_on_cube', float, n)] @@ -40,16 +41,17 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, - 'initial_sample_size': 5, - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': (-2-np.pi/10)*np.ones(n), - 'ub': 2*np.ones(n), - 'localopt_method': 'pounders', - 'dist_to_bound_multiple': 0.5, - 'components': m} + 'user': {'initial_sample_size': 5, + 'batch_mode': True, + 'num_active_gens': 1, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n), + 'localopt_method': 'pounders', + 'dist_to_bound_multiple': 0.5, + 'components': m} + } -gen_specs.update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) +gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) persis_info = per_worker_stream(persis_info, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index f1fbcb913..62873ea16 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -38,7 +38,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('fvec', float, m)], - 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + 'user': {'combine_component_func': lambda x: np.sum(np.power(x, 2))} + } gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), ('local_min', bool), ('local_pt', bool)] @@ -47,18 +48,19 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'batch_mode': True, - 'initial_sample_size': 100, - 'localopt_method': 'pounders', - 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), - 'grtol': 1e-6, - 'gatol': 1e-6, - 'num_active_gens': 1, - 'dist_to_bound_multiple': 0.5, - 'lhs_divisions': 50, - 'components': m, - 'lb': (-2-np.pi/10)*np.ones(n), - 'ub': 2*np.ones(n)} + 'user': {'initial_sample_size': 100, + 'localopt_method': 'pounders', + 'batch_mode': True, + 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), + 'grtol': 1e-6, + 'gatol': 1e-6, + 'num_active_gens': 1, + 'dist_to_bound_multiple': 0.5, + 'lhs_divisions': 50, + 'components': m, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n)} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} @@ -67,10 +69,10 @@ exit_criteria = {'sim_max': 500} sample_points = np.zeros((0, n)) -for i in range(ceil(exit_criteria['sim_max']/gen_specs['lhs_divisions'])): - sample_points = np.append(sample_points, lhs_sample(n, gen_specs['lhs_divisions']), axis=0) +for i in range(ceil(exit_criteria['sim_max']/gen_specs['user']['lhs_divisions'])): + sample_points = np.append(sample_points, lhs_sample(n, gen_specs['user']['lhs_divisions']), axis=0) -gen_specs['sample_points'] = sample_points*(gen_specs['ub']-gen_specs['lb']) + gen_specs['lb'] +gen_specs['user']['sample_points'] = sample_points*(gen_specs['user']['ub']-gen_specs['user']['lb']) + gen_specs['user']['lb'] # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 377d22bf8..db5861484 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -36,7 +36,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('fvec', float, m)], - 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + 'user': {'combine_component_func': lambda x: np.sum(np.power(x, 2))} + } gen_out += [('x', float, n), ('x_on_cube', float, n)] @@ -44,16 +45,17 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, - 'initial_sample_size': 5, - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': (-2-np.pi/10)*np.ones(n), - 'ub': 2*np.ones(n), - 'localopt_method': 'pounders', - 'dist_to_bound_multiple': 0.5, - 'components': m} + 'user': {'initial_sample_size': 5, + 'batch_mode': True, + 'num_active_gens': 1, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n), + 'localopt_method': 'pounders', + 'dist_to_bound_multiple': 0.5, + 'components': m} + } -gen_specs.update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) +gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) persis_info = per_worker_stream(persis_info, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 97350fcc0..860c0acd2 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -40,7 +40,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('fvec', float, m)], - 'combine_component_func': lambda x: np.sum(np.power(x, 2))} + 'user': {'combine_component_func': lambda x: np.sum(np.power(x, 2))} + } gen_out += [('x', float, n), ('x_on_cube', float, n)] @@ -48,16 +49,17 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, - 'initial_sample_size': 5, - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': (-2-np.pi/10)*np.ones(n), - 'ub': 2*np.ones(n), - 'localopt_method': 'pounders', - 'dist_to_bound_multiple': 0.5, - 'components': m} - -gen_specs.update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) + 'user': {'initial_sample_size': 5, + 'batch_mode': True, + 'num_active_gens': 1, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n), + 'localopt_method': 'pounders', + 'dist_to_bound_multiple': 0.5, + 'components': m} + } + +gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) persis_info = per_worker_stream(persis_info, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 559a9681a..aba1eb713 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -44,7 +44,8 @@ sim_specs = {'sim_f': sim_f, 'in': ['x', 'obj_component'], 'out': [('f_i', float)], - 'component_nan_frequency': 0.01} + 'user': {'component_nan_frequency': 0.01} + } # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, @@ -54,19 +55,21 @@ ('paused', bool), ('obj_component', int), ('pt_id', int)], - 'gen_batch_size': 2, - 'single_component_at_a_time': True, - 'combine_component_func': lambda x: np.sum(np.power(x, 2)), - 'num_active_gens': 1, - 'batch_mode': True, - 'lb': (-2-np.pi/10)*np.ones(n), - 'ub': 2*np.ones(n), - 'components': m} + 'user': {'gen_batch_size': 2, + 'batch_mode': True, + 'single_component_at_a_time': True, + 'combine_component_func': lambda x: np.sum(np.power(x, 2)), + 'num_active_gens': 1, + 'lb': (-2-np.pi/10)*np.ones(n), + 'ub': 2*np.ones(n), + 'components': m} + } alloc_specs = {'alloc_f': alloc_f, # Allocation function 'out': [('allocated', bool)], # Output fields (included in History) - 'stop_on_NaNs': True, # Should alloc_f preempt evals - 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals + 'user': {'stop_on_NaNs': True, # Should alloc_f preempt evals + 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals + } # end_alloc_specs_rst_tag persis_info = per_worker_stream(persis_info, nworkers + 1) @@ -82,7 +85,7 @@ save_libE_output(H, persis_info, __file__, nworkers) # Perform the run but not stopping on NaNs -alloc_specs.pop('stop_on_NaNs') +alloc_specs['user'].pop('stop_on_NaNs') persis_info = deepcopy(persis_info_safe) H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) @@ -90,7 +93,7 @@ assert flag == 0 # Perform the run also not stopping on partial fvec evals -alloc_specs.pop('stop_partial_fvec_eval') +alloc_specs['user'].pop('stop_partial_fvec_eval') persis_info = deepcopy(persis_info_safe) H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index d719fc6fe..f240621c6 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -37,12 +37,13 @@ gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': sim_max, - 'batch_mode': True, - 'num_active_gens': 1, - 'save_every_k': 300} + 'save_every_k': 300, + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'batch_mode': True, + 'gen_batch_size': sim_max, + 'num_active_gens': 1} + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 1741232c7..012039a29 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -27,15 +27,16 @@ num_pts = 30*(nworkers - 1) -sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} +sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)], 'user': {}} gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'gen_batch_size': num_pts, - 'num_active_gens': 1, - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2])} + 'user': {'gen_batch_size': num_pts, + 'num_active_gens': 1, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)]} @@ -51,11 +52,11 @@ for time in np.append([0], np.logspace(-5, -1, 5)): for rep in range(1): - sim_specs['pause_time'] = time + sim_specs['user']['pause_time'] = time if time == 0: - sim_specs.pop('pause_time') - gen_specs['gen_batch_size'] = num_pts//2 + sim_specs['user'].pop('pause_time') + gen_specs['user']['gen_batch_size'] = num_pts//2 persis_info['next_to_give'] = 0 persis_info['total_gen_calls'] = 1 diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index bcbec1a3d..a117a83f9 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -38,17 +38,18 @@ 'in': [], 'out': [('x', float, 2), ('batch', int), ('subbatch', int), ('prior', float), ('prop', float), ('weight', float)], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'subbatch_size': 3, - 'num_subbatches': 2, - 'num_batches': 10} + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'subbatch_size': 3, + 'num_subbatches': 2, + 'num_batches': 10} + } persis_info = per_worker_stream({}, nworkers + 1) # Tell libEnsemble when to stop exit_criteria = { - 'sim_max': gen_specs['subbatch_size']*gen_specs['num_subbatches']*gen_specs['num_batches'], + 'sim_max': gen_specs['user']['subbatch_size']*gen_specs['user']['num_subbatches']*gen_specs['user']['num_batches'], 'elapsed_wallclock_time': 300} alloc_specs = {'out': [], 'alloc_f': alloc_f} @@ -60,6 +61,6 @@ if is_master: assert flag == 0 # Change the last weights to correct values (H is a list on other cores and only array on manager) - ind = 2*gen_specs['subbatch_size']*gen_specs['num_subbatches'] + ind = 2*gen_specs['user']['subbatch_size']*gen_specs['user']['num_subbatches'] H[-ind:] = H['prior'][-ind:] + H['like'][-ind:] - H['prop'][-ind:] assert len(H) == 60, "Failed" diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 14237f09b..df50984ab 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -69,17 +69,20 @@ 'in': ['x'], 'out': [('f', float), ('cstat', int)], 'save_every_k': 400, - 'cores': cores_per_job} + 'user': {'cores': cores_per_job} + } gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': nworkers, - 'batch_mode': True, - 'num_active_gens': 1, - 'save_every_k': 20} + 'save_every_k': 20, + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'batch_mode': True, + 'gen_batch_size': nworkers, + 'num_active_gens': 1, + } + } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_nan_func_aposmm.py b/libensemble/tests/regression_tests/test_nan_func_aposmm.py index be7e59790..0be876285 100644 --- a/libensemble/tests/regression_tests/test_nan_func_aposmm.py +++ b/libensemble/tests/regression_tests/test_nan_func_aposmm.py @@ -33,16 +33,17 @@ gen_specs = {'gen_f': gen_f, 'in': [o[0] for o in gen_out] + ['f', 'f_i', 'returned'], 'out': gen_out, - 'lb': -2*np.ones(n), - 'ub': 2*np.ones(n), - 'initial_sample_size': 5, - 'num_active_gens': 1, - 'batch_mode': True} + 'user': {'initial_sample_size': 5, + 'lb': -2*np.ones(n), + 'ub': 2*np.ones(n), + 'batch_mode': True, + 'num_active_gens': 1} + } if nworkers == 3: - gen_specs['single_component_at_a_time'] = True - gen_specs['components'] = 1 - gen_specs['combine_component_func'] = np.linalg.norm + gen_specs['user']['single_component_at_a_time'] = True + gen_specs['user']['components'] = 1 + gen_specs['user']['combine_component_func'] = np.linalg.norm persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 4ba4ab3be..5d19c06c1 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -29,11 +29,12 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': [('x', float, 2)], - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'initial_sample': 100, - 'batch_mode': True, - 'num_active_gens': 1} + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'initial_sample': 100, + 'batch_mode': True, + 'num_active_gens': 1} + } persis_info = per_worker_stream({}, nworkers + 1) From 60f34daae875f6d2123020ec39ac9f4b61b23215 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 16:45:30 -0500 Subject: [PATCH 378/644] Formating one calling script --- .../regression_tests/script_test_balsam_hworld.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 02e201b00..08fbb9033 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -48,12 +48,11 @@ def build_simfunc(): 'in': ['sim_id'], 'out': [('x', float, (2,))], 'save_every_k': 20, - 'user': { - 'lb': np.array([-3, -2]), - 'ub': np.array([3, 2]), - 'gen_batch_size': nworkers, - 'batch_mode': True, - 'num_active_gens': 1} + 'user': {'lb': np.array([-3, -2]), + 'ub': np.array([3, 2]), + 'gen_batch_size': nworkers, + 'batch_mode': True, + 'num_active_gens': 1} } persis_info = per_worker_stream({}, nworkers + 1) From b791c182278522efb4fc3ab9b33ea73357481fbf Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 17:06:30 -0500 Subject: [PATCH 379/644] Fixing unit tests and adding one more allowed sim_specs key --- libensemble/libE.py | 2 +- libensemble/tests/unit_tests/setup.py | 6 +++--- .../tests/unit_tests/test_aposmm_logic.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 9e9bc7989..d609751b0 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,7 +29,7 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'clean_jobs', 'save_every_k', 'user'] +allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k', 'user'] allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] diff --git a/libensemble/tests/unit_tests/setup.py b/libensemble/tests/unit_tests/setup.py index 66114bcfe..e677fe37d 100644 --- a/libensemble/tests/unit_tests/setup.py +++ b/libensemble/tests/unit_tests/setup.py @@ -8,7 +8,7 @@ def make_criteria_and_specs_0(simx=10, n=1): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x_on_cube'], 'out': [('f', float), ('fvec', float, 3)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('priority', float), ('local_pt', bool), ('local_min', bool), ('num_active_runs', int)], 'ub': np.ones(n), 'lb': np.zeros(n), 'nu': 0} + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('priority', float), ('local_pt', bool), ('local_min', bool), ('num_active_runs', int)], 'user': {'ub': np.ones(n), 'lb': np.zeros(n), 'nu': 0}} if n == 1: gen_specs['out'] += [('x_on_cube', float)] else: @@ -20,7 +20,7 @@ def make_criteria_and_specs_0(simx=10, n=1): def make_criteria_and_specs_1(simx=10): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x'], 'out': [('g', float)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float)], } + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float)], 'user': {} } exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5} return sim_specs, gen_specs, exit_criteria @@ -28,7 +28,7 @@ def make_criteria_and_specs_1(simx=10): def make_criteria_and_specs_1A(simx=10): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x'], 'out': [('g', float)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float), ('sim_id', int)], } + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float), ('sim_id', int)], 'user': {} } exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5} return sim_specs, gen_specs, exit_criteria diff --git a/libensemble/tests/unit_tests/test_aposmm_logic.py b/libensemble/tests/unit_tests/test_aposmm_logic.py index 9d0adfd0a..7dfc22ed4 100644 --- a/libensemble/tests/unit_tests/test_aposmm_logic.py +++ b/libensemble/tests/unit_tests/test_aposmm_logic.py @@ -26,10 +26,10 @@ def test_failing_localopt_method(): hist.H['returned'] = 1 - gen_specs_0['localopt_method'] = 'BADNAME' + gen_specs_0['user']['localopt_method'] = 'BADNAME' try: - al.advance_local_run(hist.H, gen_specs_0, 0, 0, {'run_order': {0: [0, 1]}}) + al.advance_local_run(hist.H, gen_specs_0['user'], 0, 0, {'run_order': {0: [0, 1]}}) except al.APOSMMException: assert 1, "Failed like it should have" else: @@ -41,9 +41,9 @@ def test_exception_raising(): hist.H['returned'] = 1 for method in ['LN_SBPLX', 'pounders', 'scipy_COBYLA']: - gen_specs_0['localopt_method'] = method + gen_specs_0['user']['localopt_method'] = method - out = al.advance_local_run(hist.H, gen_specs_0, 0, 0, {'run_order': {0: [0, 1]}}) + out = al.advance_local_run(hist.H, gen_specs_0['user'], 0, 0, {'run_order': {0: [0, 1]}}) assert out[0] == 0, "Failed like it should have" @@ -100,11 +100,11 @@ def test_localopt_error_saving(): H['f'] = np.random.uniform(0, 1, 4) H['returned'] = True H['local_pt'][1:] = True - gen_specs_0['initial_sample_size'] = 1 - gen_specs_0['localopt_method'] = 'scipy_COBYLA' - gen_specs_0['tol'] = 0.1 - gen_specs_0['ub'] = np.ones(2) - gen_specs_0['lb'] = np.zeros(2) + gen_specs_0['user']['initial_sample_size'] = 1 + gen_specs_0['user']['localopt_method'] = 'scipy_COBYLA' + gen_specs_0['user']['tol'] = 0.1 + gen_specs_0['user']['ub'] = np.ones(2) + gen_specs_0['user']['lb'] = np.zeros(2) persis_info_1 = {'run_order': {0: [1, 2, 3]}, 'old_runs': {}, From c81ad46d8760206e85fe7a51690e659bb464fce4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 30 Oct 2019 17:11:28 -0500 Subject: [PATCH 380/644] Flake8 changes --- libensemble/gen_funcs/aposmm.py | 6 +++--- libensemble/gen_funcs/persistent_aposmm.py | 4 ++-- libensemble/libE.py | 7 ++++--- libensemble/tests/regression_tests/test_1d_sampling.py | 2 +- .../test_1d_uniform_sampling_with_comm_dup.py | 2 +- .../test_6-hump_camel_active_persistent_worker_abort.py | 4 ++-- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 4 ++-- libensemble/tests/unit_tests/setup.py | 4 ++-- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 3fab1459b..de2001db8 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -491,8 +491,8 @@ def advance_local_run(H, user_specs, c_flag, run, persis_info): advance_local_run.pt_in_run = 0 if user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', - 'LN_COBYLA', 'LN_NELDERMEAD', - 'LD_MMA']: + 'LN_COBYLA', 'LN_NELDERMEAD', + 'LD_MMA']: if user_specs['localopt_method'] in ['LD_MMA']: fields_to_pass = ['x_on_cube', 'f', 'grad'] @@ -900,7 +900,7 @@ def initialize_APOSMM(H, gen_specs): if 'single_component_at_a_time' in user_specs and user_specs['single_component_at_a_time']: assert user_specs['batch_mode'], ("Must be in batch mode when using " - "'single_component_at_a_time'") + "'single_component_at_a_time'") c_flag = True else: c_flag = False diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 13f9d8f40..7b085afd9 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -195,7 +195,7 @@ def aposmm(H, persis_info, gen_specs, libE_info): if user_specs['localopt_method'] in ['LD_MMA', 'blmvm']: fields_to_pass = ['x_on_cube', 'f', 'grad'] elif user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', - 'LN_NELDERMEAD', 'scipy_Nelder-Mead']: + 'LN_NELDERMEAD', 'scipy_Nelder-Mead']: fields_to_pass = ['x_on_cube', 'f'] elif user_specs['localopt_method'] in ['pounders']: fields_to_pass = ['x_on_cube', 'fvec'] @@ -386,7 +386,7 @@ def nlopt_callback_fun(x, grad, comm_queue, child_can_read, parent_can_read, use grad[:] = grad_recv else: assert user_specs['localopt_method'] in ['LN_SBPLX', 'LN_BOBYQA', - 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] + 'LN_COBYLA', 'LN_NELDERMEAD', 'LD_MMA'] x_recv, f_recv = comm_queue.get() assert np.array_equal(x, x_recv), "The point I gave is not the point I got back!" diff --git a/libensemble/libE.py b/libensemble/libE.py index d609751b0..4fc801de2 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -33,6 +33,7 @@ allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] + def report_manager_exception(hist, persis_info, mgr_exc=None): "Write out exception manager exception to log." if mgr_exc is not None: @@ -481,7 +482,7 @@ def check_alloc_specs(alloc_specs): assert alloc_specs['alloc_f'], "Allocation function must be specified" for k in alloc_specs.keys(): - assert k in allowed_alloc_spec_keys, "Key %s is not allowed in alloc_specs. Supported keys are: %s " % (k,allowed_alloc_spec_keys) + assert k in allowed_alloc_spec_keys, "Key %s is not allowed in alloc_specs. Supported keys are: %s " % (k, allowed_alloc_spec_keys) def check_sim_specs(sim_specs): @@ -493,7 +494,7 @@ def check_sim_specs(sim_specs): assert isinstance(sim_specs['in'], list), "'in' field must exist and be a list of field names" for k in sim_specs.keys(): - assert k in allowed_sim_spec_keys, "Key %s is not allowed in sim_specs. Supported keys are: %s " % (k,allowed_sim_spec_keys) + assert k in allowed_sim_spec_keys, "Key %s is not allowed in sim_specs. Supported keys are: %s " % (k, allowed_sim_spec_keys) def check_gen_specs(gen_specs): @@ -501,7 +502,7 @@ def check_gen_specs(gen_specs): assert not bool(gen_specs) or len(gen_specs['out']), "gen_specs must have 'out' entries" for k in gen_specs.keys(): - assert k in allowed_gen_spec_keys, "Key %s is not allowed in gen_specs. Supported keys are: %s " % (k,allowed_gen_spec_keys) + assert k in allowed_gen_spec_keys, "Key %s is not allowed in gen_specs. Supported keys are: %s " % (k, allowed_gen_spec_keys) def check_exit_criteria(exit_criteria, sim_specs, gen_specs): diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index ea55ad0e3..cea7d39ac 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -31,7 +31,7 @@ 'user': {'gen_batch_size': 500, 'lb': np.array([-3]), 'ub': np.array([3]), - } + } } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index dd5cc05a7..e4913bec6 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -49,7 +49,7 @@ 'user': {'lb': np.array([-3]), 'ub': np.array([3]), 'gen_batch_size': 500 - } + } } persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index 40c9d639a..a96beedcc 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -42,8 +42,8 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'localopt_maxeval': 4 - } - } + } + } alloc_specs = {'alloc_f': alloc_f, 'out': gen_out} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index e1c74b961..87cbfc508 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -37,8 +37,8 @@ 'save_every_k': 300, # Tell libE to save every 300 gen entries 'user': {'gen_batch_size': 500, # Tell gen_f how much to generate per call 'lb': np.array([-3, -2]), # Tell gen_f lower bounds - 'ub': np.array([3, 2]), # Tell gen_f upper bounds - } + 'ub': np.array([3, 2]) # Tell gen_f upper bounds + } } # end_gen_specs_rst_tag diff --git a/libensemble/tests/unit_tests/setup.py b/libensemble/tests/unit_tests/setup.py index e677fe37d..a07da9bac 100644 --- a/libensemble/tests/unit_tests/setup.py +++ b/libensemble/tests/unit_tests/setup.py @@ -20,7 +20,7 @@ def make_criteria_and_specs_0(simx=10, n=1): def make_criteria_and_specs_1(simx=10): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x'], 'out': [('g', float)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float)], 'user': {} } + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float)], 'user': {}} exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5} return sim_specs, gen_specs, exit_criteria @@ -28,7 +28,7 @@ def make_criteria_and_specs_1(simx=10): def make_criteria_and_specs_1A(simx=10): sim_specs = {'sim_f': np.linalg.norm, 'in': ['x'], 'out': [('g', float)], } - gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float), ('sim_id', int)], 'user': {} } + gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float), ('sim_id', int)], 'user': {}} exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5} return sim_specs, gen_specs, exit_criteria From 559f795c1722175724d69bbf2d67c9dcbe6c5977 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 08:36:16 -0500 Subject: [PATCH 381/644] Adjusting forces to use 'user' fields --- libensemble/libE.py | 2 +- .../tests/scaling_tests/forces/forces_simf.py | 18 +++++------ .../scaling_tests/forces/run_libe_forces.py | 32 ++++++++++--------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 4fc801de2..14409d7ad 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,7 +29,7 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k', 'user'] +allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k', 'profile', 'user'] allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] diff --git a/libensemble/tests/scaling_tests/forces/forces_simf.py b/libensemble/tests/scaling_tests/forces/forces_simf.py index 99360ff57..7988d031a 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simf.py +++ b/libensemble/tests/scaling_tests/forces/forces_simf.py @@ -45,17 +45,17 @@ def run_forces(x, persis_info, sim_specs, libE_info): calc_status = 0 # Returns to worker - simdir_basename = sim_specs['simdir_basename'] - # cores = sim_specs['cores'] - keys = sim_specs['keys'] - sim_particles = sim_specs['sim_particles'] - sim_timesteps = sim_specs['sim_timesteps'] - time_limit = sim_specs['sim_kill_minutes'] * 60.0 + simdir_basename = sim_specs['user']['simdir_basename'] + # cores = sim_specs['user']['cores'] + keys = sim_specs['user']['keys'] + sim_particles = sim_specs['user']['sim_particles'] + sim_timesteps = sim_specs['user']['sim_timesteps'] + time_limit = sim_specs['user']['sim_kill_minutes'] * 60.0 # Get from dictionary if key exists, else return default (e.g. 0) - cores = sim_specs.get('cores', None) - kill_rate = sim_specs.get('kill_rate', 0) - particle_variance = sim_specs.get('particle_variance', 0) + cores = sim_specs['user'].get('cores', None) + kill_rate = sim_specs['user'].get('kill_rate', 0) + particle_variance = sim_specs['user'].get('particle_variance', 0) # Composing variable names and x values to set up simulation # arguments = [] diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 21eab128e..1b1439d3e 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -69,17 +69,18 @@ sim_specs = {'sim_f': run_forces, # Function whose output is being minimized 'in': ['x'], # Name of input data structure for sim func 'out': [('energy', float)], # Output from sim func - 'keys': ['seed'], # Key/keys for input data 'sim_dir': './sim', # Sim dir to be copied for each worker - 'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) 'profile': False, # Don't have libE profile run - 'simdir_basename': 'force', # Used by sim_f to name sim directories - 'cores': 2, # Used by sim_f to set number of cores used - 'sim_particles': 1e3, # Used by sim_f for number of particles - 'sim_timesteps': 5, # Used by sim_f for number of timesteps - 'sim_kill_minutes': 10.0, # Used by sim_f to set max run time - 'particle_variance': 0.2, # Used by sim_f to vary load imbalance - 'kill_rate': 0.5, # Fraction of bad sim_f evals (tests kills) + 'user': {'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) + 'simdir_basename': 'force', # Used by sim_f to name sim directories + 'keys': ['seed'], # Key/keys for input data + 'cores': 2, # Used by sim_f to set number of cores used + 'sim_particles': 1e3, # Used by sim_f for number of particles + 'sim_timesteps': 5, # Used by sim_f for number of timesteps + 'sim_kill_minutes': 10.0, # Used by sim_f to set max run time + 'particle_variance': 0.2, # Used by sim_f to vary load imbalance + 'kill_rate': 0.5, # Fraction of bad sim_f evals (tests kills) + } } # end_sim_specs_rst_tag @@ -87,12 +88,13 @@ gen_specs = {'gen_f': uniform_random_sample, # Generator function 'in': ['sim_id'], # Generator input 'out': [('x', float, (1,))], # Name, type and size of data produced (must match sim_specs 'in') - 'lb': np.array([0]), # Lower bound for random sample array (1D) - 'ub': np.array([32767]), # Upper bound for random sample array (1D) - 'gen_batch_size': 1000, # How many random samples to generate in one call - 'batch_mode': True, # If true wait for sims to process before generate more - 'num_active_gens': 1, # Only one active generator at a time. - 'save_every_k': 1000 # Save every K steps + 'save_every_k': 1000, # Save every K steps + 'user': {'lb': np.array([0]), # Lower bound for random sample array (1D) + 'ub': np.array([32767]), # Upper bound for random sample array (1D) + 'gen_batch_size': 1000, # How many random samples to generate in one call + 'batch_mode': True, # If true wait for sims to process before generate more + 'num_active_gens': 1, # Only one active generator at a time. + } } # Maximum number of simulations From d3153a5eb8cdf48c25f73aea9a9c437f81904469 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 09:51:30 -0500 Subject: [PATCH 382/644] Fixing 'user' in a sim_func --- libensemble/sim_funcs/six_hump_camel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index 8848c638b..7f51e14d8 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -91,7 +91,7 @@ def six_hump_camel_simple(x, persis_info, sim_specs, _): O['f'] = six_hump_camel_func(x[0][0]) - if 'pause_time' in sim_specs: + if 'pause_time' in sim_specs['user']: time.sleep(sim_specs['user']['pause_time']) return O, persis_info From 1f9edc24eecf38d24d59f751ad2cd11d522f175d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 09:51:52 -0500 Subject: [PATCH 383/644] Testing persistent_aposmm handling of unrecognized localopt names --- .../unit_tests/test_persistent_aposmm.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 libensemble/tests/unit_tests/test_persistent_aposmm.py diff --git a/libensemble/tests/unit_tests/test_persistent_aposmm.py b/libensemble/tests/unit_tests/test_persistent_aposmm.py new file mode 100644 index 000000000..94ddb1282 --- /dev/null +++ b/libensemble/tests/unit_tests/test_persistent_aposmm.py @@ -0,0 +1,24 @@ +import numpy as np +import libensemble.gen_funcs.persistent_aposmm as al +import libensemble.tests.unit_tests.setup as setup + +libE_specs = {'comm': {}} + +def test_persis_apossm_localopt_test(): + _, _, gen_specs_0, _, _ = setup.hist_setup1() + + H = np.zeros(4, dtype=[('f', float), ('returned', bool)]) + gen_specs_0['user']['localopt_method'] = 'BADNAME' + gen_specs_0['user']['ub'] = np.ones(2) + gen_specs_0['user']['lb'] = np.zeros(2) + + try: + al.aposmm(H, {}, gen_specs_0, libE_specs) + except NotImplementedError: + assert 1, "Failed because method is unknown." + else: + assert 0 + + +if __name__ == "__main__": + test_persis_apossm_localopt_test() From 679043ddcfb08ca610faf1b4d97e0d623cde03d2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 10:03:04 -0500 Subject: [PATCH 384/644] flake8 --- libensemble/tests/unit_tests/test_persistent_aposmm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/tests/unit_tests/test_persistent_aposmm.py b/libensemble/tests/unit_tests/test_persistent_aposmm.py index 94ddb1282..53648abc0 100644 --- a/libensemble/tests/unit_tests/test_persistent_aposmm.py +++ b/libensemble/tests/unit_tests/test_persistent_aposmm.py @@ -4,6 +4,7 @@ libE_specs = {'comm': {}} + def test_persis_apossm_localopt_test(): _, _, gen_specs_0, _, _ = setup.hist_setup1() From 9d813c8224ef7e81d9b62953a50ec123d1806eac Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 10:56:59 -0500 Subject: [PATCH 385/644] Starting reorg of specs From b97d770b3830d76c0927402828ffa843dcfafd86 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 31 Oct 2019 11:00:46 -0500 Subject: [PATCH 386/644] try installing balsam 3.5 via pip --- conda/install-balsam.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 0c358c575..a25bfb474 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,13 +11,15 @@ def install_balsam(): - # Installs Balsam in a directory on the same level as the current directory. - here = os.getcwd() - balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' - subprocess.check_call(balsamclone.split()) - os.chdir('../balsam') - subprocess.check_call('pip install -e .'.split()) - os.chdir(here) + # Installs Balsam + balsaminstall = 'pip install -Iv balsam-flow==0.3.5' + subprocess.check_call(balsaminstall.split()) + # here = os.getcwd() + # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' + # subprocess.check_call(balsamclone.split()) + # os.chdir('../balsam') + # subprocess.check_call('pip install -e .'.split()) + # os.chdir(here) def move_test_balsam(balsam_test): From 87a0065f6b819109f2e14e03f84729082c09e2ec Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 31 Oct 2019 11:27:57 -0500 Subject: [PATCH 387/644] remove unneeded options (?) --- conda/install-balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index a25bfb474..cc71af28b 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -12,7 +12,7 @@ def install_balsam(): # Installs Balsam - balsaminstall = 'pip install -Iv balsam-flow==0.3.5' + balsaminstall = 'pip install balsam-flow==0.3.5' subprocess.check_call(balsaminstall.split()) # here = os.getcwd() # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' From 3ddfcc71964d101f9eec120d4692fcd001e1bfdd Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 12:15:30 -0500 Subject: [PATCH 388/644] Enforcing libE_specs too, and moving things to libE_specs --- libensemble/libE.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 14409d7ad..ed1efa1d0 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,9 +29,10 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k', 'profile', 'user'] -allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'save_every_k', 'user'] +allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'user'] +allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'user'] allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] +allowed_libE_spec_keys = ['sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k_sims', 'save_every_k_gens', 'profile_worker'] def report_manager_exception(hist, persis_info, mgr_exc=None): @@ -476,6 +477,9 @@ def check_libE_specs(libE_specs, serial_check=False): # TODO, differentiate and test SSH/Client assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" + for k in libE_specs.keys(): + assert k in allowed_libE_spec_keys, "Key %s is not allowed in libE_specs. Supported keys are: %s " % (k, allowed_libE_spec_keys) + def check_alloc_specs(alloc_specs): assert isinstance(alloc_specs, dict), "alloc_specs must be a dictionary" From 5b0defc428bb84d3cc3657cf961b8a45968caf6d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 12:20:42 -0500 Subject: [PATCH 389/644] Making a spot to (in source) document the various spec keys --- libensemble/libE.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index ed1efa1d0..f2b349255 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -29,10 +29,31 @@ # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', 'in', 'out', 'user'] -allowed_gen_spec_keys = ['gen_f', 'in', 'out', 'user'] -allowed_alloc_spec_keys = ['alloc_f', 'in', 'out', 'user'] -allowed_libE_spec_keys = ['sim_dir', 'sim_dir_prefix', 'clean_jobs', 'save_every_k_sims', 'save_every_k_gens', 'profile_worker'] +allowed_sim_spec_keys = ['sim_f', # + 'in', # + 'out', # + 'user'] # + +allowed_gen_spec_keys = ['gen_f', # + 'in', # + 'out', # + 'user'] # + +allowed_alloc_spec_keys = ['alloc_f', # + 'in', # + 'out', # + 'user'] # + +allowed_libE_spec_keys = ['comms', # + 'comm', # + 'nprocesses', # + 'abort_on_exception', # + 'sim_dir', # + 'sim_dir_prefix', # + 'clean_jobs', # + 'save_every_k_sims', # + 'save_every_k_gens', # + 'profile_worker'] # def report_manager_exception(hist, persis_info, mgr_exc=None): From d8a7f624edcc4008ebf3f7cf8cf2e363b211c46e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 13:04:58 -0500 Subject: [PATCH 390/644] Adding option to only run MPI tests (to speed up testing) --- libensemble/tests/run-tests.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 669ee5d5d..a481b6098 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -9,6 +9,7 @@ export RUN_UNIT_TESTS=true #Recommended for pre-push / CI tests export RUN_COV_TESTS=true #Provide coverage report export RUN_REG_TESTS=true #Recommended for pre-push / CI tests export RUN_PEP_TESTS=false #Code syle conventions +export RUN_ONLY_MPI=false # Regression test options #export REG_TEST_LIST='test_number1.py test_number2.py' #selected/ordered @@ -179,7 +180,7 @@ RTEST_SHOW_OUT_ERR=false usage() { echo -e "\nUsage:" - echo " $0 [-hcsurz] [-p <2|3>] [-n ] [-a ]" 1>&2; + echo " $0 [-hcsurmz] [-p <2|3>] [-n ] [-a ]" 1>&2; echo "" echo "Options:" echo " -h Show this help message and exit" @@ -188,6 +189,7 @@ usage() { echo " -z Print stdout and stderr to screen when running regression tests (run without pytest)" echo " -u Run only the unit tests" echo " -r Run only the regression tests" + echo " -m Run the regression tests only using MPI" echo " -p {version} Select a version of python. E.g. -p 2 will run with the python2 exe" echo " Note: This will literally run the python2/python3 exe. Default runs python" echo " -n {name} Supply a name to this test run" @@ -196,7 +198,7 @@ usage() { exit 1 } -while getopts ":p:n:a:hcszur" opt; do +while getopts ":p:n:a:hcszurm" opt; do case $opt in p) echo "Parameter supplied for Python version: $OPTARG" >&2 @@ -230,6 +232,10 @@ while getopts ":p:n:a:hcszur" opt; do echo "Running only the regression tests" export RUN_UNIT_TESTS=false ;; + m) + echo "Running only the MPI regression tests" + export RUN_ONLY_MPI=true + ;; h) usage ;; @@ -308,6 +314,7 @@ tput sgr 0 echo -e "Selected:" [ $RUN_UNIT_TESTS = "true" ] && echo -e "Unit Tests" [ $RUN_REG_TESTS = "true" ] && echo -e "Regression Tests" +[ $RUN_ONLY_MPI = "true" ] && echo -e "Only MPI Regression Tests" [ $RUN_COV_TESTS = "true" ] && echo -e "Including coverage analysis" [ $RUN_PEP_TESTS = "true" ] && echo -e "PEP Code Standard Tests (static code test)" @@ -412,6 +419,10 @@ if [ "$root_found" = true ]; then COMMS_LIST=$(sed -n '/# TESTSUITE_COMMS/s/# TESTSUITE_COMMS: //p' $TEST_SCRIPT) for LAUNCHER in $COMMS_LIST do + if [ "$RUN_ONLY_MPI" = true ] && [ "$LAUNCHER" != mpi ]; then + echo "Skipping non MPI testnumber 5" + continue + fi #Need proc count here for now - still stop on failure etc. NPROCS_LIST=$(sed -n '/# TESTSUITE_NPROCS/s/# TESTSUITE_NPROCS: //p' $TEST_SCRIPT) for NPROCS in $NPROCS_LIST From 33c629e98f59e861be9e1fc570e0be58294b3643 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 13:06:04 -0500 Subject: [PATCH 391/644] Moving regression test specs to libE_specs --- libensemble/libE.py | 5 +++++ .../tests/regression_tests/script_test_balsam_hworld.py | 8 +++++--- libensemble/tests/regression_tests/test_1d_sampling.py | 2 +- .../test_1d_uniform_sampling_with_comm_dup.py | 1 - .../test_6-hump_camel_persistent_aposmm_1.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../test_6-hump_camel_persistent_aposmm_4.py | 2 +- .../test_6-hump_camel_uniform_sampling.py | 4 ++-- .../test_branin_aposmm_nlopt_and_then_scipy.py | 9 +++++---- libensemble/tests/regression_tests/test_comms.py | 1 - .../tests/regression_tests/test_jobcontroller_hworld.py | 2 -- 11 files changed, 22 insertions(+), 18 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index f2b349255..2650ba296 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -46,7 +46,12 @@ allowed_libE_spec_keys = ['comms', # 'comm', # + 'ip', # + 'port', # + 'authkey', # + 'workerID', # 'nprocesses', # + 'worker_cmd', # 'abort_on_exception', # 'sim_dir', # 'sim_dir_prefix', # diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 08fbb9033..3a58b2c2d 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -24,7 +24,11 @@ def build_simfunc(): subprocess.check_call(buildstring.split()) -libE_specs = {'comm': MPI.COMM_WORLD, 'comms': 'mpi'} +libE_specs = {'comm': MPI.COMM_WORLD, + 'comms': 'mpi', + 'save_every_k_sims': 400, + 'save_every_k_gens': 20, + } nworkers = MPI.COMM_WORLD.Get_size() - 1 is_master = MPI.COMM_WORLD.Get_rank() == 0 @@ -41,13 +45,11 @@ def build_simfunc(): sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('cstat', int)], - 'save_every_k': 400, 'user': {'cores': cores_per_job}} gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'save_every_k': 20, 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'gen_batch_size': nworkers, diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index cea7d39ac..626d08ed6 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -22,12 +22,12 @@ from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() +libE_specs['save_every_k_gens'] = 300 sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} gen_specs = {'gen_f': gen_f, 'out': [('x', float, (1,))], - 'save_every_k': 300, 'user': {'gen_batch_size': 500, 'lb': np.array([-3]), 'ub': np.array([3]), diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index e4913bec6..8e25f531c 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -45,7 +45,6 @@ gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (1,))], - 'save_every_k': 300, 'user': {'lb': np.array([-3]), 'ub': np.array([3]), 'gen_batch_size': 500 diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index e112e8688..06166be75 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -46,8 +46,7 @@ gen_specs = {'gen_f': gen_f, 'in': [], 'out': gen_out, - 'user': {'batch_mode': True, - 'initial_sample_size': 100, + 'user': {'initial_sample_size': 100, 'sample_points': np.round(minima, 1), 'localopt_method': 'LD_MMA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), @@ -55,6 +54,7 @@ 'ftol_rel': 1e-6, 'num_active_gens': 1, 'max_active_runs': 6, + 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 7fe9264c5..67f2ebc13 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -48,7 +48,6 @@ 'out': gen_out, 'user': {'initial_sample_size': 100, 'sample_points': np.round(minima, 1), - 'batch_mode': True, 'localopt_method': 'blmvm', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'grtol': 1e-4, @@ -56,6 +55,7 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, + 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 288ce5980..126ed3e33 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -46,7 +46,6 @@ 'in': [], 'out': gen_out, 'user': {'initial_sample_size': 100, - 'batch_mode': True, 'sample_points': np.round(minima, 1), 'localopt_method': 'scipy_Nelder-Mead', 'fatol': 1e-5, @@ -56,6 +55,7 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.01, 'max_active_runs': 6, + 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index 87cbfc508..5aee7dc9a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -24,17 +24,17 @@ from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() +libE_specs['save_every_k_sims'] = 400 +libE_specs['save_every_k_gens'] = 300 sim_specs = {'sim_f': sim_f, # Function whose output is being minimized 'in': ['x'], # Keys to be given to the above function 'out': [('f', float)], # Output from the function being minimized - 'save_every_k': 400, # Want progress saved every 400 evals } # end_sim_specs_rst_tag gen_specs = {'gen_f': gen_f, # Tell libE function generating sim_f input 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size - 'save_every_k': 300, # Tell libE to save every 300 gen entries 'user': {'gen_batch_size': 500, # Tell gen_f how much to generate per call 'lb': np.array([-3, -2]), # Tell gen_f lower bounds 'ub': np.array([3, 2]) # Tell gen_f upper bounds diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index dc6089255..c77fae025 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -25,20 +25,21 @@ nworkers, is_master, libE_specs, _ = parse_args() +libE_specs['clean_jobs'] = True +libE_specs['sim_dir'] = pkg_resources.resource_filename('libensemble.sim_funcs.branin', ''), # to be copied by each worker + if libE_specs['comms'] == 'tcp': sys.exit("Cannot run with tcp when repeated calls to libE -- aborting...") sim_specs = {'sim_f': sim_f, 'in': ['x'], - 'out': [('f', float)], - 'sim_dir': pkg_resources.resource_filename('libensemble.sim_funcs.branin', ''), # to be copied by each worker - 'clean_jobs': True} + 'out': [('f', float)]} if nworkers == 1: # Have the workers put their directories in a different (e.g., a faster # /sandbox/ or /scratch/ directory) # Otherwise, will just copy in same directory as sim_dir - sim_specs['sim_dir_prefix'] = '~' + libE_specs['sim_dir_prefix'] = '~' elif nworkers == 3: sim_specs['user'] = {'uniform_random_pause_ub': 0.05} diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index f240621c6..8d8f084d8 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -37,7 +37,6 @@ gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'save_every_k': 300, 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'batch_mode': True, diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index df50984ab..04220a4ae 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -68,14 +68,12 @@ sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('cstat', int)], - 'save_every_k': 400, 'user': {'cores': cores_per_job} } gen_specs = {'gen_f': gen_f, 'in': ['sim_id'], 'out': [('x', float, (2,))], - 'save_every_k': 20, 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'batch_mode': True, From 9fcc4c267e3801358065fa0f86f94644cc9d1d10 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 13:25:33 -0500 Subject: [PATCH 392/644] Moving scaling test specs to libE_specs --- libensemble/tests/scaling_tests/forces/run_libe_forces.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 1b1439d3e..9fe21d235 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -69,8 +69,6 @@ sim_specs = {'sim_f': run_forces, # Function whose output is being minimized 'in': ['x'], # Name of input data structure for sim func 'out': [('energy', float)], # Output from sim func - 'sim_dir': './sim', # Sim dir to be copied for each worker - 'profile': False, # Don't have libE profile run 'user': {'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) 'simdir_basename': 'force', # Used by sim_f to name sim directories 'keys': ['seed'], # Key/keys for input data @@ -88,7 +86,6 @@ gen_specs = {'gen_f': uniform_random_sample, # Generator function 'in': ['sim_id'], # Generator input 'out': [('x', float, (1,))], # Name, type and size of data produced (must match sim_specs 'in') - 'save_every_k': 1000, # Save every K steps 'user': {'lb': np.array([0]), # Lower bound for random sample array (1D) 'ub': np.array([32767]), # Upper bound for random sample array (1D) 'gen_batch_size': 1000, # How many random samples to generate in one call @@ -97,6 +94,10 @@ } } +libE_specs['save_every_k_gens'] = 1000 # Save every K steps +libE_specs['sim_dir'] = './sim' # Sim dir to be copied for each worker +libE_specs['profile'] = False, # Don't have libE profile run + # Maximum number of simulations sim_max = 8 exit_criteria = {'sim_max': sim_max} From 16f97148d1f6355192bc26c2290420934b1dab7b Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 31 Oct 2019 13:29:23 -0500 Subject: [PATCH 393/644] increment version --- conda/install-balsam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index cc71af28b..c27f12fa7 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -12,7 +12,7 @@ def install_balsam(): # Installs Balsam - balsaminstall = 'pip install balsam-flow==0.3.5' + balsaminstall = 'pip install balsam-flow==0.3.6' subprocess.check_call(balsaminstall.split()) # here = os.getcwd() # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' From 3feaeb6e43b05da508373e57d7ab101726a52895 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:04:52 -0500 Subject: [PATCH 394/644] Adding libE_spces to worker_main --- libensemble/libE.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 2650ba296..d1a942eb2 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -55,6 +55,7 @@ 'abort_on_exception', # 'sim_dir', # 'sim_dir_prefix', # + 'sim_dir_suffix', # 'clean_jobs', # 'save_every_k_sims', # 'save_every_k_gens', # @@ -232,7 +233,7 @@ def libE_mpi(sim_specs, gen_specs, exit_criteria, libE_specs, mpi_comm_null = libE_mpi_defaults(libE_specs) comm = libE_specs['comm'] - if libE_specs['comm'] == mpi_comm_null: + if comm == mpi_comm_null: return [], persis_info, 3 # Process not in comm rank = comm.Get_rank() @@ -251,7 +252,7 @@ def libE_mpi(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs, H0) # Worker returns a subset of MPI output - libE_mpi_worker(comm, sim_specs, gen_specs, libE_specs) + libE_mpi_worker(sim_specs, gen_specs, libE_specs) return [], persis_info, [] @@ -279,19 +280,19 @@ def on_abort(): on_abort=on_abort) -def libE_mpi_worker(mpi_comm, sim_specs, gen_specs, libE_specs): +def libE_mpi_worker(sim_specs, gen_specs, libE_specs): "Worker routine run at ranks > 0." from libensemble.comms.mpi import MainMPIComm - comm = MainMPIComm(mpi_comm) - worker_main(comm, sim_specs, gen_specs, log_comm=True) + comm = MainMPIComm(libE_specs['comm']) + worker_main(comm, sim_specs, gen_specs, libE_specs, log_comm=True) logger.debug("Worker {} exiting".format(libE_specs['comm'].Get_rank())) # ==================== Process version ================================= -def start_proc_team(nworkers, sim_specs, gen_specs, log_comm=True): +def start_proc_team(nworkers, sim_specs, gen_specs, libE_specs, log_comm=True): "Launch a process worker team." - wcomms = [QCommProcess(worker_main, sim_specs, gen_specs, w, log_comm) + wcomms = [QCommProcess(worker_main, sim_specs, gen_specs, libE_specs, w, log_comm) for w in range(1, nworkers+1)] for wcomm in wcomms: wcomm.run() @@ -322,7 +323,7 @@ def libE_local(sim_specs, gen_specs, exit_criteria, hist = History(alloc_specs, sim_specs, gen_specs, exit_criteria, H0) # Launch worker team and set up logger - wcomms = start_proc_team(nworkers, sim_specs, gen_specs) + wcomms = start_proc_team(nworkers, sim_specs, gen_specs, libE_specs) manager_logging_config() # Set up cleanup routine to shut down worker team @@ -468,7 +469,7 @@ def libE_tcp_worker(sim_specs, gen_specs, libE_specs): workerID = libE_specs['workerID'] with ClientQCommManager(ip, port, authkey, workerID) as comm: - worker_main(comm, sim_specs, gen_specs, + worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=workerID, log_comm=True) logger.debug("Worker {} exiting".format(workerID)) From 18218dde83f40635a68129ce193f42cf14156b01 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:05:36 -0500 Subject: [PATCH 395/644] Using libE_specs['save_every_k_sims/gens] --- libensemble/libE_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 2b2aed25f..0bbfc48d9 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -184,13 +184,13 @@ def _save_every_k_sims(self): "Save history every kth sim step." self._save_every_k('libE_history_after_sim_{}.npy', self.hist.sim_count, - self.sim_specs['save_every_k']) + self.libE_specs['save_every_k_sims']) def _save_every_k_gens(self): "Save history every kth gen step." self._save_every_k('libE_history_after_gen_{}.npy', self.hist.index, - self.gen_specs['save_every_k']) + self.libE_specs['save_every_k_gens']) # --- Handle outgoing messages to workers (work orders from alloc) @@ -276,9 +276,9 @@ def _receive_from_workers(self, persis_info): new_stuff = True self._handle_msg_from_worker(persis_info, w) - if 'save_every_k' in self.sim_specs: + if 'save_every_k_sims' in self.libE_specs: self._save_every_k_sims() - if 'save_every_k' in self.gen_specs: + if 'save_every_k_gens' in self.libE_specs: self._save_every_k_gens() return persis_info From 1fa4c2350432bcd87e11668060e9e421a6a97bc3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:06:16 -0500 Subject: [PATCH 396/644] Passing and using libE_specs --- libensemble/libE_worker.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/libensemble/libE_worker.py b/libensemble/libE_worker.py index 1defa57a0..90791ef06 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/libE_worker.py @@ -31,7 +31,7 @@ job_timing = False -def worker_main(comm, sim_specs, gen_specs, workerID=None, log_comm=True): +def worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=None, log_comm=True): """Evaluate calculations given to it by the manager. Creates a worker object, receives work from manager, runs worker, @@ -49,6 +49,9 @@ def worker_main(comm, sim_specs, gen_specs, workerID=None, log_comm=True): gen_specs: dict Parameters/information for generation calculations + libE_specs: dict + Parameters/information for libE operations + workerID: int Manager assigned worker ID (if None, default is comm.rank) @@ -56,7 +59,7 @@ def worker_main(comm, sim_specs, gen_specs, workerID=None, log_comm=True): Whether to send logging over comm """ - if sim_specs.get('profile'): + if libE_specs.get('profile_worker'): pr = cProfile.Profile() pr.enable() @@ -69,10 +72,10 @@ def worker_main(comm, sim_specs, gen_specs, workerID=None, log_comm=True): worker_logging_config(comm, workerID) # Set up and run worker - worker = Worker(comm, dtypes, workerID, sim_specs, gen_specs) + worker = Worker(comm, dtypes, workerID, sim_specs, gen_specs, libE_specs) worker.run() - if sim_specs.get('profile'): + if libE_specs.get('profile_worker'): pr.disable() profile_state_fname = 'worker_%d.prof' % (workerID) @@ -120,7 +123,7 @@ class Worker: Stack holding directory structure of this Worker """ - def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs): + def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs, libE_specs): """Initialise new worker object. """ @@ -128,6 +131,7 @@ def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs): self.dtypes = dtypes self.workerID = workerID self.sim_specs = sim_specs + self.libE_specs = libE_specs self.calc_iter = {EVAL_SIM_TAG: 0, EVAL_GEN_TAG: 0} self.loc_stack = None # Worker._make_sim_worker_dir(sim_specs, workerID) self._run_calc = Worker._make_runners(sim_specs, gen_specs) @@ -135,13 +139,13 @@ def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs): Worker._set_job_controller(self.workerID, self.comm) @staticmethod - def _make_sim_worker_dir(sim_specs, workerID, locs=None): - "Create a dir for sim workers if 'sim_dir' is in sim_specs" + def _make_sim_worker_dir(libE_specs, workerID, locs=None): + "Create a dir for sim workers if 'sim_dir' is in libE_specs" locs = locs or LocationStack() - if 'sim_dir' in sim_specs: - sim_dir = sim_specs['sim_dir'].rstrip('/') - prefix = sim_specs.get('sim_dir_prefix') - suffix = sim_specs.get('sim_dir_suffix', '') + if 'sim_dir' in libE_specs: + sim_dir = libE_specs['sim_dir'].rstrip('/') + prefix = libE_specs.get('sim_dir_prefix') + suffix = libE_specs.get('sim_dir_suffix', '') if suffix != '': suffix = '_' + suffix worker_dir = "{}{}_worker{}".format(sim_dir, suffix, workerID) @@ -216,7 +220,7 @@ def _handle_calc(self, Work, calc_in): out = calc(calc_in, Work['persis_info'], Work['libE_info']) elif calc_type == EVAL_SIM_TAG and not self.loc_stack: - self.loc_stack = Worker._make_sim_worker_dir(self.sim_specs, self.workerID) + self.loc_stack = Worker._make_sim_worker_dir(self.libE_specs, self.workerID) with self.loc_stack.loc(calc_type): out = calc(calc_in, Work['persis_info'], Work['libE_info']) @@ -320,5 +324,5 @@ def run(self): else: self.comm.kill_pending() finally: - if self.sim_specs.get('clean_jobs') and self.loc_stack is not None: + if self.libE_specs.get('clean_jobs') and self.loc_stack is not None: self.loc_stack.clean_locs() From 6b641ba45c1676f362fb9a7e612166588c9e625c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:07:08 -0500 Subject: [PATCH 397/644] Typo tuple --- .../regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index c77fae025..cd78866a4 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -26,7 +26,7 @@ nworkers, is_master, libE_specs, _ = parse_args() libE_specs['clean_jobs'] = True -libE_specs['sim_dir'] = pkg_resources.resource_filename('libensemble.sim_funcs.branin', ''), # to be copied by each worker +libE_specs['sim_dir'] = pkg_resources.resource_filename('libensemble.sim_funcs.branin', '') # to be copied by each worker if libE_specs['comms'] == 'tcp': sys.exit("Cannot run with tcp when repeated calls to libE -- aborting...") From d3cf62b679ce5daa6749c444422bc7bb15cfb7e4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:09:13 -0500 Subject: [PATCH 398/644] Flake8 --- libensemble/libE_worker.py | 2 +- libensemble/tests/scaling_tests/forces/run_libe_forces.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/libE_worker.py b/libensemble/libE_worker.py index 90791ef06..a133b6d1d 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/libE_worker.py @@ -50,7 +50,7 @@ def worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=None, log_comm= Parameters/information for generation calculations libE_specs: dict - Parameters/information for libE operations + Parameters/information for libE operations workerID: int Manager assigned worker ID (if None, default is comm.rank) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 9fe21d235..b647422b0 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -96,7 +96,7 @@ libE_specs['save_every_k_gens'] = 1000 # Save every K steps libE_specs['sim_dir'] = './sim' # Sim dir to be copied for each worker -libE_specs['profile'] = False, # Don't have libE profile run +libE_specs['profile'] = False # Don't have libE profile run # Maximum number of simulations sim_max = 8 From c89e4f0b72361ba84884a3c691d73d8f62b98014 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:31:34 -0500 Subject: [PATCH 399/644] Using alloc_specs for batch mode (and making batch mode default) --- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 2 +- libensemble/alloc_funcs/fast_alloc_to_aposmm.py | 2 +- libensemble/alloc_funcs/give_sim_work_first.py | 2 +- libensemble/libE.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index d8e96d4cd..257e229d2 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -108,7 +108,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): last_size = persis_info.get('last_size') if len(H): # Don't give gen instances in batch mode if points are unfinished - if (gen_specs['user'].get('batch_mode') + if (alloc_specs['user'].get('batch_mode') and not all(np.logical_or(H['returned'][last_size:], H['paused'][last_size:]))): break diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index 8ee694c62..3a260a6c0 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -33,7 +33,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): last_size = persis_info.get('last_size') if len(H): # Don't give gen instances in batch mode if points are unfinished - if (gen_specs['user'].get('batch_mode') + if (alloc_specs['user'].get('batch_mode') and not all(np.logical_or(H['returned'][last_size:], H['paused'][last_size:]))): break diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 90bbdd6d8..8525706c1 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -73,7 +73,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # No gen instances in batch mode if workers still working still_working = ~H['returned'] - if gen_specs['user'].get('batch_mode') and np.any(still_working): + if alloc_specs['user'].get('batch_mode') and np.any(still_working): break # Give gen work diff --git a/libensemble/libE.py b/libensemble/libE.py index d1a942eb2..b5c606369 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -84,7 +84,8 @@ def report_manager_exception(hist, persis_info, mgr_exc=None): def libE(sim_specs, gen_specs, exit_criteria, persis_info={}, alloc_specs={'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)]}, + 'out': [('allocated', bool)], + 'user': {'batch_mode': True}}, libE_specs={}, H0=[]): """This is the outer libEnsemble routine. From 02797282f6b69c1ed80d84c934473cd86e9e2daa Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 31 Oct 2019 14:32:25 -0500 Subject: [PATCH 400/644] insist on balsam 0.3.5 on Python 3.6 for now. --- conda/install-balsam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index c27f12fa7..56aae1150 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -12,7 +12,7 @@ def install_balsam(): # Installs Balsam - balsaminstall = 'pip install balsam-flow==0.3.6' + balsaminstall = 'pip install balsam-flow==0.3.5' subprocess.check_call(balsaminstall.split()) # here = os.getcwd() # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' @@ -42,7 +42,7 @@ def configure_coverage(): f.write(line) -if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ +if int(sys.version[2]) == 6: # Balsam 0.3.5 only supports Python 3.6 install_balsam() move_test_balsam('test_balsam_hworld.py') configure_coverage() From f244b7cff40219b6ac8e75c1986b77c42bf31e0e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:45:55 -0500 Subject: [PATCH 401/644] Don't need to declare batch_mode=True for default alloc_func, because that's the default --- libensemble/tests/regression_tests/script_test_balsam_hworld.py | 1 - .../regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py | 1 - libensemble/tests/regression_tests/test_chwirut_pounders.py | 1 - .../tests/regression_tests/test_chwirut_pounders_splitcomm.py | 1 - .../tests/regression_tests/test_chwirut_pounders_subcomm.py | 1 - libensemble/tests/regression_tests/test_comms.py | 1 - libensemble/tests/regression_tests/test_jobcontroller_hworld.py | 1 - libensemble/tests/regression_tests/test_nan_func_aposmm.py | 1 - libensemble/tests/regression_tests/test_worker_exceptions.py | 1 - 9 files changed, 9 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 3a58b2c2d..6db317c1d 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -53,7 +53,6 @@ def build_simfunc(): 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'gen_batch_size': nworkers, - 'batch_mode': True, 'num_active_gens': 1} } diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index cd78866a4..0de243fe7 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -49,7 +49,6 @@ 'in': [o[0] for o in gen_out] + ['f', 'returned'], 'out': gen_out, 'user': {'num_active_gens': 1, - 'batch_mode': True, 'lb': np.array([-5, 0]), 'ub': np.array([10, 15]), 'initial_sample_size': 20, diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders.py b/libensemble/tests/regression_tests/test_chwirut_pounders.py index 0cf677c9d..d438ea63e 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders.py @@ -42,7 +42,6 @@ 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, 'user': {'initial_sample_size': 5, - 'batch_mode': True, 'num_active_gens': 1, 'lb': (-2-np.pi/10)*np.ones(n), 'ub': 2*np.ones(n), diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index db5861484..6abcf01ea 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -46,7 +46,6 @@ 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, 'user': {'initial_sample_size': 5, - 'batch_mode': True, 'num_active_gens': 1, 'lb': (-2-np.pi/10)*np.ones(n), 'ub': 2*np.ones(n), diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 860c0acd2..33f4b5d69 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -50,7 +50,6 @@ 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], 'out': gen_out, 'user': {'initial_sample_size': 5, - 'batch_mode': True, 'num_active_gens': 1, 'lb': (-2-np.pi/10)*np.ones(n), 'ub': 2*np.ones(n), diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index 8d8f084d8..619ae2d65 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -39,7 +39,6 @@ 'out': [('x', float, (2,))], 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), - 'batch_mode': True, 'gen_batch_size': sim_max, 'num_active_gens': 1} } diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 04220a4ae..73abc3748 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -76,7 +76,6 @@ 'out': [('x', float, (2,))], 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), - 'batch_mode': True, 'gen_batch_size': nworkers, 'num_active_gens': 1, } diff --git a/libensemble/tests/regression_tests/test_nan_func_aposmm.py b/libensemble/tests/regression_tests/test_nan_func_aposmm.py index 0be876285..d78e2424a 100644 --- a/libensemble/tests/regression_tests/test_nan_func_aposmm.py +++ b/libensemble/tests/regression_tests/test_nan_func_aposmm.py @@ -36,7 +36,6 @@ 'user': {'initial_sample_size': 5, 'lb': -2*np.ones(n), 'ub': 2*np.ones(n), - 'batch_mode': True, 'num_active_gens': 1} } diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 5d19c06c1..982ce2781 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -32,7 +32,6 @@ 'user': {'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'initial_sample': 100, - 'batch_mode': True, 'num_active_gens': 1} } From fe4209709fcf3e966559233bc19c7812dd47555a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:49:42 -0500 Subject: [PATCH 402/644] Moving batch_mode to alloc_spec --- .../test_6-hump_camel_active_persistent_worker_abort.py | 3 +-- .../tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm_2.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm_3.py | 3 +-- .../regression_tests/test_6-hump_camel_persistent_aposmm_4.py | 3 +-- ...ump_camel_uniform_sampling_with_persistent_localopt_gens.py | 3 +-- .../tests/regression_tests/test_chwirut_pounders_persistent.py | 3 +-- 8 files changed, 8 insertions(+), 16 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index a96beedcc..3b0252e99 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -38,14 +38,13 @@ 'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'gen_batch_size': 2, - 'batch_mode': True, 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'localopt_maxeval': 4 } } -alloc_specs = {'alloc_f': alloc_f, 'out': gen_out} +alloc_specs = {'alloc_f': alloc_f, 'out': gen_out, 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 93c7ae864..a6fb5385f 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -37,7 +37,6 @@ 'in': [o[0] for o in gen_out] + ['f', 'grad', 'returned'], 'out': gen_out, 'user': {'num_active_gens': 1, - 'batch_mode': True, 'initial_sample_size': 100, 'sample_points': np.round(minima, 1), 'localopt_method': 'LD_MMA', @@ -50,7 +49,7 @@ } } -alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 06166be75..b8e7a27ec 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -54,12 +54,11 @@ 'ftol_rel': 1e-6, 'num_active_gens': 1, 'max_active_runs': 6, - 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index 102c7782c..f3c554ca9 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -55,12 +55,11 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, - 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 67f2ebc13..87b46b2e4 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -55,12 +55,11 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.5, 'max_active_runs': 6, - 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 126ed3e33..6fd1b0d5b 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -55,12 +55,11 @@ 'num_active_gens': 1, 'dist_to_bound_multiple': 0.01, 'max_active_runs': 6, - 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index a9f8eea70..0cfe45ca4 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -41,7 +41,6 @@ 'in': [], 'out': gen_out, 'user': {'xtol_rel': 1e-4, - 'batch_mode': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2]), 'gen_batch_size': 2, @@ -50,7 +49,7 @@ 'xtol_rel': 1e-4} } -alloc_specs = {'alloc_f': alloc_f, 'out': gen_out} +alloc_specs = {'alloc_f': alloc_f, 'out': gen_out, 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 62873ea16..2cfe788e6 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -50,7 +50,6 @@ 'out': gen_out, 'user': {'initial_sample_size': 100, 'localopt_method': 'pounders', - 'batch_mode': True, 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'grtol': 1e-6, 'gatol': 1e-6, @@ -62,7 +61,7 @@ 'ub': 2*np.ones(n)} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} persis_info = per_worker_stream({}, nworkers + 1) From 34ca70d8ec185766e50fc702adea217e76c55507 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:50:41 -0500 Subject: [PATCH 403/644] Moving batch_mode to alloc_spec --- .../test_chwirut_aposmm_one_residual_at_a_time.py | 2 +- .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py index 3d0108126..3df028cb0 100644 --- a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py @@ -43,7 +43,6 @@ 'in': [o[0] for o in gen_out] + ['f_i', 'returned'], 'out': gen_out, 'user': {'initial_sample_size': 5, - 'batch_mode': True, 'num_active_gens': 1, 'lb': LB, 'ub': UB, @@ -61,6 +60,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)], 'user': {'stop_on_NaNs': True, + 'batch_mode': True, 'stop_partial_fvec_eval': True} } diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index aba1eb713..054a60224 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -56,7 +56,6 @@ ('obj_component', int), ('pt_id', int)], 'user': {'gen_batch_size': 2, - 'batch_mode': True, 'single_component_at_a_time': True, 'combine_component_func': lambda x: np.sum(np.power(x, 2)), 'num_active_gens': 1, @@ -68,6 +67,7 @@ alloc_specs = {'alloc_f': alloc_f, # Allocation function 'out': [('allocated', bool)], # Output fields (included in History) 'user': {'stop_on_NaNs': True, # Should alloc_f preempt evals + 'batch_mode': True, 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals } # end_alloc_specs_rst_tag From 789c861fc151c2482b0a240cfc85e62ad52f2661 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 14:51:06 -0500 Subject: [PATCH 404/644] Testing batch_mode=False for default alloc_f --- .../test_6-hump_camel_elapsed_time_abort.py | 8 ++++++-- ...st_6-hump_camel_with_different_nodes_uniform_sample.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index bf1b1cb52..879984ba5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -20,6 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream, eprint nworkers, is_master, libE_specs, _ = parse_args() @@ -35,18 +36,21 @@ 'out': [('x', float, (2,))], 'user': {'gen_batch_size': 5, 'num_active_gens': 1, - 'batch_mode': False, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } +alloc_specs={'alloc_f': give_sim_work_first, + 'out': [('allocated', bool)], + 'user': {'batch_mode': False}} + persis_info = per_worker_stream({}, nworkers + 1) exit_criteria = {'elapsed_wallclock_time': 1} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs) + libE_specs=libE_specs,alloc_specs=alloc_specs) if is_master: eprint(flag) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index b787596d8..4c1b27bbc 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -21,6 +21,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel_with_different_ranks_and_nodes as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() @@ -57,7 +58,6 @@ ('x', float, n), ('x_on_cube', float, n)], 'user': {'initial_batch_size': 5, - 'batch_mode': False, 'max_ranks_per_node': 8, 'num_active_gens': 1, 'give_all_with_same_priority': True, @@ -66,13 +66,17 @@ 'ub': np.array([3, 2])} } +alloc_specs={'alloc_f': give_sim_work_first, + 'out': [('allocated', bool)], + 'user': {'batch_mode': False}}, + persis_info = per_worker_stream({}, nworkers + 1) exit_criteria = {'sim_max': 10, 'elapsed_wallclock_time': 300} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs) + libE_specs=libE_specs, alloc_specs=alloc_specs) if is_master: assert flag == 0 From 4b2bc963c34ecadf585643c8075c6e74601f11cd Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 31 Oct 2019 15:40:05 -0500 Subject: [PATCH 405/644] revert to normal --- conda/install-balsam.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 56aae1150..0c358c575 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,15 +11,13 @@ def install_balsam(): - # Installs Balsam - balsaminstall = 'pip install balsam-flow==0.3.5' - subprocess.check_call(balsaminstall.split()) - # here = os.getcwd() - # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' - # subprocess.check_call(balsamclone.split()) - # os.chdir('../balsam') - # subprocess.check_call('pip install -e .'.split()) - # os.chdir(here) + # Installs Balsam in a directory on the same level as the current directory. + here = os.getcwd() + balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' + subprocess.check_call(balsamclone.split()) + os.chdir('../balsam') + subprocess.check_call('pip install -e .'.split()) + os.chdir(here) def move_test_balsam(balsam_test): @@ -42,7 +40,7 @@ def configure_coverage(): f.write(line) -if int(sys.version[2]) == 6: # Balsam 0.3.5 only supports Python 3.6 +if int(sys.version[2]) >= 6: # Balsam only supports Python 3.6+ install_balsam() move_test_balsam('test_balsam_hworld.py') configure_coverage() From cf9833b8e62d8fe19508878873134b70e9ef0a3d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 15:51:51 -0500 Subject: [PATCH 406/644] Moving APOSMM check for batch mode to alloc_f, and fixing some other typos --- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 3 +++ libensemble/gen_funcs/aposmm.py | 2 -- .../tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py | 2 +- .../test_6-hump_camel_with_different_nodes_uniform_sample.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 257e229d2..3aedc0fd8 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -28,6 +28,9 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): Work = {} gen_count = count_gens(W) + if gen_specs.get('single_component_at_a_time'): + assert alloc_specs['batch_mode'], ("Must be in batch mode when using " + "'single_component_at_a_time'") if len(H) != persis_info['H_len']: # Something new is in the history. persis_info['need_to_give'].update(H['sim_id'][persis_info['H_len']:].tolist()) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index de2001db8..84b53c949 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -899,8 +899,6 @@ def initialize_APOSMM(H, gen_specs): n = len(user_specs['ub']) if 'single_component_at_a_time' in user_specs and user_specs['single_component_at_a_time']: - assert user_specs['batch_mode'], ("Must be in batch mode when using " - "'single_component_at_a_time'") c_flag = True else: c_flag = False diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index a6fb5385f..106dcd9e5 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -93,7 +93,7 @@ def libE_mpi_abort(): gen_specs['user']['rk_const'] = 0.01*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi) gen_specs['user']['lhs_divisions'] = 2 # APOSMM can be called when some run is incomplete - gen_specs['user'].pop('batch_mode') + alloc_specs['user'].pop('batch_mode') gen_specs['user'].pop('xtol_rel') gen_specs['user']['ftol_rel'] = 1e-2 diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index 4c1b27bbc..c158d1000 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -68,7 +68,7 @@ alloc_specs={'alloc_f': give_sim_work_first, 'out': [('allocated', bool)], - 'user': {'batch_mode': False}}, + 'user': {'batch_mode': False}} persis_info = per_worker_stream({}, nworkers + 1) From 19f002eb4170c3a11d55bd41772c74f64ff0db08 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 17:55:41 -0500 Subject: [PATCH 407/644] Flake8 --- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 2 +- .../test_6-hump_camel_elapsed_time_abort.py | 8 ++++---- ...st_6-hump_camel_with_different_nodes_uniform_sample.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 3aedc0fd8..f8c9983e2 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -30,7 +30,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if gen_specs.get('single_component_at_a_time'): assert alloc_specs['batch_mode'], ("Must be in batch mode when using " - "'single_component_at_a_time'") + "'single_component_at_a_time'") if len(H) != persis_info['H_len']: # Something new is in the history. persis_info['need_to_give'].update(H['sim_id'][persis_info['H_len']:].tolist()) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index 879984ba5..46e968542 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -40,9 +40,9 @@ 'ub': np.array([3, 2])} } -alloc_specs={'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)], - 'user': {'batch_mode': False}} +alloc_specs = {'alloc_f': give_sim_work_first, + 'out': [('allocated', bool)], + 'user': {'batch_mode': False}} persis_info = per_worker_stream({}, nworkers + 1) @@ -50,7 +50,7 @@ # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, - libE_specs=libE_specs,alloc_specs=alloc_specs) + libE_specs=libE_specs, alloc_specs=alloc_specs) if is_master: eprint(flag) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index c158d1000..de7890af4 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -66,9 +66,9 @@ 'ub': np.array([3, 2])} } -alloc_specs={'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)], - 'user': {'batch_mode': False}} +alloc_specs = {'alloc_f': give_sim_work_first, + 'out': [('allocated', bool)], + 'user': {'batch_mode': False}} persis_info = per_worker_stream({}, nworkers + 1) From ee679742bd1c2b1f3d1362d817bbf42e2a2ffa5c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 20:28:03 -0500 Subject: [PATCH 408/644] Correcly using 'user' fields --- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index f8c9983e2..c52fcce06 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -28,8 +28,8 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): Work = {} gen_count = count_gens(W) - if gen_specs.get('single_component_at_a_time'): - assert alloc_specs['batch_mode'], ("Must be in batch mode when using " + if gen_specs['user'].get('single_component_at_a_time'): + assert alloc_specs['user']['batch_mode'], ("Must be in batch mode when using " "'single_component_at_a_time'") if len(H) != persis_info['H_len']: # Something new is in the history. From eb59fb99baf98a94e399bfc6e72b62bd11e0643e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 20:47:12 -0500 Subject: [PATCH 409/644] Flake8 --- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index c52fcce06..6bcc9ffec 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -30,7 +30,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if gen_specs['user'].get('single_component_at_a_time'): assert alloc_specs['user']['batch_mode'], ("Must be in batch mode when using " - "'single_component_at_a_time'") + "'single_component_at_a_time'") if len(H) != persis_info['H_len']: # Something new is in the history. persis_info['need_to_give'].update(H['sim_id'][persis_info['H_len']:].tolist()) From 9066997ee14f4b475aee4f43d9d54d6493ea7ba4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 21:07:58 -0500 Subject: [PATCH 410/644] Trying python3.8, and seeing if OSX is failing. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7c9bd938a..9f3792897 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - 3.5 - 3.6 - 3.7 + - 3.8 os: linux From d105dc9301e011453f6feda04a826b01d0828223 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 21:13:21 -0500 Subject: [PATCH 411/644] Trying to enforce 3.7 for OSX --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f3792897..72d30dde0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: - 3.5 - 3.6 - 3.7 - - 3.8 os: linux @@ -23,7 +22,7 @@ matrix: osx_image: xcode10.1 env: MPI=mpich PY=3 language: generic - python: 3 + python: 3.7 services: From 18692b00e0da5e2cfd2ef2b7887349a089d8db19 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 21:13:21 -0500 Subject: [PATCH 412/644] Trying to enforce 3.7 for OSX (cherry picked from commit d105dc9301e011453f6feda04a826b01d0828223) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7c9bd938a..72d30dde0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: osx_image: xcode10.1 env: MPI=mpich PY=3 language: generic - python: 3 + python: 3.7 services: From e6438e201a36edbdbe2e56cdaa66c1f49711f996 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 31 Oct 2019 21:32:35 -0500 Subject: [PATCH 413/644] Trying to enforce 3.7 for OSX --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 72d30dde0..766ca2ea3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ matrix: include: - os: osx osx_image: xcode10.1 - env: MPI=mpich PY=3 + env: MPI=mpich PY=3.7 language: generic python: 3.7 From ac9e4ae8ddd31deacbbaa9ef326137bb43e0e3d8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 06:13:32 -0500 Subject: [PATCH 414/644] Trying python3.7, and seeing if OSX is failing. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 766ca2ea3..1eaae6c71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: #- conda update -q -y conda - conda info -a # For debugging any issues with conda - conda config --add channels conda-forge - - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION + - conda create --yes --name condaenv python=3.7 - source activate condaenv # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; From 2ee4916524dd79bee6b750bdfdd1411bb791d3d9 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 06:13:32 -0500 Subject: [PATCH 415/644] Trying python3.7, and seeing if OSX is failing. (cherry picked from commit ac9e4ae8ddd31deacbbaa9ef326137bb43e0e3d8) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 72d30dde0..9a91ab120 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: #- conda update -q -y conda - conda info -a # For debugging any issues with conda - conda config --add channels conda-forge - - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION + - conda create --yes --name condaenv python=3.7 - source activate condaenv # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; From d47bf79506c6e78568fa0de0107f930114bf45c9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 09:53:56 -0500 Subject: [PATCH 416/644] wget recent release tar, install --- .travis.yml | 4 +++- conda/install-balsam.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1eaae6c71..3775b05e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,9 @@ install: # For confirmation of MPI library being used. - python conda/find_mpi.py # locate compilers - mpiexec --version # Show MPI library details - - pip install -e . + - pip install -e . # Installing libEnsemble + - wget https://github.com/balsam-alcf/balsam/archive/0.3.5.1.tar.gz + - mkdir ../balsam; tar xf balsam-0.3.5.1.tar.gz -C ../balsam; - python conda/install-balsam.py before_script: diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 0c358c575..afc8cefc1 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -13,8 +13,8 @@ def install_balsam(): # Installs Balsam in a directory on the same level as the current directory. here = os.getcwd() - balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' - subprocess.check_call(balsamclone.split()) + # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' + # subprocess.check_call(balsamclone.split()) os.chdir('../balsam') subprocess.check_call('pip install -e .'.split()) os.chdir(here) From e41f3ac25aa0e52519284d5ed07109a2c0bcec28 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 10:31:10 -0500 Subject: [PATCH 417/644] adjust tar name, install dirx --- .travis.yml | 2 +- conda/install-balsam.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3775b05e7..e688a4f4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,7 +87,7 @@ install: - mpiexec --version # Show MPI library details - pip install -e . # Installing libEnsemble - wget https://github.com/balsam-alcf/balsam/archive/0.3.5.1.tar.gz - - mkdir ../balsam; tar xf balsam-0.3.5.1.tar.gz -C ../balsam; + - mkdir ../balsam; tar xf 0.3.5.1.tar.gz -C ../balsam; - python conda/install-balsam.py before_script: diff --git a/conda/install-balsam.py b/conda/install-balsam.py index afc8cefc1..c75e5b7fc 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -15,7 +15,7 @@ def install_balsam(): here = os.getcwd() # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' # subprocess.check_call(balsamclone.split()) - os.chdir('../balsam') + os.chdir('../balsam/balsam-0.3.5.1') subprocess.check_call('pip install -e .'.split()) os.chdir(here) From 7e9914692d9cb8bce3ad490e71b90d286c6845a1 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 11:18:48 -0500 Subject: [PATCH 418/644] replace older balsamactivate --- conda/install-balsam.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index c75e5b7fc..839637912 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -13,18 +13,21 @@ def install_balsam(): # Installs Balsam in a directory on the same level as the current directory. here = os.getcwd() - # balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam' - # subprocess.check_call(balsamclone.split()) os.chdir('../balsam/balsam-0.3.5.1') subprocess.check_call('pip install -e .'.split()) + + # Replace old version of balsamactivate + os.chdir('./balsam/scripts'); os.remove('balsamactivate') + subprocess.check_call('wget https://raw.githubusercontent.com/balsam-alcf/balsam/master/balsam/scripts/balsamactivate'.split()) + os.chdir(here) def move_test_balsam(balsam_test): # Moves specified test from /conda to /regression_tests - reg_dir_with_btest = os.path.join('./libensemble/tests/regression_tests', balsam_test) + reg_dir_with_btest = './libensemble/tests/regression_tests/' + balsam_test if not os.path.isfile(reg_dir_with_btest): - os.rename('./conda/{}'.format(balsam_test), reg_dir_with_btest) + os.rename('./conda/' + balsam_test, reg_dir_with_btest) def configure_coverage(): From 8420fb7483314618db727b42012aef616d46f55c Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 11:31:52 -0500 Subject: [PATCH 419/644] rearrange new balsamactivate and pip install --- conda/install-balsam.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conda/install-balsam.py b/conda/install-balsam.py index 839637912..43aaa1609 100644 --- a/conda/install-balsam.py +++ b/conda/install-balsam.py @@ -11,15 +11,17 @@ def install_balsam(): - # Installs Balsam in a directory on the same level as the current directory. here = os.getcwd() - os.chdir('../balsam/balsam-0.3.5.1') - subprocess.check_call('pip install -e .'.split()) + os.chdir('../balsam/balsam-0.3.5.1/balsam/scripts') # Replace old version of balsamactivate - os.chdir('./balsam/scripts'); os.remove('balsamactivate') + os.remove('balsamactivate') subprocess.check_call('wget https://raw.githubusercontent.com/balsam-alcf/balsam/master/balsam/scripts/balsamactivate'.split()) + # Pip install Balsam + os.chdir('../..') + subprocess.check_call('pip install -e .'.split()) + os.chdir(here) From 45a0fb8329ff9eba046502bf2577d77b2799fa95 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 11:45:52 -0500 Subject: [PATCH 420/644] ensure condaenv python version (reverts recent fix for 3.8 tests on osx, but 3.7 now enforced on that platform (?) ) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e688a4f4f..a6eaf013b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ before_install: #- conda update -q -y conda - conda info -a # For debugging any issues with conda - conda config --add channels conda-forge - - conda create --yes --name condaenv python=3.7 + - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION - source activate condaenv # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; From 6ca88621b0f5f630ebd24cfe4b4a99ee0b732970 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 1 Nov 2019 12:12:15 -0500 Subject: [PATCH 421/644] conditional env version by system, clean up travis.yml --- .travis.yml | 59 +++++++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6eaf013b..8f2a2f793 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,15 +6,17 @@ python: - 3.6 - 3.7 + os: linux + env: global: - HYDRA_LAUNCHER=fork - OMPI_MCA_rmaps_base_oversubscribe=yes matrix: - MPI=mpich - #- MPI=openmpi + matrix: include: @@ -28,22 +30,12 @@ matrix: services: - postgresql -# matrix: -# allow_failures: -# - env: MPI=openmpi - -# addons: -# apt: -# packages: -# - gfortran -# - libblas-dev -# - liblapack-dev cache: pip: true apt: true -# Setup Miniconda + before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh -O miniconda.sh; @@ -54,12 +46,15 @@ before_install: - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - conda config --set always_yes yes --set changeps1 no - #- conda update -q -y conda - - conda info -a # For debugging any issues with conda + - conda info -a # For debugging conda issues - conda config --add channels conda-forge - - conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + conda create --yes --name condaenv python=3.7; + else + conda create --yes --name condaenv python=$TRAVIS_PYTHON_VERSION; + fi - source activate condaenv - # sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /; + install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then @@ -69,46 +64,42 @@ install: COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; fi - # wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz; - # mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk; + - conda install $COMPILERS - #S- conda install nlopt petsc4py petsc mpi4py scipy $MPI - - conda install libblas libopenblas # Attempt to prevent 'File exists' error + - conda install libblas libopenblas # Prevent 'File exists' error - conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI - # pip install these as the conda installs downgrade pytest on python3.4 - - pip install flake8 + - pip install flake8 # Conda installs downgrade pytest on python3.4 - pip install pytest - pip install pytest-cov - pip install pytest-timeout - pip install mock - pip install coveralls - # For confirmation of MPI library being used. - - python conda/find_mpi.py # locate compilers - - mpiexec --version # Show MPI library details - - pip install -e . # Installing libEnsemble + - python conda/find_mpi.py # Locate compilers. Confirm MPI library + - mpiexec --version # Show MPI library details + - pip install -e . # Installing libEnsemble - wget https://github.com/balsam-alcf/balsam/archive/0.3.5.1.tar.gz - mkdir ../balsam; tar xf 0.3.5.1.tar.gz -C ../balsam; - python conda/install-balsam.py + before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - - source setbalsampath.sh - # Allow 10000 files to be open at once (critical for persistent_aposmm) - - ulimit -Sn 10000 - # - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - # echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.14.sdk" > setenv.sh; - # source setenv.sh; - # fi + - source setbalsampath.sh # Imperfect method for env var persist after setup + - ulimit -Sn 10000 # More concurrent file descriptors (for persis aposmm) + + # Run test (-z show output) script: - ./libensemble/tests/run-tests.sh -z -# Coverage + +# Track code coverage after_success: - mv libensemble/tests/.cov* . - coveralls + after_failure: - cat libensemble/tests/regression_tests/log.err From 4f24e7a51249fdf86e04e487f7e74de813caf114 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 13:03:26 -0500 Subject: [PATCH 422/644] Starting to document new specs From f98764ce6c54f680816a5a16914372e69c9f391f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 14:00:30 -0500 Subject: [PATCH 423/644] Moving function from libE.py to a utils file --- docs/data_structures/alloc_specs.rst | 3 + docs/data_structures/history_array.rst | 5 +- libensemble/history.py | 4 +- libensemble/libE.py | 212 +--------------- libensemble/libE_fields.py | 11 - .../tests/unit_tests/tofix__test_worker.py | 2 +- libensemble/utils.py | 228 ++++++++++++++++++ 7 files changed, 239 insertions(+), 226 deletions(-) delete mode 100644 libensemble/libE_fields.py create mode 100644 libensemble/utils.py diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 4131d6c83..55d108b4f 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -18,6 +18,9 @@ Allocation function specifications to be set in user calling script and passed t 'out' [list of tuples] : Default: [('allocated',bool)] + 'batch_mode' [bool] : + Default: True + .. note:: diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index ab0c7a1b5..29cfba004 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -5,7 +5,7 @@ history array :: H: numpy structured array - History to store output from ``gen_f``/``sim_f``/``alloc_f`` for each entry + History to store output from gen_f/sim_f/alloc_f for each entry Fields in ``H`` include those specified in ``sim_specs['out']``, ``gen_specs['out']``, and ``alloc_specs['out']``. All values are initiated to @@ -15,8 +15,9 @@ Fields in ``H`` include those specified in ``sim_specs['out']``, Below are the protected fields used in ``H`` -.. literalinclude:: ../../libensemble/libE_fields.py +.. literalinclude:: ../../libensemble/utils.py :start-at: libE_fields + :end-before: end_libE_fields_rst_tag .. seealso:: diff --git a/libensemble/history.py b/libensemble/history.py index d21bb1c2a..e73db4622 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -2,7 +2,7 @@ import time import logging -from libensemble.libE_fields import libE_fields +from libensemble.utils import libE_fields logger = logging.getLogger(__name__) @@ -20,7 +20,7 @@ class History: :ivar numpy_structured_array H: History array storing rows for each point. Field names are in - libensemble/libE_fields.py + libensemble/utils.py :ivar int offset: Starting index for this ensemble (after H0 read in) diff --git a/libensemble/libE.py b/libensemble/libE.py index b5c606369..fe9aaf0dd 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -4,7 +4,7 @@ """ -__all__ = ['libE', 'check_inputs'] +__all__ = ['libE'] import os import logging @@ -24,62 +24,12 @@ from libensemble.comms.logs import manager_logging_config from libensemble.comms.tcp_mgr import ServerQCommManager, ClientQCommManager from libensemble.controller import JobController +from libensemble.utils import check_inputs, _USER_SIM_ID_WARNING, report_manager_exception logger = logging.getLogger(__name__) # To change logging level for just this module # logger.setLevel(logging.DEBUG) -allowed_sim_spec_keys = ['sim_f', # - 'in', # - 'out', # - 'user'] # - -allowed_gen_spec_keys = ['gen_f', # - 'in', # - 'out', # - 'user'] # - -allowed_alloc_spec_keys = ['alloc_f', # - 'in', # - 'out', # - 'user'] # - -allowed_libE_spec_keys = ['comms', # - 'comm', # - 'ip', # - 'port', # - 'authkey', # - 'workerID', # - 'nprocesses', # - 'worker_cmd', # - 'abort_on_exception', # - 'sim_dir', # - 'sim_dir_prefix', # - 'sim_dir_suffix', # - 'clean_jobs', # - 'save_every_k_sims', # - 'save_every_k_gens', # - 'profile_worker'] # - - -def report_manager_exception(hist, persis_info, mgr_exc=None): - "Write out exception manager exception to log." - if mgr_exc is not None: - from_line, msg, exc = mgr_exc.args - logger.error("---- {} ----".format(from_line)) - logger.error("Message: {}".format(msg)) - logger.error(exc) - else: - logger.error(traceback.format_exc()) - logger.error("Manager exception raised .. aborting ensemble:") - logger.error("Dumping ensemble history with {} sims evaluated:". - format(hist.sim_count)) - - filename = 'libE_history_at_abort_' + str(hist.sim_count) - np.save(filename + '.npy', hist.trim_H()) - with open(filename + '.pickle', "wb") as f: - pickle.dump(persis_info, f) - def libE(sim_specs, gen_specs, exit_criteria, persis_info={}, @@ -475,161 +425,3 @@ def libE_tcp_worker(sim_specs, gen_specs, libE_specs): logger.debug("Worker {} exiting".format(workerID)) -# ==================== Common input checking ================================= -_USER_SIM_ID_WARNING = \ - ('\n' + 79*'*' + '\n' + - "User generator script will be creating sim_id.\n" + - "Take care to do this sequentially.\n" + - "Also, any information given back for existing sim_id values will be overwritten!\n" + - "So everything in gen_specs['out'] should be in gen_specs['in']!" + - '\n' + 79*'*' + '\n\n') - - -def check_consistent_field(name, field0, field1): - "Check that new field (field1) is compatible with an old field (field0)." - assert field0.ndim == field1.ndim, \ - "H0 and H have different ndim for field {}".format(name) - assert (np.all(np.array(field1.shape) >= np.array(field0.shape))), \ - "H too small to receive all components of H0 in field {}".format(name) - - -def check_libE_specs(libE_specs, serial_check=False): - assert isinstance(libE_specs, dict), "libE_specs must be a dictionary" - comms_type = libE_specs.get('comms', 'mpi') - if comms_type in ['mpi']: - if not serial_check: - assert libE_specs['comm'].Get_size() > 1, "Manager only - must be at least one worker (2 MPI tasks)" - elif comms_type in ['local']: - assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" - elif comms_type in ['tcp']: - # TODO, differentiate and test SSH/Client - assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" - - for k in libE_specs.keys(): - assert k in allowed_libE_spec_keys, "Key %s is not allowed in libE_specs. Supported keys are: %s " % (k, allowed_libE_spec_keys) - - -def check_alloc_specs(alloc_specs): - assert isinstance(alloc_specs, dict), "alloc_specs must be a dictionary" - assert alloc_specs['alloc_f'], "Allocation function must be specified" - - for k in alloc_specs.keys(): - assert k in allowed_alloc_spec_keys, "Key %s is not allowed in alloc_specs. Supported keys are: %s " % (k, allowed_alloc_spec_keys) - - -def check_sim_specs(sim_specs): - assert isinstance(sim_specs, dict), "sim_specs must be a dictionary" - assert any([term_field in sim_specs for term_field in ['sim_f', 'in', 'out']]), \ - "sim_specs must contain 'sim_f', 'in', 'out'" - - assert len(sim_specs['out']), "sim_specs must have 'out' entries" - assert isinstance(sim_specs['in'], list), "'in' field must exist and be a list of field names" - - for k in sim_specs.keys(): - assert k in allowed_sim_spec_keys, "Key %s is not allowed in sim_specs. Supported keys are: %s " % (k, allowed_sim_spec_keys) - - -def check_gen_specs(gen_specs): - assert isinstance(gen_specs, dict), "gen_specs must be a dictionary" - assert not bool(gen_specs) or len(gen_specs['out']), "gen_specs must have 'out' entries" - - for k in gen_specs.keys(): - assert k in allowed_gen_spec_keys, "Key %s is not allowed in gen_specs. Supported keys are: %s " % (k, allowed_gen_spec_keys) - - -def check_exit_criteria(exit_criteria, sim_specs, gen_specs): - assert isinstance(exit_criteria, dict), "exit_criteria must be a dictionary" - - assert len(exit_criteria) > 0, "Must have some exit criterion" - - # Ensure termination criteria are valid - valid_term_fields = ['sim_max', 'gen_max', - 'elapsed_wallclock_time', 'stop_val'] - assert all([term_field in valid_term_fields for term_field in exit_criteria]), \ - "Valid termination options: " + str(valid_term_fields) - - # Make sure stop-values match parameters in gen_specs or sim_specs - if 'stop_val' in exit_criteria: - stop_name = exit_criteria['stop_val'][0] - sim_out_names = [e[0] for e in sim_specs['out']] - gen_out_names = [e[0] for e in gen_specs['out']] - assert stop_name in sim_out_names + gen_out_names, \ - "Can't stop on {} if it's not in a sim/gen output".format(stop_name) - - -def check_H(H0, sim_specs, alloc_specs, gen_specs): - if len(H0): - # Handle if gen outputs sim IDs - from libensemble.libE_fields import libE_fields - - # Set up dummy history to see if it agrees with H0 - Dummy_H = np.zeros(1 + len(H0), dtype=libE_fields + list(set(sum([k['out'] for k in [sim_specs, alloc_specs, gen_specs] if k], [])))) # Combines all 'out' fields (if they exist) in sim_specs, gen_specs, or alloc_specs - - fields = H0.dtype.names - - # Prior history must contain the fields in new history - assert set(fields).issubset(set(Dummy_H.dtype.names)), \ - "H0 contains fields {} not in the History.".\ - format(set(fields).difference(set(Dummy_H.dtype.names))) - - # Prior history cannot contain unreturned points - # assert 'returned' not in fields or np.all(H0['returned']), \ - # "H0 contains unreturned points." - - # Fail if prior history contains unreturned points (or returned but not given). - assert('returned' not in fields or np.all(H0['given'] == H0['returned'])), \ - 'H0 contains unreturned or invalid points' - - # Check dimensional compatibility of fields - for field in fields: - check_consistent_field(field, H0[field], Dummy_H[field]) - - -def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=None, exit_criteria=None, H0=None, serial_check=False): - """ - Check if the libEnsemble arguments are of the correct data type and contain - sufficient information to perform a run. There is no return value. An - exception is raised if any of the checks fail. - - Parameters - ---------- - - libE_specs, alloc_specs, sim_specs, gen_specs, exit_criteria: :obj:`dict`, optional - - libEnsemble data structures - - H0: :obj:`numpy structured array`, optional - - A previous libEnsemble history to be prepended to the history in the - current libEnsemble run - :doc:`(example)` - - serial_check : :obj:`boolean` - - If True, assumes running a serial check. This means, for example, - the details of current MPI communicator are not checked (can be - run with libE_specs{'comm': 'mpi'} without running through mpiexec. - - """ - # Detailed checking based on Required Keys in docs for each specs - if libE_specs is not None: - check_libE_specs(libE_specs, serial_check) - - if alloc_specs is not None: - check_alloc_specs(alloc_specs) - - if sim_specs is not None: - check_sim_specs(sim_specs) - - if gen_specs is not None: - check_gen_specs(gen_specs) - - if exit_criteria is not None: - assert sim_specs is not None and gen_specs is not None, \ - "Can't check exit_criteria without sim_specs and gen_specs" - check_exit_criteria(exit_criteria, sim_specs, gen_specs) - - if H0 is not None: - assert sim_specs is not None and alloc_specs is not None and gen_specs is not None, \ - "Can't check H0 without sim_specs, alloc_specs, gen_specs" - check_H(H0, sim_specs, alloc_specs, gen_specs) diff --git a/libensemble/libE_fields.py b/libensemble/libE_fields.py deleted file mode 100644 index 32f23eba0..000000000 --- a/libensemble/libE_fields.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Below are the fields used within libEnsemble -""" -libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that generated the entry - ('gen_time', float), # Time (since epoch) entry was entered into H - ('given', bool), # True if entry has been given for sim eval - ('returned', bool), # True if entry has been returned from sim eval - ('given_time', float), # Time (since epoch) that the entry was given - ('sim_worker', int), # Worker that did (or is doing) the sim eval - ] diff --git a/libensemble/tests/unit_tests/tofix__test_worker.py b/libensemble/tests/unit_tests/tofix__test_worker.py index e4db0d837..8ef863734 100644 --- a/libensemble/tests/unit_tests/tofix__test_worker.py +++ b/libensemble/tests/unit_tests/tofix__test_worker.py @@ -1,5 +1,5 @@ import libensemble.tests.unit_tests.setup as setup -from libensemble.libE_fields import libE_fields +from libensemble.utils import libE_fields from libensemble.libE_worker import Worker from libensemble.message_numbers import EVAL_SIM_TAG import numpy as np diff --git a/libensemble/utils.py b/libensemble/utils.py new file mode 100644 index 000000000..f5d34a476 --- /dev/null +++ b/libensemble/utils.py @@ -0,0 +1,228 @@ +""" +libEnsemble utilities +============================================ + +""" + + +""" +Below are the fields used within libEnsemble +""" +libE_fields = [('sim_id', int), # Unique id of entry in H that was generated + ('gen_worker', int), # Worker that generated the entry + ('gen_time', float), # Time (since epoch) entry was entered into H + ('given', bool), # True if entry has been given for sim eval + ('returned', bool), # True if entry has been returned from sim eval + ('given_time', float), # Time (since epoch) that the entry was given + ('sim_worker', int), # Worker that did (or is doing) the sim eval + ] +# end_libE_fields_rst_tag + +allowed_sim_spec_keys = ['sim_f', # + 'in', # + 'out', # + 'user'] # + +allowed_gen_spec_keys = ['gen_f', # + 'in', # + 'out', # + 'user'] # + +allowed_alloc_spec_keys = ['alloc_f', # + 'in', # + 'out', # + 'user'] # + +allowed_libE_spec_keys = ['comms', # + 'comm', # + 'ip', # + 'port', # + 'authkey', # + 'workerID', # + 'nprocesses', # + 'worker_cmd', # + 'abort_on_exception', # + 'sim_dir', # + 'sim_dir_prefix', # + 'sim_dir_suffix', # + 'clean_jobs', # + 'save_every_k_sims', # + 'save_every_k_gens', # + 'profile_worker'] # + + +def report_manager_exception(hist, persis_info, mgr_exc=None): + "Write out exception manager exception to log." + if mgr_exc is not None: + from_line, msg, exc = mgr_exc.args + logger.error("---- {} ----".format(from_line)) + logger.error("Message: {}".format(msg)) + logger.error(exc) + else: + logger.error(traceback.format_exc()) + logger.error("Manager exception raised .. aborting ensemble:") + logger.error("Dumping ensemble history with {} sims evaluated:". + format(hist.sim_count)) + + filename = 'libE_history_at_abort_' + str(hist.sim_count) + np.save(filename + '.npy', hist.trim_H()) + with open(filename + '.pickle', "wb") as f: + pickle.dump(persis_info, f) + + + +# ==================== Common input checking ================================= +_USER_SIM_ID_WARNING = \ + ('\n' + 79*'*' + '\n' + + "User generator script will be creating sim_id.\n" + + "Take care to do this sequentially.\n" + + "Also, any information given back for existing sim_id values will be overwritten!\n" + + "So everything in gen_specs['out'] should be in gen_specs['in']!" + + '\n' + 79*'*' + '\n\n') + + +def check_consistent_field(name, field0, field1): + "Check that new field (field1) is compatible with an old field (field0)." + assert field0.ndim == field1.ndim, \ + "H0 and H have different ndim for field {}".format(name) + assert (np.all(np.array(field1.shape) >= np.array(field0.shape))), \ + "H too small to receive all components of H0 in field {}".format(name) + + +def check_libE_specs(libE_specs, serial_check=False): + assert isinstance(libE_specs, dict), "libE_specs must be a dictionary" + comms_type = libE_specs.get('comms', 'mpi') + if comms_type in ['mpi']: + if not serial_check: + assert libE_specs['comm'].Get_size() > 1, "Manager only - must be at least one worker (2 MPI tasks)" + elif comms_type in ['local']: + assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" + elif comms_type in ['tcp']: + # TODO, differentiate and test SSH/Client + assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" + + for k in libE_specs.keys(): + assert k in allowed_libE_spec_keys, "Key %s is not allowed in libE_specs. Supported keys are: %s " % (k, allowed_libE_spec_keys) + + +def check_alloc_specs(alloc_specs): + assert isinstance(alloc_specs, dict), "alloc_specs must be a dictionary" + assert alloc_specs['alloc_f'], "Allocation function must be specified" + + for k in alloc_specs.keys(): + assert k in allowed_alloc_spec_keys, "Key %s is not allowed in alloc_specs. Supported keys are: %s " % (k, allowed_alloc_spec_keys) + + +def check_sim_specs(sim_specs): + assert isinstance(sim_specs, dict), "sim_specs must be a dictionary" + assert any([term_field in sim_specs for term_field in ['sim_f', 'in', 'out']]), \ + "sim_specs must contain 'sim_f', 'in', 'out'" + + assert len(sim_specs['out']), "sim_specs must have 'out' entries" + assert isinstance(sim_specs['in'], list), "'in' field must exist and be a list of field names" + + for k in sim_specs.keys(): + assert k in allowed_sim_spec_keys, "Key %s is not allowed in sim_specs. Supported keys are: %s " % (k, allowed_sim_spec_keys) + + +def check_gen_specs(gen_specs): + assert isinstance(gen_specs, dict), "gen_specs must be a dictionary" + assert not bool(gen_specs) or len(gen_specs['out']), "gen_specs must have 'out' entries" + + for k in gen_specs.keys(): + assert k in allowed_gen_spec_keys, "Key %s is not allowed in gen_specs. Supported keys are: %s " % (k, allowed_gen_spec_keys) + + +def check_exit_criteria(exit_criteria, sim_specs, gen_specs): + assert isinstance(exit_criteria, dict), "exit_criteria must be a dictionary" + + assert len(exit_criteria) > 0, "Must have some exit criterion" + + # Ensure termination criteria are valid + valid_term_fields = ['sim_max', 'gen_max', + 'elapsed_wallclock_time', 'stop_val'] + assert all([term_field in valid_term_fields for term_field in exit_criteria]), \ + "Valid termination options: " + str(valid_term_fields) + + # Make sure stop-values match parameters in gen_specs or sim_specs + if 'stop_val' in exit_criteria: + stop_name = exit_criteria['stop_val'][0] + sim_out_names = [e[0] for e in sim_specs['out']] + gen_out_names = [e[0] for e in gen_specs['out']] + assert stop_name in sim_out_names + gen_out_names, \ + "Can't stop on {} if it's not in a sim/gen output".format(stop_name) + + +def check_H(H0, sim_specs, alloc_specs, gen_specs): + if len(H0): + # Set up dummy history to see if it agrees with H0 + Dummy_H = np.zeros(1 + len(H0), dtype=libE_fields + list(set(sum([k['out'] for k in [sim_specs, alloc_specs, gen_specs] if k], [])))) # Combines all 'out' fields (if they exist) in sim_specs, gen_specs, or alloc_specs + + fields = H0.dtype.names + + # Prior history must contain the fields in new history + assert set(fields).issubset(set(Dummy_H.dtype.names)), \ + "H0 contains fields {} not in the History.".\ + format(set(fields).difference(set(Dummy_H.dtype.names))) + + # Prior history cannot contain unreturned points + # assert 'returned' not in fields or np.all(H0['returned']), \ + # "H0 contains unreturned points." + + # Fail if prior history contains unreturned points (or returned but not given). + assert('returned' not in fields or np.all(H0['given'] == H0['returned'])), \ + 'H0 contains unreturned or invalid points' + + # Check dimensional compatibility of fields + for field in fields: + check_consistent_field(field, H0[field], Dummy_H[field]) + + +def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=None, exit_criteria=None, H0=None, serial_check=False): + """ + Check if the libEnsemble arguments are of the correct data type and contain + sufficient information to perform a run. There is no return value. An + exception is raised if any of the checks fail. + + Parameters + ---------- + + libE_specs, alloc_specs, sim_specs, gen_specs, exit_criteria: :obj:`dict`, optional + + libEnsemble data structures + + H0: :obj:`numpy structured array`, optional + + A previous libEnsemble history to be prepended to the history in the + current libEnsemble run + :doc:`(example)` + + serial_check : :obj:`boolean` + + If True, assumes running a serial check. This means, for example, + the details of current MPI communicator are not checked (can be + run with libE_specs{'comm': 'mpi'} without running through mpiexec. + + """ + # Detailed checking based on Required Keys in docs for each specs + if libE_specs is not None: + check_libE_specs(libE_specs, serial_check) + + if alloc_specs is not None: + check_alloc_specs(alloc_specs) + + if sim_specs is not None: + check_sim_specs(sim_specs) + + if gen_specs is not None: + check_gen_specs(gen_specs) + + if exit_criteria is not None: + assert sim_specs is not None and gen_specs is not None, \ + "Can't check exit_criteria without sim_specs and gen_specs" + check_exit_criteria(exit_criteria, sim_specs, gen_specs) + + if H0 is not None: + assert sim_specs is not None and alloc_specs is not None and gen_specs is not None, \ + "Can't check H0 without sim_specs, alloc_specs, gen_specs" + check_H(H0, sim_specs, alloc_specs, gen_specs) From 5ba6df1bb032bde48cc62e69ab9a740fc5ab3203 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 14:14:49 -0500 Subject: [PATCH 424/644] An inital movement of documenting 'user' --- docs/data_structures/alloc_specs.rst | 3 ++- docs/data_structures/gen_specs.rst | 20 +++++++++----------- docs/data_structures/libE_specs.rst | 16 +++++++++++++++- docs/data_structures/sim_specs.rst | 25 +++++++------------------ 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 55d108b4f..4ed81d8e1 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -4,7 +4,8 @@ alloc_specs =========== -Allocation function specifications to be set in user calling script and passed to libE.libE():: +Allocation function specifications to be set in user calling script and passed +to main ``libE()`` routine:: alloc_specs: [dict, optional] : diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index c2cfd8c8a..8e8f15711 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -3,7 +3,8 @@ gen_specs ========= -Generation function specifications to be set in user calling script and passed to libE.libE():: +Generation function specifications to be set in user calling script and passed +to main ``libE()`` routine:: gen_specs: [dict]: @@ -15,21 +16,18 @@ Generation function specifications to be set in user calling script and passed t field names (as strings) that will be given to gen_f 'out' [list of tuples (field name, data type, [size])] : gen_f outputs that will be stored in the libEnsemble history - - Optional keys : - - 'save_every_k' [int] : - Save history array to file after every k generated points. + 'user' [dict]: + Data structure to contain problem specific constants and/or input data .. note:: - * The user may define other fields to be passed to the generator function. + * The user may define fields only in ``'user'`` to be passed to the generator function. * The tuples defined in the 'out' list are entered into the master :ref:`history array` - * The generator 'out' field will generally include a variable(s) which is used for the simulator 'in' field, - in which case only the variable name is required for the simulator 'in' field. E.g. The + * The generator ``'out'`` field will generally include a variable(s) which is used for the simulator 'in' field, + in which case only the variable name is required for the simulator ``'in'`` field. E.g. The **test_6-hump_camel_uniform_sampling.py** example below, matches the corresponding - :ref:`sim_specs example`, where 'x' is defined in the gen_specs 'out' field to give - two positional floats. + :ref:`sim_specs example`, where ``'x'`` is defined in the gen_specs ``'out'`` field to give + two-dimensional floats. .. seealso:: diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index 2d47e4e19..5fbff8752 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -5,7 +5,7 @@ libE_specs Specifications for libEnsemble:: - libE_specs: [dict] : + libE_specs: [dict, optional] : 'comms' [string] : Manager/Worker communications mode. Default: mpi Options are 'mpi', 'local', 'tcp' @@ -16,6 +16,20 @@ Specifications for libEnsemble:: 'abort_on_exception' [boolean] : In MPI mode, whether to call MPI_ABORT on an exception. Default: True IF False, an exception will be raised by the manager. + 'save_every_k_sims' [int] : + Save history array to file after every k simulated points. + 'save_every_k_gens' [int] : + Save history array to file after every k generated points. + 'sim_dir' [str] : + Name of simulation directory which will be copied for each worker + 'clean_jobs' [bool] : + Clean up sim_dirs after libEnsemble completes. Default: False + 'sim_dir_prefix' [str] : + A prefix path specifying where to create sim directories + 'sim_dir_suffix' [str] : + A suffix to add to worker copies of sim_dir to distinguish runs. + 'profile_worker' [Boolean] : + Profile using cProfile. Default: False .. seealso:: Examples in `common.py`_ diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index 02fca0801..8fb726bb3 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -3,40 +3,29 @@ sim_specs ========= -Simulation function specifications to be set in user calling script and passed to ``libE.libE()``:: +Simulation function specifications to be set in user calling script and passed +to main ``libE()`` routine:: sim_specs: [dict]: - Required keys : - 'sim_f' [func] : the simulation function being evaluated 'in' [list] : - field names (as strings) that will be given to sim_f + field names (as strings) to be given to sim_f by alloc_f 'out' [list of tuples (field name, data type, [size])] : sim_f outputs that will be stored in the libEnsemble history + 'user' [dict,optional]: + Data structure to contain problem specific constants and/or input data - Optional keys : - - 'save_every_k' [int] : - Save history array to file after every k simulated points. - 'sim_dir' [str] : - Name of simulation directory which will be copied for each worker - 'sim_dir_prefix' [str] : - A prefix path specifying where to create sim directories - 'sim_dir_suffix' [str] : - A suffix to add to worker copies of sim_dir to distinguish runs. - 'profile' [Boolean] : - Profile using cProfile. Default: False Additional entires in sim_specs will be given to sim_f .. note:: - * The user may define other fields to be passed to the simulator function. + * The user may define fields only in ``'user'`` to be passed to the simulator function. * The tuples defined in the ``'out'`` list are entered into the master :ref:`history array` - * The ``sim_dir_prefix`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. + * The ``libE_specs['sim_dir_prefix']`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. .. seealso:: From 853d6372c577a57a99c020e52bd235274eeaabcf Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 14:19:06 -0500 Subject: [PATCH 425/644] adding a word --- libensemble/libE.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index fe9aaf0dd..9aa0fbc85 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -102,7 +102,9 @@ def libE(sim_specs, gen_specs, exit_criteria, exit_flag: :obj:`int` - Flag containing job status: 0 = No errors, + Flag containing final job status: + + 0 = No errors 1 = Exception occured 2 = Manager timed out and ended simulation 3 = Current process is not in libEnsemble MPI communicator From 49f814cc5171b22da6e72fec253349b0ff637603 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 15:25:14 -0500 Subject: [PATCH 426/644] Moving imports from libE to utils --- libensemble/libE.py | 7 +------ libensemble/utils.py | 10 +++++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 9aa0fbc85..089c87e07 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -8,12 +8,9 @@ import os import logging -import traceback import random import socket -import pickle # Only used when saving output on error -import numpy as np import libensemble.util.launcher as launcher from libensemble.util.timer import Timer from libensemble.history import History @@ -102,7 +99,7 @@ def libE(sim_specs, gen_specs, exit_criteria, exit_flag: :obj:`int` - Flag containing final job status: + Flag containing final job status: 0 = No errors 1 = Exception occured @@ -425,5 +422,3 @@ def libE_tcp_worker(sim_specs, gen_specs, libE_specs): worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=workerID, log_comm=True) logger.debug("Worker {} exiting".format(workerID)) - - diff --git a/libensemble/utils.py b/libensemble/utils.py index f5d34a476..435ef1cd1 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -4,6 +4,15 @@ """ +import traceback +import logging +import numpy as np +import pickle # Only used when saving output on error + +logger = logging.getLogger(__name__) +# To change logging level for just this module +# logger.setLevel(logging.DEBUG) + """ Below are the fields used within libEnsemble @@ -70,7 +79,6 @@ def report_manager_exception(hist, persis_info, mgr_exc=None): pickle.dump(persis_info, f) - # ==================== Common input checking ================================= _USER_SIM_ID_WARNING = \ ('\n' + 79*'*' + '\n' + From 79c110ec14ce548802abfa7f21d37bf984e0fecd Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 16:03:47 -0500 Subject: [PATCH 427/644] Starting re-org of documentation --- docs/data_structures/data_structures.rst | 25 +++++++------ docs/data_structures/gen_specs.rst | 2 +- docs/data_structures/sim_specs.rst | 36 +++++++++++-------- .../test_6-hump_camel_uniform_sampling.py | 14 ++++---- .../scaling_tests/forces/run_libe_forces.py | 23 ++++++------ 5 files changed, 55 insertions(+), 45 deletions(-) diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index dc55d7ae6..71c2398db 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -1,25 +1,30 @@ Data Structures =============== -Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` -dictionary with the ``check_inputs()`` function within the ``libE module``. -Provide any combination of these data structures as keyword arguments. -For example:: +This section outlines the data structures used by libEnsemble. + +.. note:: + Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` + dictionary with the ``check_inputs()`` function from the ``utils`` module. + Provide any combination of these data structures as keyword arguments. + For example:: from libensemble.libE import check_inputs check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: libEnsemble Data Structures: - history_array - worker_array - work_dict - libE_specs sim_specs gen_specs - exit_criteria alloc_specs + libE_specs persis_info + exit_criteria + + + history_array + worker_array + work_dict diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 8e8f15711..152105ca6 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -16,7 +16,7 @@ to main ``libE()`` routine:: field names (as strings) that will be given to gen_f 'out' [list of tuples (field name, data type, [size])] : gen_f outputs that will be stored in the libEnsemble history - 'user' [dict]: + 'user' [dict]: Data structure to contain problem specific constants and/or input data .. note:: diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index 8fb726bb3..1ad4138c6 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -1,31 +1,37 @@ -.. _datastruct-sim-specs: +********************* +Input data structures +********************* +We first describe the dictionaries given to libEnsemble to specify the +inputs/outputs of the ensemble of calculations to be performed. sim_specs ========= +.. _datastruct-sim-specs: -Simulation function specifications to be set in user calling script and passed -to main ``libE()`` routine:: - +Used to specify the simulation function, its inputs and outputs, and user data:: sim_specs: [dict]: - 'sim_f' [func] : + 'sim_f' [func]: the simulation function being evaluated - 'in' [list] : + 'in' [list]: field names (as strings) to be given to sim_f by alloc_f - 'out' [list of tuples (field name, data type, [size])] : - sim_f outputs that will be stored in the libEnsemble history - 'user' [dict,optional]: + 'out' [list of tuples (field name, data type, [size])]: + sim_f outputs to be stored in the libEnsemble history + 'user' [dict, optional]: Data structure to contain problem specific constants and/or input data +.. note:: + * The entirety of ``sim_specs`` is passed from the worker each time a + simulation is requested by the allocation function. + * The tuples in ``sim_specs['out']`` are entered into the master + :ref:`history array` - Additional entires in sim_specs will be given to sim_f - -.. note:: - * The user may define fields only in ``'user'`` to be passed to the simulator function. - * The tuples defined in the ``'out'`` list are entered into the master :ref:`history array` - * The ``libE_specs['sim_dir_prefix']`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. + * The ``libE_specs['sim_dir_prefix']`` option may be used to create + simulation working directories in node local/scratch storage when workers + are distributed. This may have a performance benefit with I/O heavy + simulations. .. seealso:: diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index 5aee7dc9a..d6263f0dc 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -28,16 +28,16 @@ libE_specs['save_every_k_gens'] = 300 sim_specs = {'sim_f': sim_f, # Function whose output is being minimized - 'in': ['x'], # Keys to be given to the above function - 'out': [('f', float)], # Output from the function being minimized + 'in': ['x'], # Keys to be given to sim_f + 'out': [('f', float)], # Name of the outputs from sim_f } # end_sim_specs_rst_tag -gen_specs = {'gen_f': gen_f, # Tell libE function generating sim_f input - 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size - 'user': {'gen_batch_size': 500, # Tell gen_f how much to generate per call - 'lb': np.array([-3, -2]), # Tell gen_f lower bounds - 'ub': np.array([3, 2]) # Tell gen_f upper bounds +gen_specs = {'gen_f': gen_f, # Function generating sim_f input + 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size + 'user': {'gen_batch_size': 500, # Used by this specific gen_f + 'lb': np.array([-3, -2]), # Used by this specific gen_f + 'ub': np.array([3, 2]) # Used by this specific gen_f } } # end_gen_specs_rst_tag diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index b647422b0..1ab8a0fd2 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -67,18 +67,17 @@ # State the objective function, its arguments, output, and necessary parameters (and their sizes) sim_specs = {'sim_f': run_forces, # Function whose output is being minimized - 'in': ['x'], # Name of input data structure for sim func - 'out': [('energy', float)], # Output from sim func - 'user': {'sim_dir_suffix': 'test', # Suffix for copied sim dirs (to ID run) - 'simdir_basename': 'force', # Used by sim_f to name sim directories - 'keys': ['seed'], # Key/keys for input data - 'cores': 2, # Used by sim_f to set number of cores used - 'sim_particles': 1e3, # Used by sim_f for number of particles - 'sim_timesteps': 5, # Used by sim_f for number of timesteps - 'sim_kill_minutes': 10.0, # Used by sim_f to set max run time - 'particle_variance': 0.2, # Used by sim_f to vary load imbalance - 'kill_rate': 0.5, # Fraction of bad sim_f evals (tests kills) - } + 'in': ['x'], # Name of input for sim_f + 'out': [('energy', float)], # Name, type of output from sim_f + 'user': {'sim_dir_suffix': 'test', + 'simdir_basename': 'force', + 'keys': ['seed'], + 'cores': 2, + 'sim_particles': 1e3, + 'sim_timesteps': 5, + 'sim_kill_minutes': 10.0, + 'particle_variance': 0.2, + 'kill_rate': 0.5} # Used by this specific sim_f } # end_sim_specs_rst_tag From 756d0f4c115287d724e7d3b4bb3b83862cbca7eb Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 16:04:27 -0500 Subject: [PATCH 428/644] Whitespace --- .../scaling_tests/forces/run_libe_forces.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 1ab8a0fd2..d0ed23af8 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -69,14 +69,14 @@ sim_specs = {'sim_f': run_forces, # Function whose output is being minimized 'in': ['x'], # Name of input for sim_f 'out': [('energy', float)], # Name, type of output from sim_f - 'user': {'sim_dir_suffix': 'test', - 'simdir_basename': 'force', - 'keys': ['seed'], - 'cores': 2, - 'sim_particles': 1e3, - 'sim_timesteps': 5, - 'sim_kill_minutes': 10.0, - 'particle_variance': 0.2, + 'user': {'sim_dir_suffix': 'test', + 'simdir_basename': 'force', + 'keys': ['seed'], + 'cores': 2, + 'sim_particles': 1e3, + 'sim_timesteps': 5, + 'sim_kill_minutes': 10.0, + 'particle_variance': 0.2, 'kill_rate': 0.5} # Used by this specific sim_f } # end_sim_specs_rst_tag From 004d361020794850ed95826970d6372ffe51d8dc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 1 Nov 2019 16:06:06 -0500 Subject: [PATCH 429/644] adding another level of organization to data structure section --- docs/data_structures/history_array.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 29cfba004..d74488f3d 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -1,3 +1,7 @@ +*********************************** +Internal libEnsemble data structure +*********************************** + .. _datastruct-history-array: history array From ea313628f214e6bcae98a485dbd5eebb6323a3ac Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 5 Nov 2019 17:27:41 -0600 Subject: [PATCH 430/644] Trialing new affinity diagrams, structural changes, remove unnecessary content, reorganization --- docs/images/centralized_Balsam_ThS.png | Bin 0 -> 57918 bytes docs/images/centralized_Bb.png | Bin 0 -> 35065 bytes docs/images/distributed_Bb.png | Bin 0 -> 26099 bytes docs/index.rst | 3 +- docs/platforms/bebop.rst | 46 ++++++++++------------- docs/platforms/platforms_index.rst | 50 +++++++++++++++++++++++++ docs/platforms/theta.rst | 23 ++++-------- 7 files changed, 77 insertions(+), 45 deletions(-) create mode 100644 docs/images/centralized_Balsam_ThS.png create mode 100644 docs/images/centralized_Bb.png create mode 100644 docs/images/distributed_Bb.png create mode 100644 docs/platforms/platforms_index.rst diff --git a/docs/images/centralized_Balsam_ThS.png b/docs/images/centralized_Balsam_ThS.png new file mode 100644 index 0000000000000000000000000000000000000000..f6ff9451d904ea2450a687108b54599cf88123e6 GIT binary patch literal 57918 zcmaf52UwHK(ndtEfT$={5DQ?1kkG3ELg)}$C{h9;p_fpS&_ttHkfNwG0V$$@fHdi# zqV%qSv{QxSkdiA16>mVhA& zJGg;Q;42q5tOL%$4*TnCC`45HlBnn25;y3M_bx% zz7EdL7y+oNxG)4Xbygc=>)`6;=4megQv#pWUA^p_!8<4he{~JO9~1Brg$Rp^|EfXo z0nJN7gdyiZInvJ72@Fb=Rs;MK1x2b31UsA?2^7*Q{ThIs7Y6@purP0!sFIzT3lwGG z?B?QOW2%n^Gonq($By9X;O6?P8W=wG&xgIp?smUQv35QVwsya&yMsx2{!y7$0z&}d zVI44d0>%Zj=J1Db(E5Z+PFO1sM{QNChL(+)p_du_R|BpXmp_`-15I55L#H(&X-x96 z755ZF=!t<6Rd>8L%FV&V5#oZ8*2TC;ni#ogOXHl}h-6J)l#!#FxT&E4R0$?2ZD62c zXK#SeL2CG#x}sfC25u^*;?mB}UdHZ{N-ol9ZwV=V69-TiG-@O!h0qp-`C=XQ#LO@@ z1a}u3Ur;P&jCUnEpwv-lEwY}i0m2rqDh-Oo!Al?RCaQ^rNvL_b7%4ewXo4vyVSLCC zO%r}iYaBIRl(?Pl!b1i?Ak7^xdbDMRcWq)atc2qq*4U1M*kp{NlG zf`+*vOc2J}>Ut&wh&Rp&=P8PHGV#%rva=sM{*}BAiK%(l94|LkT@~ADpA1il`ml#tbb6cOj~xZ44ZpJdL0j2pr`I=IQAT z#h~=j((0-zCaOdih_{ce1RetOB&gvubs$=XCQ>S(k`@U}lfk#Oax#bk)R+ zoF&~6o<7FT8WK{rI^YX4cXcp!Lp|_ERLN6STg<>0M}~N7pmE}&5Dir+2^)2ksD?Vh z)z{vP02EBs-rZT;(UfeYBVnlPLc)rfVLhEC?Y&8OsJ*wgp0=2phYM5+@2o^rR+92T zBix9__DWJ1GmM9ss)n|THjeD7=LQ4qpwUQOvW5#z(^LZK>S$;rZlo`+rKjg?L{L#h znwg496LrBZRuvW3vBfx&)V0*y43)&4u`U>-v4;uL9d4q8uyeBq=1tYWK;Ku~$P?yd zA`Zh4l$CJ^bw^t#N2rn`-bTk%%ot%RCIzPFYmDSDA#r;SG`Iid_T zaC%O967~r2n356+VJj`6u7R~hdw?2_B!nZJY^?0ypouUN1^Vxe)3;SKmeg<*H}SGl zL!b#rRZU$tCjv^^MazSr@1$!4oMYTnB-SJ|eRW}c36>UwZt(uM!5{9=U!)WT@A?jfZ!J429+}*?^Tx4R|*1IwZ6tO^=aGBdH!cSZZ?nV?BhTAG@wSRz`@*whWKj@DMfdy0Cy zs^A@TOuSv)@hYae21Hi}T?bJuoV1CWkG{663Q0}U!`@xX9qM9eN0YL%p{g0iQ_Drn z(?r?V*w9B)Lkz14%BA%kT#O)2c8+M;1e{&m#c@y#I~#i?Jc(e7H#WsXbyPGpU3E!f zhIojCnUN#X5%1vXfHL-=)mL&iHq%E+L-9Cuq?ip{!-l4R#!gVYfvXWpN>^9eRFrIn z_Og+%(?BVsL~UH9bbxSFCDldU>=5>*u3ktVqLYq0n5`#x*<&CsX1aC=eOID~F;UV^ zm5A~504*w!4eiu4^~BJ4DP4@Wv#PQaR??B6t0ZA6p{FF~Yv=@VwU=_zmJ-uZhxtIj zS)u8oF5!jMgLxBmoY62BI02>Pqk`5@LTl@riMiQ7{NC6bwZ(>+Yf^MrzI`x?l&Wic7md z?Uc1N9i=c*UI=ZhsSS}f5u};1wX>Tv(Xb9Ia(nK0V2_$iGccL+oBuy|t zIygeHM68Vg6z}2#QzscIOS{^b>Y|+Sj(VnWA0L>hk{bbOY@(`fgD}GxlWf)CM&9;l zf`R>xVIk`gl#h~7q^38MNwYHUT9lsBvQvn z*Tz&6;%cj}FRrHM3XXCmF=Km2jJ~U}2@2=p3r-VFNoi@gqa)5&jqIhN>WmZDl9Ysl zUR0%&0H6Wye`1;)fCE1N0T5w`ZU3XXbaec5>MC$U(vyjJzbg-owrUeArl-5}`dpRL z4I#Urdv^~)_@nM0MyTxZVwv90JX2?fi!gI@jxH&2cxFEG`q<;x$3C*1Z-~K=q(%c;mgk?0`n6y6csL$w%6{!Kp%DuWc#@cS6{4uPZLOoz^5Dsn zCjx?z=R_0~sNtBrK>Jnx#rt-yDXrf}&s)UEQop`js_+>7f`8yg$3;KdzTzYyB* zqM4~B_)#*t9+Vh;e6HIK)9MvDLz=jY$T%>=AVqFK!*@&5TqFSvf~L~#A;GmGMUSsCdfieD&7f;*I$e{r2-ui#WvJ$5ba z>C>-!;&r+cF&*1Qk+rU-scCe&Zz6>b%rKUdUk;6o;5d#R^%s5_ZP)lX>5}>Dr?i*B zbDbpG3w(>s@43w1)7jrY@%4yjsdEn$#njr`x^2_-vcHGfVBY}q#ufD^kLYW zyy91{X4tb6SLa7lSa>9)e&$tFSS-DMW;8iD8Rpyn3bQ*_@oFWd`rW&C=7Nzw&Q%0{ z+U&S^SL)1GFbj!3LC!Qt@@uoLfQX2jKL&%jkWH7_dH4K<3n*=EmdlLE#-NoKX=#i` z#>S^IJAaOS&BwP^$_4nghy`P=Rzn0&_z9uinK+K7w38$y5sp21c4HmSH1mq0I=p{$ z2v9@g>aj9e?btogJ0Ao4>c77zn}EUO4&JFW-JK95q|5%5g*2~LUtcepchN7xkEyGz zjo(i>=<<{v=zUt}GZxAl1O4#u?ey1`_WK4YTHc`Qop<4BP$j_?403!23;h z@Y&=sf=nh$@&!t{G1%5E+I?>btc6GuiG=9+#xNnC_t3p^^&$eZpDcD!hoaCkIA}~4 z0e{cQ(GikPAC;21WFcS!w0m(#$ka=EU*5s$9jMuLCa+AHxO-5mFfvN-w2+`3_~pw{i<|om+;Ibl}kcA(eS+M%rh!!_QUCA^yT-@A!fTgn0&Zf zxvQ0ABZ@LhN_s@&X!}L@wC$38R+rr6HJxak$fo8V^ViNX6W#~EA836atwq*`m-e4T zp>FQt)~-}!i@TZ=|Mo}KR5>bAv5rm`8zQkYmmYlvH{R76N%yIPP27~rK42EIgpB?B z_gBVCi4!xxz!XzND#JzdHdpfNkM{37u)i@cBvjtxLlE7&Yrdz?9xdlim1S{ z7bDl|>h6R(b{9EsbdY2oLf(ycg!5Z18y(l)_h>)AvO$l-jd}L(hOYmh4eS2>o4aN4 zkqF{D(#`CR_IRDhJJS!ZXRK2Dpn0#DKf4 zE*G1s4BEWMl}4m^2`GP>?Q&m)IK4Vu_x{ml%hnl6)YKqqDM-jyWw-9Yv9F+s5iVukYHA=kb9)1EtqEO%3etdi`Kj)WYRTbvw zLQc$bU&zsi+T%W17hywhCOT{-h? z1x*;9FS#!CwN6_37V8PszSl0wLOM?_TV37sPOJ=Pq%CfopVxTTrG*}RV$whvF~z0t zby7`DjRKxsua<|J^{&s8DB=7$h%@-x4+^V7#D+5tmo^AfP*C72sVt)Uh7etVK6#8! zm-(+kHPqD)ckZi(*iZGBjFnASMpiTr9hH3EfVBzF9L)NbbVuN%qjx>UEQ=iU`Aj5- z31`jh!rtaY#V-Pj^qX}hJ`Nsy6iR@95RC{r1+TT4txl+-q$8 z5wFcP@8S92W$D*uS(YwgE)qWu9${w>cBPh|TYK3dtLEgC`LX4ik)B?V>p&^6DD~+} zOFp$z>^;(})3<73PAjo#DkUfH-)%2izcM=%%))a;U#KlSm1f>hi9a=Y^Dl-#Yw)~t ztXz#5Ud}dp1qdFw0+X!v=-q{8^Fb~C3biR3;qIg76M|B7;uWv*r5k1l*ABjV^~$o= z{Q;S#soCALgSBPYPz&3?8FS;%9@@HSGrKq3fJ|j_Iev_#HaMZVy8a8%8~yO%m{ z2tK|s2DTL>cXhb|a$pcjn*dj+DoTE3eLci74u%iw*x6{pT}594L%{Jus%mU)WxaXh z#_gXYp^{WRVK=G5cahW}jeejgm9WM3&E)bb7i?NDxRZXVlQ{({Uj&QhXjCUBAE zW4$+jX6CegtG?A9#w)Am`S7`P=REo)38aWIh`MocNdFOGCKuWp)#)(uX+okR_pj+b=0nKX9*q+ zP0euvds>lqKbLjd)u~MGil>Q!E;>Q4`OgV{IL`Z6PeRHDdp~1-M2o;4OiBnkT?Qmf zuBx?$JOz5OFhKm01p2=7GfgtzmtmfEnp|n>13R2tHF_V(=P^{GxV5Aq@zZ} zufDO7hc*3}{5U^z_0zn(JlUW8EG#S!h*z{~2cq+1x_KmB!Fh0ME#}iFLqk*39y=xn zS681{*cF;O_7}IO_#TS=`sf0Bjyl?cTUl9AQ@B1S;bCwI3dPB1>z?l#^g3XJ9_?)m zpQYy*LuOARCF0hZ34VUEmSb}{Z{92@uI))}8CSG_w{TLmvF+=MG%EH+G923cuJ|6X z4B5+Ht|l($yH#yUOmFUE6P%$c5~rR#__eAd+V<~DB>AiYR|Tdk3JS!Yg<37?>*>u% zH|7CAkk)E_4)N~bDgUN77UI1|Og+UOwo&lI<0YdBgPZ-$agqWjo=LM6QI1O!ROSu~ zqtnyV$9{HZ4s=!b7CMm?g(jN9<>vNIZ-#J)G*ErcfEH^KLY~C;g-r*OGd&0KYdx}M z_TJf7-VP&~t2a2Oe=u*>6>iqy@^kKeDRhcb&TfFe?r{#QDBevEOLR%RbkrQf@t#6lGdHz0V6I-MeECGc0*bm0RzTw$?c^-ERs>LBN<)JN zko~!aafPc}a?`TSxcmoT=|p3+q_eLSW*^-rTL)}*GsSh{bMAl-O(R1tFD@@59UL4U zgj_5puD`qX>N)QaP))U*?hq=6k9#mi4?wA+#aI5T1Zt~4I2p>9(b-q({kA{n+U6DU zo_m++X<%@*%2=_9`$Xymt|Qt-%x9 zdi!ZdRrBt08t|y8Q3d;hy`9J>?`x<&7*+ALXOK#DIB)dsU>?4eZ=pj|WY)5;#1kLI zi+y{2PkC!=v}pCL{UDh6F?oghmKLAO!V{OTZp|mQgT-xsrWvJ&K-~OzFwfzmAa$Hg zP|JNtD_-7@9U$e|?835#u_d0PbCtSM$`@Zog8foY^+|n^l(Z}NVhDktWyQiA1JR5# zJi>8YQRw{px?okrF-C~Y>nfFuo2TfR7gDB%4^oc5Xff#!J&^=#c*4*gbHJogEm*x7K<`{+Rc^`TpFl9IKpy*$@nY zK%jS@^02z5ADb#u2MkuD-^IH{0UllO8{v8)R+UZbya4Hz%W7@pU)^5+LRNgPpRUp! z6&riL2Yw@XY35sXdEx~Tk6woW?46^!4_EWpbRmFu{84_Oq?S(zeMg zcA^xU5V4<_F6+RIJ%@LV2qi%3sGq}AhxjN@R#VSkwDqvIUOTb*u9h9~KI-5{p=0Bn zvt2X7d6vt5Q-63`S|G%rC$y_a+WTdD%UV($oyeBf(;{Jj*xVm&^X( zmxyCALZjy`x5urv!AQXg_F?@=yZyrr`FGbQ0T8F1gyr3-G8-92nP+5g&#VKhPdaja z^OJ$Dw<+tpC$C!-x1TIm&SaNnWN0kCKCQlTqXOl?dbLgDRKV*NlUPpK{*3B$c7`m* z9s(PfL z`WD|e+gvswou`ZA-Sd`yvmZDRibX(q>koDeT;o$RGBoruWo>+XX)vI&?D!C?;)Wh= zWpCW0b0sq9t=fl*IL?vh2~e8xPaR^X9GAhQZ;B3oMg+#*D~XjF z-@6HMk_#_di3z+er$C$;(FX=U_eS`~*)NS*JbQMW1vxQSOu#}rviu=S@DINhSJ&QZAP1Fyp}7+???6#ERC%Z#LH|{Fdja zA{QZS=F$FfF;xRFR_mwkK|e0 z*gN!xY159ANCNvbnf9vMoODZG))r)&US5>Qx+O&=0C(2(F#<@@Q~j2HZ0S#+N>WkC z&v570)KUI;Ir4*LPhi86$2HZ|A}09XxpV`qBqeyw0# z=O^ZX!}lccU6j5TOZxBb7+Js75`CLv!r{nW#=}Rja;^~-=_l0dvlykt24~;A$sPVy z{}|O@z*nmjRh;+5<27e$FTWoT@||c5W+J56moUcgVF$`d&ImrCW2+%zgcpz}0#cc& zCXX$e^4tlQZnAZSPTeZu95;AXq9I=f&UbOaaQt!2!Ii5cJgCYx05Zl`sFQs~A%P;U z$Zw3_1taU*_W{s%M^HQVy2gscJBqB>-VInZ}^w9D4(4r769V$ z>48sqop_&?CRp3LfB6KCb3H!wQ($oa!44`_xc9(EfT^^fNTKG;W^(Uas2A(C0{HXn z`SXxUR`xOV5Zef+-Mee$MtCA56s_GcKM$qY*{d~8sMYTCn;V&{x1kAs->?ql9{M^f zpR3N~Hw}06mwV(-uSf@w@2a^jtc>C2;HOTZ()} z2sqPaCZkPYxTfMkPih5X<>l%*gS4yHJTnb!>IoA&3qP$j~iL?{D0{}sQJ5_EBgBBhVlxihx>sHiI=_| zzTeWKnUj?@nFqvak@#V1dU|@HjOJ^lX~zk!#XNj?)X>17+7E0`m-58Tp-9~}a0Uxp z#cOEnm$|yN!P|)hHudcEzD8$>GO=xKhwt3j{@HcJMxc*jau27=*^qsqFeEc@G3$@p z_uE7yosf>2@dIbF{60vFv}a?o-VVqR@OSL%+wZ)9E2SC zaL;Omp<-lYM4_7DXG}|?Vz|##{~b&JS;WHD`gBUG!um^O2-M8XtZiwsul?~QbCvZ& z#OFYa$>8Wr5_jr)3Lvo8kB(;wn=AlFz{`l!s!m-m;mB}4#25eh(1}HjQh-uvzVpvd z6PZi$d>_yI8>Mi(C_tQ^wUG~uMYIeYabTYkb``Cgia4@%KeN*)=*O1oJr3(D-Y``; z-8YAgk_W!;W`5yU>3hU_C2msI^?i8@Kjp>ueI$cdw+C4+(_fQ&u0!eXS!HrOk$`Ax z&pKp%_OaBPRK>R}kkW6@a;+oaN9P&J&zZcRW#qqhIf3m9e_wdVaZYi09^w`}7s2f@ zx;(b?l(8;7bGWE-b`JlW;x+X>Gb*`zhkolt=ob87{Hss*ZygtanI?f z&r{882Q2yD=)I)w;Vg1Jw^t5zk)BLXJ$mswX?MhB{?>iVlI!a5rTmeA>RR)H)0a*! zR==Eux3gz8A&ZoPl1$$3;rx2X@o*j6Oq%uKTAm;Vda{l`#o6iw$I<(y>eBD z=8i|q_sL0o44iinSAupWykNfuq`9BnqkI>T-01rgbSwB~rw0wp<-3*NBOR{883%S* zF);jR%N>26>quSY#h3p}&pONGhvP5F9@prUb8jDjykYA+6(RIh`|YVPNxFRub5EXd zWAEgT$g<=i!=D8IXC2*ZpSF6;Ut}d6xA1QbYRQ_1<1cXKd@Au)82j$W_^C@TtXI)b zQ{qsP{E$PqtY{sdw3Fo(AiRs;=>gkR_@~612lxkyuY|W$rQ9Fb3p}N1`?k==MZVpK zdX=e5V|U<7W2W=p>30R&9OkeN63RgE>%NJ*jyUc%#gLwr)Q^!9c0CC1b~+F-+0;1Z zS8iB3WZ9>rj`97KJv+P(UpQ7a!?BBDf@ps=^i%T z`SV;}9Ix38yl`t(iifmU8{betEiVsNIa`Di8*yQkp2D0Q{l|)+DL~pdWY`6{n&}eP zqAn?db)RA%^JG%n62B|mA&$L}qal|ogM{w>C7^7^eWRt8IhP^^hT77`3-^pU3s5Rw z#2uJ~2lcn}y=S5OCDl5P-_*akli#90A*jU|N|Rp?%R!gCcrsUf7PhT^k^57BKj`nx z|I^V&>=7 zl4h2ycZ+BvqnqG?HgFU_l|TCFzl@QE8ESfjJ5Tx6LAc-97xK4x=-%%vW`xMsJ2rpq zElzCS88NJpkCl!5?33V?Z_kzU-l1l#Tjp%`a?p64as;GahVgByzBa zD3lE@j$$Rv2wZuxF5xG#k0V1)*pU~Py+7`v;i6jOe-m&j&rp$-WiV&(;U9}ey0;bl z7C-UTx0-21VkIjJII*9)ZWqKJvfMOs@lROZdS2)j^3#OGHh0zP(>!%~b{4+v;4;=e zT}80rn8t6vcE;rVBz!R!nX9~YW2(VQ9yHu_`*koOhsi@J`9i^0myfje`|c zKR!H5mI<_RJvj}l&FGq^CO%B)82mE5pR&>UeEq&948BYhIES_iByKa0EGM4uE_c&vyZiHVZS5^I zI^umXl`p-?iPMfG zGHaquHQS9-=!L&Vln?<_$ zx67XG5)c|I6Vtzfu{|qR_nFgkJL?Q(V=Y67boGm;CLiKOvx0fcW*k_mpUv zHf%?IIY?6$nvm5read~Zr!3Sd8`X~nley{?n=dH6K5(o3sz|_XW081bE#vC_cU)u5 z98slmOyrLZ$K(n>9UptN*1^|rN_b*>MO3JDw%2M~E)gFcy3b6Vt*Kf3F|-aSP}_yp zhbhb5St{EO$42>cEM;^beth!e>1Y5r$)f&oafX0V_%>Y2N1irX0D}oDH&j$qT*wJQ zo?#*M)en?+vn_B_-$BnAbUuPp*M7o;*JW=8JC~fi_I$uJrb6GUHzoOE`111BW}U=C z$B8D6>}~5yrFyNp&sTJvc8f5{!8+^Kb&}u-1`zE9znhGvXR`zu1~y*4?i**K^k@Eq z4y#g{29%sX4BC}*4XXb*H@4myuE9OD#@aP!IXVKdt=RA)hqKW1I>(-Jr6}vc^0Mod zO(811G%$MQacseuwxmfXlHz;^Ve8;gC5pfOdaSHOUD~-H@}+^0x{CAvdG^?g)v?HS zNpH!Yyc84VS^l%WM2BUr3!V6_Lbx!1Q_?Qz$2)#cB2F&@&Yln)^+Db9>&O*Pj-SI? z8XV0sM7xNa=8KE(X@()pD2s_Qk?_xx(oUKICUWB5J~`#Sk6c3~yde$6M@CM{!s?{H zbXhR23P^-MuX&#M^U+!_pXv2}tC#O4?u51O)umR`H2jN@SY7{S>hyd5@j}el(y*;{ zgoPCi#r^$MYqyU;#my_jw}9m^plpPP$7v++{Fw`|DqVN-{4uwrZdPua)VvO zF!cldc;TavGvlkB$Bu=yPLMXG>sue@e&6^`<-}^6>~)kaF)w(NSN3rgc(?zwuR8f# zqok%d-PT8%__y7Mi_eCWdT)p=cr3<^DQ)`HS1Xqf@E0Bba?&ACccN$PQ=84Cuk9mE zW~k21p1ctU>Ntozjg6=62J86d#UD;z@w2?}kZ)hPn-oB~I z%1k(IX5I3wvtBPtb%AS2;S;m1;%oDtWotGOC}4}-M`ciP@xbaa=Z)|G&jY7}WllC{ zhWO~K>w%H!R>^!9A>!#zg46)G<71EcFI)HYnU#6XAA(J<>a;y4HJz#eR22EdkUkr( zqtoUW3RjqE(JYs_-*Cy&*!5cgt~1h8psMi{q)uPx+dj_^R)M8YmbdQRaVTE~=J3o_ zG%n*X^IZHt0L@e$EOGqSg<5icbhu)}oR-n}EH7UL-w~EP$pilcV5ry#H!Fo;CBySW z)6=&$HaGQdYuwCE=p7_&&$E8vcxF;O2sxz74Smo@VY~)k>iWxT@7`e(LnaXGg2`>J6y&Fl=fO(+_&eL=0WN%|{Qcr76X@Jb;^5n#d+@ zwV89Olk_$i6!dNtxyod3(Cuc7p$+wZIqxN_ARoM4a?KohEYm`^eOZY6SUL~WQVJ-r zCmIXNm{ne0EhoppRb4{IHrgkX2adi1@MmFY<5c|zTQP1ZcP|AC1fv!7Ha{bj!#OX` zgg9PT5?5EA$JlIP@%*>0zL|D{h8taN2C{N|CRNJvp)>(Cj5hPIK)WUzYuh>v{Myn3 z;`AD4c?b9`v{Fu{q|2)v<;1o!a5_f}OXMUR+44Jf!uk!r!6ka~sRxq(Dr}9DR zvI2+_3}qMPW!t}|cnxlR*6lB#+5_8FW6nY_GCG!#lY<150}2P~YXKMVjvBSRlm9~C z20ehzG$>A64G7gaBKOK+EN~Q9#(i~=C@ z3;%z&2P1%IPV?-&3VrWRsNP}+4)S&2AC3(co9aY>dOh-Ew2Iao>I9~Rm2VavLx?g!}i)e$D@LnCfk>n=qMAGV50W_ zQjA}1aHvTwF9OMCPsK5YFAI2;6*5{3NH(K%Z+*d`fJT{_SJGXQJo9uon4uue`CxAu zQKaD3y~&e?R=*rDta#nd7D4}Oi;$;+=5{l4Vl#lNtTGqOG8nph-A*!RstaA_)4NS| zhs;h z8-`DT|E}KnM|Ze6T98(gz30mAct^P&{~S*H47P2%u;=}`S`m;7>mWxdSGaU6KJW*^o|UUtSpx*Y8@TA+BOcRv0{dBeF|Wve@UeUY zE5te?9eAUM>&=}8;5)_LPC4hVZV)8ZE?v6x0yw6;{btvIS_SK5cR$tbUa51vG1T$g z$)CmnczMl>)ejs_)tWQ^;Q)+Pnx+=CpfQgR$LGWy)?raN# zt{r*+`w*~GC*`gkt!6t`UN0Z^W(9Gd78-t$p02)B1F*cLq-08txtX3eCZSOqggxeY zt-^sT2h!|@cNi5_!4jJyZu-?z*M%Z%NXgeFOKOjBR-J{_NtjP-#auu{!bf67^|kVV z@kB@4o?@c&IC<*Lr|#}X>PUjIl@Ib{w8$wE{kdSF^CZfw=E-t0cdAo&cKzc>Eebq4 zGc(fv-j`?UTT`;(1-B%*vG~cPXuvYu-U}g#rc3^4cyvKyC}eNK<-6bR?5A39=I3~aUH~!^a29p!S2XGa!068%8ApKhE`VLpuBW|_ic>2!fX_N9s9tqmyc>4F`X)Z zRFv-afy28PVL$3;;liEW-DPKr4>fRz0?E8(6EALKM z;8`1KIQBnl+Jl;{|EoDiTBYd!(Wf)0v>f+8D!I}s#s2@5%HQ($IZwgrts~Zj{|lR@ z%})*i`r2>%;4S~M#eX6B;x5k#Qk?U>T^>}eFZIc8Df~Y&JE}IBSaFkf^n6%m9_yDm zkf`L~Llx-m32py+SCb%+XQ(3PhX3+8=%73lDR%DJQC?r{L7DX$#%ZpqewGfv z0*`ITa`%KDN|)IVV+17ebeB$Jyn6Np9pz@X8+BX6$n(Ua9m1rJ-TI-kO{!LC)QmtT zW>?zLT7U@8^>Au(+2wFn_3zU+S1ze0UFko5J-eHocJj7^Bk)EqcZEKnDtDM;Uzc5_ z#U&qKSjx`cGnbu~L}x#ZY&~0;R)jFSS z?Y|V*?ArVKgxBWF6W9FfakPoi;D|Pl)sEr*X2WhVP`M}IYBK76Y2GS};r8uglPwJ@ z;CuAe%CK-m@yLqf;?W!lj73a-W~YJ7_>cD=UB}-5p79qO1dfs!t;7eR%-wCI_IrsF zx@UGocgBshWKus8?et(kJVXod;dDQ0hJA0<7kkf{_Jp$jC_9dSKK{O)xYaVi*%)#Ji;emIb^13 zc$Fw7eCyzE$Na-sE+xQCYl#~CiMtx&GU1Nf&iSWnkMRvI)A--x(yt~HU4obN?EMbW z_AMutH8uuNwn-o^DX@dK(tbmK1aUQ~?nPghYZHREBmlm-4DiMRV&IBBEB#Q|pXh>5 z3`V1KY3GxSms-02HF#R|xgAjICSC`|eTAjt!{k!@5of&RV}OqYY0FGwp|9k?o13#u zLzySx#(xfmt4Yhq=#eJQ<`>4)+QH@IGPv=?3IJL&No+Gx+b!MzYtX3g7;y#AYlA=< zb1&uPpQC}saNTTK`PUdAb^Ce25RO=GiWHS385M6qyuhhKS<=({v(K2FKIiCZ$Rp_CKd>Vpch&9d070&&sd?Zph8aL}N8H|F z8dUx@jlU4AI#*uhnJN&9_@AvdM09p9Z;46Tv*!ImP5*DBAbM!edh?fZ0`&c<2^5B9idI?#q4Y}v_#cO05XcQodb8%HeT6&t%eg-znl#k z!;YtKTB0=XKr%}|t#-kZkp;BuL*Ndu5 zg;xi>=$>)kx|Lqa3$bN94p(%94%5k3|ZPb*OHeCa4Zy$Q%V z`K`?gaqJ|1=}uFs9-_yTMpPMcVa@sm8}3-BJ)4`(lYggVIF`t9kOKDs5!}l(f0h5- zJ%F+z%bFdqp=wk$NF6Oifwy5+E-YUjJ%$gtr(;H-pN5IZNk=Vj^;XGE`6$00yHDSK zG5yoTFyQ)LKk)2OhijV_glr)bPpjAz#RK>iDj&8{#F}tlr`z9 zh1~a*jYZh{&O%n1bNkxWxmqdIz^p*0V_jdy4>>xi02*<&GDk`%o*6)gAfBia_ z?RECe`F9u^x%y&cAgZj~7*bcRq!Q%d==dPu&2j<2eh)dU_kPl|T0-TqC>Ir_dPisuMjUByBeK z>mC2h%tx%!n0cO$p(o1%afxT27ef3_t$l0oSV-e%MMtZOT15oAgOE}NGt^}=_uEZ@ z^6nBGFKBnq$Z_hr`~cT(<~;vhKm`G#Al>$<+&x`>#-T2Cs40Rb^oLKoZ)fjoW!N6R zgCtE`IFkTV{nAn&O?iTL&<78 zu@Q%NGQ=Bi$jQe%=Z@!WF1F5-8JnY@J`%I&?>9cDEOAbLPsN;;wC~jcjfF&GkCt&@ zC{&~i>i79lmxT+WNz(`Lx2?=eZZ8Kmmx=83&58$+qUbT=_i+ZwMbrNzKktL!bHf}- zbz~a(dUlEP4`%(U|8`;*<5BhwY1wGNqXj%~JKlUQx@*!oUC&JWwtY=5#4p)?Byzpd zo!z_7sZ}2z;o!)6Za@lsp+O$U$;g{&#>&`3rX?HvJp!!y0G{1vTn~ftpttW7@i%tm zUf8ug(B%O(# zvn4`vaU#MMDKd2b6#Z0ZG$pNZ6PI+^*uSqFCt^%mQoY(Q;?cnqjk+1rpCd$%GH3P z_Z>Wo;baWeRv()cK4l$o@=e>nQS0|{5jWHKndD@@aQDa6j<>1@rrkR>n!i*N;qXpL z4}?mRkqiAFzutJ>Tro3rg-YzGWCdD&vQrSGeBwEpOHxuQ-|{O5C7J#`E_J&37d!qM z$nNtb^I*25amo9LZvn)jmt?brXg-me_33Au}>ZxCRYUU)1sQd^HTMf1m< zoPWG@*(J36!!^(gA#-0q0X`+QQugXz<~*-|SmIdtW2g+MxxJp8u*~&s7aMc6T5Lik zXC659I3_C^2fX1snP{wlt!9s3==!+F_Y61ItT#o-f$K`A>wSEm{69N9s>yJ6?$#u; z&@mWW#{+Vry7<9I8`+hdP!KX!=r7U1X6pbuMGI`!)|Lr0pGNmY)+E!;NJom6vfo~O zR2ds5_W3{L58Po~eIqL}+jP%Rcs!4AU>cUmApORK{o2XMhCN%LEpj4x^#_5v4iik9 zrRr+xxR(<_Bpl0`@kk8gxj${dW9z~l#+cxxLwx-mAeHOF8r#O?WiczgnQM1%zL-o=o!$$9 z@~~TLc*q@Y?8}8kwKr&o+E4wI0sM4|Nml=-+X~WGJ7sG`{>l}Yi22%KM%43({N_Nu zlXQPESaV7uEq(`D7un*48(pu+Um`|hL5|b!BrUR$<0**5y~y?FTXXEzS=hXziuhS` zZIf0&BeKFCi|FmZvfT!EX*}3T6^_3w*{6{(3&A(`Kq4AZv%?$)O-WC7|}Z(5M{qjj*ilN zuV1y=;N8_6kY2Wn(RAu{TFfuU;a_a_zx54)?PsC!WdDNSfI9x2p?1X4a5$h;N(sYv zp*w8iFZ?@iOnJHT8^uhC@hFOUtN$Vi5Z-?{On}}#CjT2cguoJ@u*Jp2h68`^^0!$q z<7y0WX@7Cq{|5Ab0nN8fq=u%2n-{MC-8cIkME^6$j?qdJ0olVKTa&Qw>}c28)BH!w zr2}TC%Sw{8|48KtD6Ofrqvb(;~?OGaH|nuP24ExhMP@-*0x7<=lL zY@>fvF;ZTF5CvnJz4gC*shs})ehx5a+Ksb)XMv}>GaA5&%XL-l2An>PDyNNCrnvCk z9|Htc;b4HGws{9Nxv*JpAAkg$z!Z39%tD|sN(9!~7dFss?w*}#?L-o8EyM&_0P zc@P}PTu*yD7_g5f5A?FO=ckX~icHmEuK{;x&)U$JQc+QnKhG5IC(Q4>k+{2aaQotrH78L=h=yv?GNLjmkQDtx8$E{_ZxYP|g4E(&{6c8A8AKN&qps$O}tIA2~~E(0>bwY5VvlS+KQJKX9iE|zAnP&B#i zoLK(7b7TS}{pAJB?Jo^{)pmcd_YqfS2BGfi*RCfbA5gz`KgdV zyG?&^u{G&pBki6w!5w4+>w#zq&}Vwy0XY^LDUz5y3o*0*w~1}gR;|V z>9l+~%G}Z`vn<*^huAb}X1oGP>dHbFSq$+P_wD5@v%fraG}0&44qNc{$syW>qd#sa zdV~9MZ0IG~VA0jH1AwaZQT7B;jf{jCwP&h-<^o7-`=op`X0sK3AO;0CHgA?e3&ao4 zIDrVP>IDg2WO${!p3nSG9a?xKRT{+NK5X2>a6{Q1)tE7zZu z<;3ySI~;n@#kZG#UEKRQxZq;bOIcW(f4{hxChzWx@bQ__ek5ZnT{ZvQi1}XZ;}nq0 z`|<=Xl=bdwd*yr1_J7>{$@$1Xy3s*JKXQsFc@QU1lKO^DnL+o>!(aS;-xJj?wc$xT zbL4zu3{3kb*kgMRuooYHOylLL?nXvKJaXqiW_4)v5`V>(oJT-~LpZ3w50qAaZsmL> z{Pnc8=;cINX=oB@g_ck6zVYO&wo*HPVd)TDP-5Fs(4*IlD{TdB9Z{jYpBbVu!=Sap z|Nrt#4XJ^Std$?CfH}>YGI@#5@6j_bh@1#TmcC`n0C%%pc#BI)Og$Dyr^d&%{oL{4 zyF0mb=jXzeVSKF2TH7hxZP&$swYni&MY~$FbEP-&2kk051-^7}CswhW`S70}>X0ju z+w}CKqC_b8DFaU2Ckq%3t6N)Am)$?dk|~+i^yt*USC3&G&;CBzT>Htixq^F)$SDtJ zn-PiHiQLoM#h%yrsBe{c-B?D6TTjZkKqE^E4I2-0L6!g`&nbGXfPjh#IkH*6Ko+B$ zKy*~pvOIiO@Lu5l?L=(e#1csBcQ!*lZbxB0e*tk7K8$I1!pquDkQ9n-R2LlhYGl;o zd1c_f@wdA>WAF$FaA5T(P>1S2nos7}9$BsJb;uX<=sHZg#^=U#;B;S3{=6)6UKyo1 zoGW-W9sGbmUEW0*@1GAZ?qU=XTokk@Aw1Ld_5quvC5u5o$MOBHVB*7)Tg-X=e+nf? ze0XNbHtjTP3`R@AAT1gS4h{hU0mWiZ_I$8e#Kpyv*4BIrfLRO|c0MGZ1iN?i0Lo>Iv>V#{E@px?I+FnZtoBUGp zAr47l`91KmEM2@H_QpLN+HF6#DewMJT9h`fuXnhqLE?5`Ts8M`sErFW9h#q;lPrUE zDhcmIU_PK}5g02r?Nb$7Y9$J;jvuD;asH>o{f-}k_@JywH7QaRN{NOqS;2*b7q?6) z-lZK(G8|*GuSxUK@RsK0fNOT91(<5ap1xzBU@o6u(JtF{z|xLK?T46A1k{Bo%yR8Q zO0?|v^ZMj<(bH=A*5Ky^dU|?7!3Oyd!f~PIOr*_I`(r;Ak?$61cLqV(y#9q7`L$sK z7n0L-J)Ng()N{U$p78U!#>dARC=;;eRZ-?sxT^$7Dth?p^Fm=V9jyP!-QCPBo}(x1 zrY!&7;bB^b;71uajzGo5WM$Wr%XB_RDxcvz8;kJWyzd^c^1hr4mY1{2^gM^4;P+(f z)(p1s#qE_24DF`CNzwe%;`b|k8zg@DU{SblBuA0t!AfNcIH7%d`E|&TVKBbngwrv$$UlMmG z@kdkyBv2m?zovaoFot0_B3~^>u_T8zZt(i`>(iaeAG>FpJjVWywyzGWs`0r}!O3gvdj1H(vH*mwmP8?N6c)?14H7ZGTh8*UB4t# z1_skdzKsj%TiD#^c_@x;11+O-TGZCrMD4NZ$nX6ROM~ZzLO5!?qT-4CcO{8;g?&4>KexeawEbb@YHFp`OQ>MzYEC*<2-E7@75O zWNCPdE+!i8Z7p;DW$839>rOJ8F3`wb=naXs!Os5s{5!)%PogrydR9j-v13znBb7Wj z#Kewxr>B|-(X}geGAYFefS0+Ro`PdC)HRwtsYz#yTzRtaNIp_jw2xQcrC#3n5>gWR zja-rNK%w;%I(q^I{D?LxGIDL)u~@)LrlnFrr)LH5*(gY0>if48kuuDp3D=_QY(&sqk* zFL`ua${T?LmcEKooxD%wjhl__5bxfQT`i1Bm{#k;l5a zh|z5Peu@PTV^gf%LXks_s1%{6pIC3PeYY6IEoo~9q_ZlA80e#Tuk;b?_tCi9F;>2? z-tXwhoUE-Cr|esM6~L7~lB1FBBC<8L8BZY6%1}1qv_hir{zH%VJX)U*v#cDeY6E4K z`gLo9i8|#g0b}?XL|?IEi5s!QGdi)!;#Wxdi#wi@X(XkkT?jbKR{_}VQ^y$6(YI@}hJq~6h%9!qiKa1AzDW)9+IsX;wG_+)pfu>TV}I9bNno*P!; zSsqZ6b)3`QixteS6cNltO#+U;H3~h4gga;?yQo~~mEyp0Ej6aB9i}IRBSl{V=thKi zi)~DZA7H6!kE)}6bx&X8U0s7SF6<9k(71YubiA|%E+g76` ztjQR?LNU{%z?HDN@_=i@m?NfAll6}j&-y2uN`2y~d>7;18UbayM1>f1uSX39}Y4PicaLHmwb>tJX zKOZt-NhMQN#><{tYzH0BKio6jNELTIU6GBYt4P<6?%{hlk&pwgXG*^7c2RYD3e+^F zMPDinU;5%38xSNvQ!u7V-f?;QR7dXV?5Le#@9_ZAX{|%5mT#|ZbR&Tv6b!Wz#6Jxs zqjLlIuD+iaZG1>Zs&ch8NttH!@rQNscz5@5wPSjy{wxq&L-|c)h@z%`d9u z=6tkBFaCpqOi~Owi?-NT7rZy+>TX#RD>NGdWA$O4ji6_Z5eo5GoHyiwM3~H8XQNJ# zZrEsKmy<{6c6Wt_5-@*p%6h1 zvE3q#*xHrwP*-|)J9oWzG`TRGfbU!40`2RT(?GIb3gxa6@`Vx{<#h3ddC(9kp(LfN z*C|2ykDlm(^g>tzvQlE_emHA;^Rpi#qSqfh7J6~qDd%!ZdwpR{4Um?%FTzMlR_?Fa zPR6-f@UZ;S0Aq<4dab=DIGstlVXI`<(m_wWAjK9z&fPP}C2Uk~9sGTrGFAO2sqi6g z9g^7|IC~q94ulyvU@-d_&`HqIixBC9W2rHAroGOFZI{qP{Lg$-$MO4k+>$! z{H+rQrd6>JMcoJd2Nuz_+F*`vM|6E)9F z3LT9{Honnjb~8S6(^(OJqikc-fktjO^G3JF&zKRDQWWNo_4f%iZp?PFyMBlfVB8zd zQiu%ap!x)k^^y;@yfB#!4!XGW`pLa6xpoV?Y&DfDW9f!bIiz>wQO?Or;fzi{8hLuH z)LV1k=~kh=Tp|3d>OSx`Qhte?af=Xm)&#=d%Q8{1hHf+UcAp-6>swk|rJ;-b+_ebS zJLZK9YeCuzAArgNbV?HxN+L-YbAZx|Iy{nG-p+b|t=^vKawgi}voVemEX6L_My)vY5m{RcRL!K-IZG(2IGl>CN6tEd-Ed^FT#8r(Qi zRl(F&Ljgelhab>RdU=wZ&8HPavuikU{EVK>;1Q21U$(K%D#U-C_!=nWrmewgJ?Q~2 zQV?7`eF?%lBwqzdKrH(zVD>b`C0+Otnw+7LD|)=rcyN}}CO!}Oj~7zT-umoVuN@Q{ zepv`9iRwpqz|9cEqZsOo{BP1b6yI_bx}G!%DNM5tAPTrf+OIgpULE43R&`8={9wBd z4t|@gI9XH`yxhND?=$FHH@7x?`SJ$}NFOu#;+b%GbOPt&LJR$i8zBKE{99p8k*J7IuHMHaf&~M%O0gMbGy%u~Uj6#Bu+mq4eGbP*E7u$P@OQLcXYFM`ez5 z_3A|D(Ah-_>l!xko^~Z%RASioWTn~Q_{<%IzwqGE`1_b9X42#39^3G$^O`J{P-e|1 zr-V^;f)`jlk8GM&H6@I0xa>YrBeLEvTBfcQ;pwEtItsUtl5am8S$jZ>0ocn8z*LnQ zn&nMqKdpN;vIaP<%4?sYX7v@NEPc0!8gzn|*?R;c$kQOa6qEcm@nz90O>s%ZE0TWc z3xxwrtGiSfawP%uvHnY4<{xCBNtKVA@-(+-2?}g=zzco}6QD+>BYw@^Ipf=x@A#O; zE8g-jKQ*QTQX~&R4vY&hMhmwD4Pu&KC6BxywwYuf-pjLvlbGVHB>S{c?pdQ%`)3=g zXjg7yVLonGriF}0-*!+sp*X5t4~(}H9v zoK?F+ZzKJP+mCIZtdsJ5w2)pM%)x!R8~fqrvOlQ)DM30XFH%MPDD{qOvSueGv=7I% zh*T9c+UvTOa$X#i#_I1fxQnY%0hJT5I(^H0x>c8yDe2-Sp@tnUq~ZopE6OzXRS?M+ z@}Zs=OfYkYD;e@17k+`HXMvf?)FvVz+#5TI*zD^C0izm0R&@>m3OXFg#0N|x(weu` z(w2Ll*)yc_I3#99wjGE9TJKpe*z-P0&6r?qjJWZjkcVjtaTNp-RaWqf@FFIy=&C$^Sf#TbjI9 zvWQF~v8!_!&`P}{f{qr|PmagWGy@tuBqW?Q z2whg`MQw3acR%w)ainK?LSg@cb=!EYrb*?KD5WznBNPatzvPFJKo+i*wD3DaH6_Bq zc+gLa$1iE^BWk|eP^b`#7T8Zg`u6ty0YFI2V;{^Hvf_XP=a?Z$)FGzfcYr!sQ|wp} zlo;=yPM%S-63droT=bD{Y0111Y0K({-E#BGOwKn&3sae2n*HVUdMHiw%M-==E~uO`w2r6XeD3L54>4=7k_s94Oi9G*wwelzn3XIf3R# zkn%Pt`F*l-F|#Or#~}~uE$xBA!d@ywf~~-xp9II;)CAE1szUFC^H>iHuWa^-8L|Oi zCXfMzGj-Ky8}G;My3KcUP)OhcaZP@IaZc7A%T1*hWYw)5dy^$Tz~ zcq9nX92n&8d0btHPXlNRB&YH84i!{@22IGMF}aBYWv(xJd@Jwlh1)81N~D#v3D1ym zvArFjT9{|(aFCRF=3wDX^#XEpNQ}=9L^fz80MZpyznGjl=k#E{LN@*7!)z`lhrbpx zAY1m44d3L7aKzqBG!lwJo)n$mB?IzqiQVs4(QZmrt-9L8gh{cvvU%jceaoIsVqtRR z54zK+WZN4GsMcFhb*6n*ihbaqw*wh=e_LL}ydec{VAsb5L@C9>?b^e6fee)@D6D(~ zdG})w=E6hSup0thff6GNe-Q%%xbDV2k}$vKRG7mZG5J4D&t9NZzj z{sj`8fW{A!KLN>d6`$Ol@c4J9nLxH9E|QXjlF=F%aU~Q1QNFpp8$o1a4|_g;#QBN2}R;8sd+4pdtrz}J=-=CE1U)!Sg zmCt7P^qCkLYR+x!JFOBsPCL$iAES?&$VgkbTI9T^`f@(^7F3I!)F!>)c29Q&wJlD_ z`x6MABaxfjzB3Tr901q+Zt$xcCTb3^h7V$~)Q>+zW`h-v+-G45y!t_3_EAf+pM9(p z(iNWJWf_0pDqba)?}88D>az==i?b1i4CMcsQCsXSw_Zav&Tq-C%ZbjaFC6=^H4^7`~d3N&#%6bgnSnjS5o5fd+)Z2_I zXSX422bBY!Ks`eyuaC5#ZCt?j`st`A3`gjI!t2|Lrl+)SS{gk94{7rbXNVzPH7Ha< z34D{Y-ejijWbgJ$qQXxD-S_Lz#$*=`l=O#Bz2mA7jOPn zrIx$!7gdYQAPm@hvLnl3kOJ3^ZX%%dUf$FBGZ)Z5qKe4s#iHZKQt!t@)f9Sq|&ar#D>)i8n2Z{nb>?P}}cfVB7B$tbauA=ReY1x2}BX zeLU$Sp6;t7VZ-d4#iqj0em1uIwkf-Ed(*+DnN1T699BV81U64D@t$^Zzks%zWi#q1f0Cl+J=b-7Vo6f2#5g z2+kGBluL=mJxl5?VQxb)6@Ny;6Qn==HbyUaF~rDj)5}A5^PSk78K4ARPaXGoNte(*MC1*58E5rO;5i-S3XaLE2}5b0s28mz$ZpPxd;Wnb5D^S zY3r-BbVPzI%@414g_QjBE;`#LeXHr)w>&H_Kl*)zz!h2wfs_aTCA%2FT!JgoM*ICe z!)#}B0N!-1F9=Y&{-u4zV zF-&)_@Q^6xt7cM?lT;}Db`G~w_$>9%M@w{RrZ%I|&Xe~H!pmMM&CClR?bE)KrI4W>rF7rZFa2jANO1$)LZN*I)`;z! zUjjPj+b70}O~l0|KlG7sUg0g%z($2-p=Z|(HBe{9%2VQ}e^Ww)gpxrE~yC{663z`m!Rx2S0M5r2F|3lF* z=Cc!3vcQv&=emb?=C9eCx~=OUI=B=+j{`VNfSNRV?!KW)RVYJpn_U#b1izWbGDeeO%-_k{;7A zz(+MkMu@*WtmfL>T!EAf+oS3e>gx)AQJpUL&rj=sk^Ly)5cwygo|x%Wev{pfYHj-c zmbgT@fY2DR`0dG-O)S;&@X%nc3-7~e@!TBqtuuqs@pPOg6KpziSY+2R3l%Vo?qi^Y zmJg=rsN$v%zk7{$i=58QB9B<+%TT12Tt;kn`@i=Bl$N);qA?S#xVGW@11;;a?3K|o zxAe2>%@w!wqxvJy;j?>Bj7*J;xMd`scwXRXl)u>Jp3Omr;m%u(jgkWci4i`?4VP^mX&Pz}r29$>gO=_MD3844-p* z_`n3H(cYIyr!R1IkIN&Zkt z7I7Rt9ngPsW_m7_K67R}6>h@x<5K5y+h-8iJ1FJ2VjuA+;ZMtwPalbTYpb8#j8V9zF`WtR=I z{*B@sx8;=_HiQ`KNuKs*+M_lO_ z-K*cXoR}~sYr{)=D-hP@#oc4#-Ft81=8C%AIw7%fcc^;Vp>tS59OrIa0&YdWFjIH& zOO4WUY}@f3xV&X(ENB>gmTP5{Gvln=4WwBeX!rfn0}_frX->HmXTC{FJWc9y~z}xx6Y@ujjppZ>sIWpG7o)(@=w-|_@Ct6 z0|Jql56U(Qa*}uJvtD*x#eXiYi+ydJk$5!Sb7Yd2QIz$wA^ghtyz2#X_D`Od8mUTJ z6hlSXpZcGZNaJagRXnLFY&@QMgKokxm^X;+cvv7w^<30I7K~<6E`O+6%Di~i^G(6# zvGDcc3TP}eG>4r#1hO7YTqft+Dt+VP>$`AqR@j@=4{TnK?b8eT<{(Ma7{=*)eS`w< zh4kA>8TYgA2DA-)n(kp`pFD*KSDND&((in7;T~8G=@;Z&`I;1vWtgDzM#SW~UUdka-}={VmPxq3?9lYHyc{Nk z++baMb%}S{)_oSDw#TBe@loZ7euky-nMcc`H^xRZ7~l|EhwEIgKX z4;_xDdr;OHQd&tXi;R0S=!Q)mk}n!cC}n8{?FI-%cbxdhSM@f((RIMlv~va8-1BA1 z2Yb#L=tfT6)Ky$%)blTTTl6N5Gxg3=6hxL#LtM*k!#|62DQL#u?k|FWTp1_cNgB=s z?l8*Op|gO82bYn}XLdIGP7#D>7SdlF+-SWfu&?H`*9@b-iuD@ zIoD?l8WSQ%P~3T5Jf6zlU$MkorzIHg-&tB8527Tm2u_4}`|BjZ;bb=G=>wygm^vjL zXNh(d7twP}gy)5sS+iz`xisp@LF`1&KMjUTX(C1 zPqB}lk9AqpuxT#bC(IgOR4=>bTRZ!6)M;R)QO-sJleBW!$LIR>)|=xKQHQ73E_d)2 zZTsoq9=?pT;>p`J1hmQu5DM>Z zcR%<(&Kn1OCik-*liC;|c^NAw>kQF{M?d8qYiH@tieHubV}M~tLj9j%M;W_diWNAK z;|ju~?{t>bXMmQT`l>k;_jKIy_30q{Yl;yB z7qS3D>+`vRFtWenhm|Qe+E9juLWIX+_3al(ZL2a1kuPAe{MaE-R}PB?p^NS31DwiU z*hu{OEiFw>#r`sO(9bO1_FN@Ob7f`lv7;yKiFr74P&aot9RX{2LFLTT4s2|UVxR5G z^o|+=YOd<&rsJ;EUQxm`x8(XSi`-S|18q7QodRK1|8^quAHuSs?7svJ?k zTW#tnjXaPHYyznMuvl|-XY_NDfc|QQVtJv`_XhPdmS|O4E770UL*`49I>E3du?#Uq zX};;6A5g|la#iTzi;Mx`nu$3f(xI2CW-)Q{>5X)Go5|8>8)~B{Jc`wI!`=5^^!0%8 z>?>1B22`g)J$r+Eq15g**U&^HL=qh zAJ#PQR`kY-Y`91%sCJaTX}&sr7B%4)g0jFKPW}rqX+7@SHU6M}pNnhwuwbIOuJhd3 zI7IWcWfZY_#14q*Y``1tQl!LrzS_AcXQ-;GG>M!;yCp83&&*phveRU~$>~{?QaZRs zK>%hY@NsabWAAbnWuRm2=s-}7p(>1n4g=yB*L>&EHa70LdrDt5q=y(yy%05tWj|W| zIVO~ZE7-@)yiDHSVyfNpERRQ0u$hBSh^zvAS5MWWB;gT}M1V&(4+ySEYl^fZvG#!l z&nm!a6f;(l2JYb(yYIvFRClyVcwB|v8(&%M`eaWirLP>I3=mLk+vU-L`A&1~-}Bw) z@|EX@o#dFD@h^a|!Iv$DlRA_G0iv|%7>fh@#gw^2a2E4Y$KSq}M?`&mC6}k8h_7uHJ7hLo+6wz4;$}KTY6X^ek~Dc=tY= z?P%4hrWa=YJ>hkVl1IE;&nIPDI9l5o#m+d_;NJ3eAu^+Rgqz!SI{t5pIzJE27zw(+ zEb3zd27%Di>6n-@7-*_17cb!AL|P5{Q6|4>ZN;@Yg@e3vl8d58ONaBHS6&aAn27(1 z4;&_YBGy*S_@ZFZ$!t`FE!3O)F?`S+D~cI+Gc=x zmh!)iN>@SErZ1}PB7rNEOO&JgVGAXGW5Pa(CP`MWx>r$STyec1=z)=9(`@5PSsMww zvK~qAzn(Wc$I`KgBr$So93A?tjrF$sgXx>47|!vbU-^Efnya=;j7Zi=AWyiXGQv8p z75oR&VB0=01}mHFt*_F+QXU?DCitrMKY-%Xfb55T9Lw$h%?Gt6 zyIu=WymG*pJ%%JN0=m9dhrMxs!1;I(QNS^(VORFvV5an~inpaq+NF-%*SZ|=dGK2gYj|*<$N%9p zcrMb>NZztE=@hG}5duFEn^@_kKZM{cQC| z+<$oQ{5W|lV+TY$KbzrW8wV?O3AJ(qdIw@ru%ZwT6ndcxa1aJ4*a4m!y)gsAhT5}q zTP2Wr1S*VpI(zebNMea`l6$dZ!>eq zaJp3M^P}I2uzF!7G z5M}*?dEC*G6g@C^+rS#F&t_-iIT2vxQbAXP$CwfJEa3tO7l0j?VJnHm#B*=fh{>H+b|DiUp497J>u z8VdO=@qeco4IaEO>^p$p z15blqa$kj$dX0T?@2dhdpn*Jh0V^LcPpOzXl|MxGBX}dBmRY30ICl61jP+;Z^Az4- zfy>4+W!k?D?a>gw=u7=te6#8>3+t8I@FJ;N^3wB60UQ~?e>1<|JTQOsgZ^Q)(P+?w z1D}LRG;WMO|NRb8INsvh>wg)^{I?zuZ{P4R5v14^kx*tdE8+pbT zz4QQiZn<;5f}vCYZO;qk#3<}pI#rl|`ngO0K5P2*R>@p{s{7ads2u(tYz+L7@s&Q5 z8z>^HNPo>-JT{QVXTzG0Ngw^goxl~1kT0;dZ~PbEa4~pckEGxF-`wZ=!yY~lk$edR zzt#U4vSm?G9*jy~UH>CKWBflnT!H{zOdMYUQzc&{T6~|4hrugN3VYauc1rrcvju=s z5>QduL9R@Vq6^2G&;9Y=Kv1_wfxCOnL|Xx~dz7e;JfXI5k>+(EpYlUl6;zHp5i;uaR5@;grSWcr{$)}2O|Mbe}C~`PPW5)A z2-)O+f;uxc-9AbW?9Ta{E`YRgAn#c@by&daLl^1o7i%ZTFdsU34gcFaRBK=f9J%9< z?ehPPx2W#>32aAkdIp8ri!c9$ufO5##lo-ZTB>38|HWE-0#WV4ew%*fhlgZ`?;M)B zGm}Yt0p$lt&TEbMY;3zewwiL(Mcvo`U>qZicg(4tdi&V28fYM(1?yQ+(}DSKBm zi%*Cd5Z8OhLYW3iSP;p@hePElRRyN(buic8gM1g8p_JD=wr`V(rRD&ga4+2WH7Mr% z3w#+t{tg=|#mHueOROcY)BA%sj&Qk(hmo-9Zr z2{Kwm-%mPTzeTPKk~~<4^o-B%9vH;fA*U*G=H3MZK^+q?AM!Rl1%@Qhoo9eHL5Ufi z9riP@*Osw^c59Hg8A;~EBJD@xe^XUsZ?9hSphJAD!iNuY@G8Fk!NCP+6$^yUfX;xl`z{2?hA|-Bhc&7G z4bJ60@Vsi^)$G9a9mBaa(2Ckf@!)SOev84^+_s;DDKU(l*J{84w@_+VWa-`I9iRvT z#1;dT;ACZmffWJSU2nFmP6B^VFQ{XBi~&x~i@`g4@2xkI0n)!S$!ijLGFAz2{%=(P zOm{0+Qu(?x&_s+qkO;b1X_O53d#66~xoo~tDo2BFnaoMNvhY9xpnmN){CJC1&LHKh zSJDZW2VUGK@FAdIR#svm-M+5oE$)9Hx!l-X8XEX)PN!QPwc*~dhkk#U>>k-JYZqKA z34@QpL1UZ$Y$_mhD)49uwQ5ls%mpUNZh8`sP{Y|9Q1Y|~17LfCm%h@NB&&x1!``L2 z(%&n=fq)GiU79Zjv4o~tb_yt|@@ltFkHIr603+-F&5%B`u%3Hp(z25Og=0VZm?s9x zpqxP$6cW9I3i$D@-(d0DH9x|UVV^Y28-Nt>{rOdb6>6(SG&0ZMO}loj5o^Ow*4^`( z(C;H~4TY-|>jtP!PX0cLuHn^w<75G=88*njxz;u5ZP}W)-$42G=5Kxf|NepoTpNCV z{{j~SRn;aLI%%KZn+x`5;(ImSQk8{Kfn=QGcqe^LWA>97?Aq@JuOrmbto7RdxtrJe zrCaE69_Ngn=EkUF4h4ikBwV&3N~%OS+2IoE%T$ONN$o`|{alYF}ucEX#fA z5`1%RJWq<>XWPJss`_|v?W5a`(7VrEiGfp+|7MMEit!2gQ91(d#@c1Z-D}d_L=2_s z`gKd|p}XVWkBAFGog=aO9<2-3ZZWYqGNQwa2R;F}Y5|o6KjI9!ayPnFylSQ?AibpVH-!F$)KJ8-lJ_9FpdZ7}!H<;?r~rTm9@fO#0fFDF8HykFKF#lr)3>{uS1eFcUK*`!?D7qVfJ4OopfUx7N^) z3LO&V-M*7qn0bFF(jO4*#7M^u$#ybtg}+E-0LWJ>x;lDxz`fTwMe*(%&a-x;sN^+7 zjnmXHR8@N*uF91!KMR93f-u0!K=H5$Ov9meUM%l^`d8AJl>ci}!wz+5#6$>0$rg_% z=ek|u9aReKE-Vxq5G3ke)jQYv%e`h_&g9K(VY?_tjf;{l-m|2BeI8-CaN$>bvM^x3 zaRc~TG^l7PKxr+N=|}8&Gm^i6T$ZF*SFSKuWcqkRFdXlvCe9Ewmdn)C(w=Hv8*T6B z2I!i}Kc<^KaCoCjsxvh{^61%3=!MlJi-uMgzCv8nQIPf6Ehw%3XW6;RBu3vnCM?~I z);wMMGUhMhjHLi<2xjVy8zNqVOnNerAtAOyXKn^E4Tv?iMwX7K@zmE!Dl=zY>hAZG zjJ2Q&2#7R`bxSD|>R8_Q|293<;=5XFsnG-PS7z)&sxpzm!3_^w*ST*zKJVU`(WjMiKPH$ivn?1Ixf(xL z_4qmKT5I(j=-MBkPFES>*4gIo8JBxoUNYBtpW#E-W!Xtaew3l8?M>@*CrZgj&e$_G zmtXp~HgaIG$-^TSZ4>4Qo9?Wd+QB=!ZXwv`25rDO4Jjz31bhBJ;oJ| z;_8p})l{KH@+jHU2u`~v+;iTKbgGCKdP zXE_ri!yaF8f`Wq1^orlbvF|Jqj(qj1OO2_iGx3n2GdO6a80lxm#KmpT(DK?mO@F;6 zpHa^&E>5*#)?Qt=X@scXAq`77%VlO_Lfe{h`o?uoxYt{(X;3C7-!@y_d*-JTJ*Ifu zMENqeYbBp#8WB_V4?{{Mu%zq#1&os6q%+^FIw6b9A(31;tUu9Ue4~1Kd)tPK^?By5 zRecoPv)APO(*{rae&3=>y*%sVYOlkVruSCf%eX_7A|rR^x%=04j?hwmjX&3BO$g+$ zhPp!M{Vdu!f|j`gWkHA-nP?={-#7+ca~Hr{fs%^3@vc)JtOzL-ZPv&0BIUSZ%DNiu zr!UK4?tvI~7t0rtQVH4A2!Rx~Nc@`H`rC||b8~Y~SRbE8&tOBXGT`6RICnGf9XYB^ zf9%uCC9T-1mS+rFXiXuwnOhH51`Fg z9eOgfR^=2h;RdfLYA>Uy;wtI*Ge(l~MvpCgZ0j2t7)YJod}h1WQpSnu+P1G=ySqm* zLIFQ4M)Ai3E*O9~)8(9Mec&26v68^0Ka>+qqMYdOU9-mpy;ZXADm>v+@s*)|o(xHQ z@Kl|1yW2`wilNn2q^b6lsoP3#&Z1ub;`g%=JJ5}3B_ zqn|LpGS76GKij2JWOA3_wG#^*H%C-VU%Sp|@w<&1cLxsC`pViJsMhgbfWNF@UAZ)% zUod9YKjZyrGjU=>iA<_UgU+(bqkqF)Vkza~?Nd{~^Mx|Q$vV^AL&u5?*Oe|+gOYJQ zL`b-Ub@s;}6_a&aAGC@6@IW+-+NrJ*v+Z5Ce8QAc$-tX)tXxc^4XqOyzML!nB4_<~ zn#aZPiE~DshJ$(e;8FA~dip~7ybb;e&Qd3~jw5R+yR$KLCn~42^VHocN%N;%WlszA zO|Z=1cQZ$hltDX3{5;_O3S4WrxVX+?D=Srld**vmQ<5a)I$=_UP*936}PU&*aK z2e}z?Y(9dWF|7tKzw@9eleZZCLtyqT9E!{{^*flW?Lz@4^w)|^brgp4avjgcGH#z2 zPh>P3BsY+#i`+UotqTd*+SiHp%#+&W#MQZ6wU>CTI^}RS+S24)Ti0*@Q6dpFNj&#) zmf@xG-9ZFxcyyhVuV+^}sHakZ1+#YmeC-nD(Ed!Y8#Ms_*Z}iYnjDtDofwb6mFau< z`K#nsKRMhRbGK3d-eldq55x0zd+REE-J(IUFn7+e?a9kcwTXqdGs_Wa5Il6w$T{pE z%iSTw+EDmhY*GA04l260BfER1|HB(gn(DM4^SM(_@37U}UGPJNilk7)Fex)5vTPD#aCTZOQ&6N6;1S@I^&21z2`Ji(LvU~VA z5i`h4q=3(!JdzO8-XX$UzqcV0Z!oL>RYonSOP8=6>v{v3){!af0{2R}rqs(mq+w7M#^f*Zkh z%+2<0{{Uhvse*hWApO$+GinvgG^Q)iT)7!l2q<%K*!Aw)<(ZzsJ+tfE?d8QrQBQ_DYk-po z){cU0>ufKZ{1Ov|Bq?G}r`^}7Z-cExQ?@8&2nPu~_%%Afa7_QbY?Seh97ZHFU29d7 zlKLjaMh&&Bz7bF9f%LqrwiidYLy2&^*DmM=#);$9}^9&EgIg=&MJT6x?3$kvm zXY2qRJRf^sAetBQb+DQCK-}U&DF66v*NW2aSaZVUPZ;whs^}%&BH5a*=J)Rbw73eV z2qo=!&6Ll!nXRTev0w=(=BU|C!oURS&$uMZ1TDIyzI6sEi2U3AbbV9R}NbRAt?6vxI$gOq<+cvxlSLzm zqvs^8uiQmwK+^sjJIsi-yhT&hZuL;e9&C#FAyNAs1Bsuucedi9|uu<%i zyI{ct+QGDv-m$#mUX+6RXWqpw9n{&4Ts6K>oS4<(&uy? zY6BHg5%INUENm~&{oaLGPb{94dw`87j9>>7sAzl8stzIl8OiAQuF>r?VDR2u-N}H7 z(GC(aD%&uCoj_V(V+;Xce^fMsf8PTZR53%2??fAC62{9bPG2&xUFLPIuI}s+>{%=% zQU$ebG~fm7yfL65yZ%>m!J-xc0eDwz8b_*MY9?E^3fPPC2CRQU%On96Fz1TczXyUG z4}SlC%axzrv8`E z)t3lQvC?aG*7d?UzDgM0Pk=Fl1;%wa*J58AeQr>RM8krSor?SLSFrJtCd;&PcYZ%8+PZj0NIWY_zp+ zZM0{5zie6Y7V+h}a3T@9@FJf%U7)Z!DwF zH*~R~ z*fO6EuNSK);<7BQx~vbwP2T>f0kRPU>2`CuH+rq1pGveq^*hiVvEPb{bjW^_%X%8M zN9QKzt#e~m1Z9Wo&phU*kz_DzRZS9AC&he5()HJitXy%XoxQNEEUmCyq`q@>fVx4O zeza$#A_!aPzlSaVQrJwDC%U!xK}3r1ZS&FvYGreqA0NVe6Nay4t$mlP{c095k-PgI z?bfu*OVm1hF=07!6mkr_-`hp_+V9+WPoj3|k$}PA!=Kegni)DikPkZ32%+meG*jqa zn5ZnW6f$u9+%%+;!5=rDmuCRn z$D>o8NTmHwyNhbvcShPp%d2%=Gq2CoIf~6*VEJJ$&}RKsp!8R|kFh(~?XF+a1`QE# zQQL_HJeTdr>{3aq#)AsMN~hD~%tn8(wT5=v3r!|)Crb9WJU~|do8S4{gAAAMwU;Bx zX1z%P$3L6Dc^>!8RGn_q8+cZC4b|zYJYGQ0yTW#<2P+3eNdLP?OUUa-yjasKweWJ7 zj`pD9B{FB6tnyUvcw`WDl-7KPju)oO6}Tzx+8;L&ErRb_4a;aTzb;0TXVf|u-h1zS zmQk@?(;B;-xmKrP!PeIOIbr_RV_bWJuBwb4$BUlk$CVcxI(FxqNVVV|u9w{NOF=91 z8M-tm=b~Vr2+OtqtvfTrQJ6b$H_s!#L~+`5?o=5@9zEI3m#PfJmzC&xUqE`G%``NQ zJ<>lmP$)8MhbTxrTdGuiZ60~MsebxIVs-=>gBS1VAC`LPcdm=5-#%iNbe0HrUcUTp zfN5ZjLQd!3&!2%NbJ&q|c26I7>%wLp>%8zyq~)x1wC}Y)twpdH7$%lh!$!r?n(g&+ zy=>C;1{lAHV~cKbkMbpKNtbs>-d{J_EBd*#i$+)at9B+FJ=UyNaFLyuJ9@n*+u94L zn;&l(uBlB;T}T>M9hwtbog^H+W2vB-Y7wHHo@M9+k8R1Ia@eG>TYaYAP=JL8hxz>O zGvQBBPTH5EXytjVQRL^4?3MBskl3>OPSWKhm)Km?1=mK zyuC#xGwo&q=IqlJc9ovyJkQC6N_H8m%LTz{M+ascknwqXt;A60_tzU7yCrnC|U6wu|QHgpSkxmJ}+Af z{T$l1as9ViuxbZ}Z1k4k)uJfI;^alTRNP1!?8Bk9Ryk4Zr!w;6zIt7oiHy8s4}?fX zzw+&*P)?TZ=;tt|SfpQrnYJO-+lv ziYK=2V9Z$tl^<3~sLn{u6Zs)$Rc@F(1=;aGI1b)owLN937;|dQe5q5p5bj#9?rsd? z%xm;Q!ekK}{O{Dm?(FF?J%F?JrYlST#_r<0^VDJrN6x6nQ@(TqQEFALw8;7=XL1GJ z8QHKc5zVNHON&-!>vWs5W9II(8oSBMi}(}<@4aTB)NzNU&KtF#8iVVn&KNH@RAy;L zk^j*+QeuxHNd?Guax!pEG^4SYh-NWknSNbikWA1dA|akvJ`+66tzfS2*8p`Xrcu5H^ge1E&s2%#|I^x+$5YvMZL26n zkp^StA#;>ujv}*+k*Pv5XUc46D4AzMh6o{Ko@v51ZewPOjm-174d1$S-|zQ5@B9Aw z{hr@`Zd-d_*Lj`iI@fuuwT@%i8<;hDH@NrY^q!w%s8PF@73*6nUhBiAl-x5JWbfoXyw|v(+;^PG_p|7oMd6eWf_0(X zXb88g56L&G+<~KpIX1Rx@>x_e5O%_%DP(fa!x8jejAu9{p&K^dZ^S}4M zB)#}_5)jT{&RC;nSX1FSo}J&{$lF=24b^>v;Itls8A9*jWB=<-msEsFP73u@MmkB> zt}R^_ecXl5d)kDE#chDVZ4A4JU(v5IFYLM7lN(vSfYBOuo(UULUJ-Br~i>1l4AV>`TYhVU(#hE zNYrh40YPaNhJBUdTHvkw==8pqY?N4&R5`Mt=LLRTU{(6UW=+#Lt&90B6m73Z7wYdn zv$2#W8HU*(kLpf$TX1%>x`L!+4<=t6(%w1iO^i?YpOEadtE7P|`pCda?#1r&p!Hc~ zXnc4_?2`-xtDd|lGo(9D*_lr7mee)PX*?YS518k@H(54Jurr%R)IL$SrMt2@C|q4r zdM;VQjsSn)^#3}Y(hE4cv7NH5f~K0b;ITX>_CzF$*eqM%k$1&#Z^e5KXMO0|GWRP4 z{Z>{_cgts}5q;WeRJ=(&TF?IU#N1ITkn7Ek=gD&?mAPx>-gq zc!bPy1YBnA_R!2r1^ORO z?GYC?MDcf;o2cml4@&(9J%P>G`4ZNzZEG2`L9qQb-Sq%}=L<1n_SCvgr}Xd{YPXCmfv@s)j8Sj+ zD}I;L;`?0T{z&LHedfj3OM16|^B6PXw?W@TQ}G2Vv{%c&=s65|Lb1sf57D0Z%2UkV zlUy{v92a=CXiIYRxmkrp!|J@p?$*i6<{@Juzv4T`2C~~>vKILnc1Rjk9!B5q$kDR#|PjR z%|j5^dMyC1j1+@`7Q+f$84&1P1#!)5gfn`>By5e5)0s+$Wwwd5`L=IBQl-962ArXr zv&$zC@(aR3f$A|rbcRdN!8of1Rm^BdNe~Ibapg{R!b^wg5-MGD4wqf@$LqN@I!|k&QS3ANZZ2)67GY~>Km7?Ebk)LfF(ufAJIt4CfA~) zYMy#3bgtv@S=Q;g;X>g(d3(og%@+@~{0zRajuwwL8ePY?Y}vA445 z90+Tna>(1zT~E`+NG^anOR z+44$PT$f~BZ7{GqmO78(W95}XM+CGF#<;hA!YB-{E)=lrM_jr$RdIT?L7fwj!@&^_ zPQH!~8=Y)42q5e^b^4j>)n@(E{7nP6d(YcuICxcGr<2#HbT7r${3xGt&@3a7j>zXW z;2-6-H2Z`3Ld@Txg6tQv*Q8XxgnWo+^q-O6ADr{q=JT8#a!X)zV@4;zecfXYRXNs@kkPG`LXsDC$DtUy4BC6T< z4Y|QF9~1`-RK3a`X#5)E>03)WW!mSgU_)X^Tu@)mW)W7^vVa=-5}aGL5)ipojX&Fx zX1`pTLFK-A(D|~(95uN;a>7|x{p`W&ADaA4ZCg89Ys-<5>5ZX0o#~RC;Vy>>wcB0p zRp2B3kRag+gWKde2kS`_Pg)Zt&+|}JUQ=oUf87uaO` z?BZbICyijug@_J~gm;02Y9)x!n* z7ja@al`CC#h)+naK}E^ivO4yTe5$ABp~IuH-tGX(rw=|O{W7uW!|@dLP<-;;sYya1#ywix!OE@wW11X2!r z9pr|Ikf_vp)>0TX@Io`YAv^|%AV0F3O==CE`O3ps2MjCg@GCNl8pX~V)e^)d4joo{ zVMZmqoyN9lZT`sc4=CIK2m52pQ2X_q_0RFjgpzHrBA&+^iYn55*Bh5;pYQL%iTE8n z@2(!>Z-v*ew?dFF!}IxWc!w60x z%#Gh|n3``J4gJ> z1$b$QSOgofP$=o5wjW3nn1C~=Mq#K5v_~j#B&ICfig|lQ<#7ZYMt4~5zEJ_be?GSfQ9!jbJ7vAd_C#bahU6$o$2!kRfBf zos5jrucS5g5BoGh8p#ng7I?cKuGM}6MGc}#clpB!xnfqJ>9YW}CyW^mI4SNASE_$* z=fQr1(uj~rcQ)&so0zGozk7s)0ji7%nLuPSM_9UXjd>3b%=1B8Km9Ps?E_Qa`+#8g z7$X{lARx^KMLFUC%!2QLxZy33k-x%SA+awHRNw_pgrNJ;cP|-{noWx<6c$cm8?w<% zh<+3(F$yqSFiDEjLAgxaP?V2GXM%21r9;^1O&CZ%_69xCwHF%pB*t9SqmEPzcLF7;27O${T}{pI{?OHyT}>dq;xxniKG$S zQuw!s2*k^P9u-8@EN)L@fX7A#d}IVWAwx<yhEceKkCehrqYP-{}cQs7OA=(BjmC zB*T0-ky&6kv(DSW``I*HjM-(_URf0sCWtlzNaP6hXx#ZU= zKZK%;sAkoy>|h9tP`+r2Hvq_x49x5_(Ay%ie3(QE;GQw&+RDoIvh@^~iGcj{mwMRJ zQo@fw3Xn=X_;=H!j4g*5-}rndDllm#%w%otN~fRSiwt5Z^!B&tXr^_`V>Knhdx%t1 z4oFOh8ta=u>95^8no5DU<3eIV6*|A?@?InP{)D9yGU{6VoyTipqdzF{h02e66MNI7 z!cn|`Cft}VRc}4YJ|9Tgw}%aH+{mTTX}#Pr)cgDF{C-+fQYXSW|A&Xmf)oJ4Fzi0> zcyFCac03rq|54>5yK6?;LK|jlTFw%4gP6)B|t zoHjfze{kFXWrW0iL_Y?m_MGkyQId>4$+}do%Exkpcgr<_^(qOBEBToZ@N?;HL_2*s z`;r@f=Bp6#N6%pw~n5QBT=2w4SU7aP(5({>7Nn^$zW zQ~?^id~yWaF7WR&0;#LoG<=_>+5HT^jw|UimxcVWLta9JUyZPP|L-SR(TQ!0_Loe+ zJm7y6hgU)Nf_L@xramJa@*me*crMR72m}s4P(8X!6g}Hy6!~X@fr@)ohXe!mdndfk zZ~zhY<&$0Jl+PrEzrN$d(MdB+Ty{1{Mjl_N!}}dD5lJPrzc9SL;1?Y&X(!;pXzSHk6 zq1jT40#Bz9X7Mx6PN6$kp~rNRO~4smx8{}(3k0S|zO@OVc>CY3jc(J)$Fy&EY@1{m zO$lCaz5akwp36c$BRaQtzhrvo z;1?19no~3l8K5Z;U`hhgl?VR{cl!VeVUg_)(9j)i^k=v21zdBjxP^x*FDKgQ`|u2B zNs{ZDf8#Ctbgr&T?}}%MiKDYRR60vyW|u9vbG|1)6en1M+LKpM+MX1#=wNwZ-XUF= zzHeIkNm`}kB8P68YIup}yqHbX zDN(0u%S&8+-oDFT=TY7L-1LCQqh(KjcWBLN&PEHA#JnH74R!x2oDSqClOkH8-5g@c z=Q|A_e!hpw0xeXkDSG-cyw4t1x2s4sK&8WLGR^9BG)xo&Xmc(?91!tWoHNUfRzK-Q zuYMMpE#d+BXDRlaH`dmUnmok&Me1`QqNRVNOw35J8(B(JG2p=%zygSN7?G>MSsh@3 z#vqM(l5kv2&6nR(=P-u8xfl}#u!~#Tq9qK12xycOguDfOApq?nB4GfC@*hD%0EG7R zkxifvoQYYCK9{Euf0as6W6~2i|K54i^i*lfvhvLJH;w0JN73MZK+6+nms)NKK+d@k zqL4$^kJ<(nL{K%||AY1rfrW6C--*#A722c;^&l#=$Rs8;M)Lc30k#VzGh|qV^$aQ} z5Sk)s28t4P<$qc`Ai)I^E`jKm%tbfU@-POD6$FzY8mR+$B*cN(Nf_dd_RFbroJ^q^ zzybiS-Baj#gdlhToh1+|sX-W$3EfxjMhq}6E$#Z!*xgy=mGta)Cjo+LyIBLU#PNlm zd++@;ozESOG9NPRE{-{rZ7m9p1S$tKGm!`8?Tec{jyG>2`b67dwW2HD|_@2)p!2}DkGFlRLv#?ko606W7@zUaRYb7aE#0nVN= zfirv`62Tzcffv>{&v%ON=nyET&&>X`WNaK;6P@B%{q8U$TjP=v08IPo!Nx3Cf1 zu>Zaj1yI_N2m-9h7(?3#EFh%G1Ry?SW&xi@;S5@={g}P}c^c&^dUCPJJ)?x9PDxLB zi-2c;Yx5Tu(AWLx!A-F1M)+T~Kf9v?Ms)Cl|5XCLq3$IQy(FV14OoT<=n2Do#!4q< znv1`}%Q-;w!(1tw;3UBOq;}dZ4M2~QVJCRd(5}b?8F-Ys!58B}LnAwyc;IjOa2;C{ z;3;TS+1#FHN6*{7oqzPZ^i<&xdn$oST3%?;$>Sif#k;SJ%Tkh7ap5UK#LG?MU2!$J zq;Air6+}fYVCNtdWdi9{*nsyt0QLp3I$;TfXj@EoviI=Fysb@rxdeVi;U-O~rw3KC z^-liF7S1styM2j*jk)e}_&uQ=FQmihFGwmQZB#^b=WVl!3jDd7!JZ~~aW6^1aG}%G zt!9U6?8-}8LjFYLK$~>tDtgq$m`VK_L4eGtPxBkknXwCq{s_AiHa1yu(xn$|ghXga z`xxf9Pj{5z=tO$9AiYr_qQ|n{HJ?mVW{?4@rU^`UEkQ-jz(j3;EsAjUwb6s4!I^tR zrm*awHx7Taln(y@G&FIxG0&TROCt<=GDQiMe)H9vn`<(+&$D;8`>LIgnn z;Yn?}#fxm(29iR^aV2y&Ectex4mQ>FT?ELLXc6fWsh=^Zh3>>3V>ku3$wp zE1(34hR#Ao$!t2qr7k%kqBJ=ivFX7j8Hleycv|2F+tfnmvYhXevNtSl@0;wA!Z0L=9@0CrIZaHElJdHBUEa8#aexk#H11!N^l> zi1K#R7ays(RWX4*Cc}_J_qiMP+0oEz%1P1_7x{6!jH7AS2*`h&uwDTdV=h8AgcOpq zIGR}3>o>#sZb3w0Ut_!SXgb~qq9;3Ii%_J;1s);Ue`xOuMIrx761jAdLjX$g|4#G=7``AQgI9!aD{n+;6Uui;jqMt^}=-6iRe4Oe%58 z{OR|0uM)2Is*$)ZWY=pUalg#b#rXBBVMY4HG7&QzyU*?#vluD1BPa}CNakVrHQ~w> zMpcU)5;)@wkq%P-P}mUW)taq;kWvk0azK)8NQ-epKa0Gv4gs2P1or=J(yuYbrB{oL zD<~X;IoAOW{M)jbH|RhJOT%IEEAF zIb3I7oQVq0}o&5u{eK9@+!X6pC;X$&Wok=f_=VAu}J48 zI2Y|c5+E&nPOSms(H|_avY+H)m&Z{N!qY|8CYFk}i|99@N*IxZax*)xEc6Qdr&}Ku zI-+SqL_5+1y9PHDxlI-8^$XfR;Jp<8{g7njs8b}r(8+{%H>FRKf-5?bqS2H!`(Nuh z_{=&sR6UYoVzoA93c<*^5JJr`6KEC2SjHs=S%q7M)zplM6J+P9%WO|KWOST3y6T13 zm84kM>_NPeuaEg&&--q4-0uFHWQYxgZZY1KWYw*9Y=2#K>fD{4Wgb>i#y@5Cvxk5Po zcQT^7ZW{$I+7d^xz*cD_j$i`5*5@C`mnF-*=PP_kVQ6?#V}11B@3`vyiS=9WpRvAN z!J7)8i{^YP^4r^j<+M^uieVydgjbLc`?@k56^lv(2`txS_HWwEuRC0>5d8a%Z2Gn8 zHfrYWHDs|uw-l2>f(1n*P1wtLm8lAntS8<;R*4({JGhTeb17k*#R-d5Aa5_;Fi4QuN=@NQnn&!sK`XTN!~=cr=hc8E`BWVkE#c@GK!PP$|xnTq=C^ zr_u2bs<_A3&|2q;XFA|;4iP}|`N(f-i%#lSs{8PiLLEl8FLlidsk4;);ral7=CHja zS8|#M@Of-q_;~UQ6iG53?x6Q+J4um3{lFBYQw z?0>54Sp9c&4!wR7kPe=I0eDW8HnjAqJ!();8)2`BlW*5t>`s2=Yw&L0W;>i4y*6h$ z&$bj&s1g8$T)daT!0?xSlj(OfqYV2R)yCDj{atuozVaLpPJsiE1#R6wK~jX1Y!#_- z;oVbsFG(4IH2uZ?Bqif72c|l)HYl~^Q7fkqkb*IIt>0LiBR-4p@gaeZ#7YR%g}e^1 z!ky6<@TA^AO$+a>}e ziuD4>%G|z+REeHj3%O2e2~Z6cGtLID8M0kW+LPj6*^Y>k^NTT77BAKoSop$dwQ`Vc zxsSYRUkO$22Nq@09R=BPQ`j^GwK1_ujh!tcg>-2)@jx7P(@Ofv((USrpE3$+tXBCS zzYKA5EZfA+xnO*XEUMGX{H`}!@-5~Ip!-4>8Zss9ex-Gf*53sFgpeBEJV%sVF{L5nf!y04#$*C!&tqE!Gt&`^H`d~Klj22l7hmB%e?bpHTmQMiV|2Ps z$#a8aCAh|=ip~7#wLQLJd$J&6O3KmvIoCtO>szhW+uH}8);KqmbC(#W#P-C6H>k$_ zeV*ZpJ@%oH>ddw!9(`k;mc#w=wiY^d^W4sEva{cc@A{|y8CVsSD^Cj}u zp(1}kF9|fpM=>&{0qTDCf%WOHGd)*)DYiS%xlwuH`gc;=M0Vv#a)Ty^ z7}=$<>*A7Amg|>Jgf`u&1LK=qmU2g)iM^B;_Hc>Lgc4H>SMU`?&r+~eIxMVbjDxXs^tX@|`FJgyo zx%o!Ua9^_152vvrcU^*5Bk4jRAgMB`pGjj}Ph}KVf0wN_#9F0Az~lDCqEP~$@};V8 zEIkHy)@b}z2KfmM4*NH1X7Oa{PNECcacUIxb=nu90eSjQsw>Dl)SF>x%AK9%4JjD9 zPa-XeOpxE5upNz!tr9+|#oYn(Ayv@7F{+yXd^3~* zjTfXgY<{`IN#wxXU=t7iA!%myc-%wc%f5Tp(z|X*!)zRM+=&{!8VLJj71GqBA@W_qJ*jKB^u9a>}A{(64}@>|2XhnFM0?MRi_yyS zy{5kt=D~oDKod}>HxtaU1iUNpul%;dH?wIeX@Lpqi&G~nY+VU3wkIBw`NHN?P(bkB zyo6$Y4kKchz`}=O?B+TB#Fg(0LxTdHol=Nc_NBc5GsI_99|d)zu;-E}YhxHJ zjM2QX+*7BDyFy#yv0T)Rm4)-iuuluOB^!R&ayfQx>w=px*xrrt=fkJ;7M=H|5jwn- z_2l%8E(UY>ILJSQF3!jI8M=bt__V%icXU&%LPOVyY8|6{(_VsWoLLT7CVL zDf)*0dcDdBDRxLBjCZd`iA9MqSHgIiS5mpmv@wI?dd3{L-1*a-PcCI;5S9$yj05{} zSXT%(s-Z2Vn%Epz+i=3O!`1I^ox{WP*772+Sb2yaC@Pbts|jtBNrd@j5Nwjt3oDHu*p$rxfO@AmaGcgkhpS} zlok&Uuk1AVp3sfA;aq4NRB;G$&5eCq!c{8{G+UiTS_EtO&A!}K1xYQ_ezFRBA}LRb z?n#2myweS|y728sANa-zT$WbKvOdUth)H^Q^u0F2pZDb=Wo>?b+TfH7=^}4`L5{sz zgOn=Z8#3Zd24SHuZmPj2E@h7S(vM|$>v`^;z+*hT;msgt$;CxGQM}FF z&Qx7iG@Mpz71g7&dui!$$WhkGqH=a;t~{$=876cg3*qC!*S=U`-jHMI>h}B%ge>2~ z(1m$WH$FV+GqB8rW6ojm)$|M{EFJy<=(BtXE!8{K|IzxD2rDA$G~1;*39oupSSudWA@GaiKy67ZR%~cf~I|5xk{`++9hH zNZF+qQRlL-41&pp7f`8i`O$10r$m|uOWEm0e&Q=T6%Zc51>mpToX6(#xX;Zq3VNze z+6&Pvn3dm`gNcj2NoBUQnC(fh)4F1voJ{@i4XKZAcqJJ%?@T}c{FtL2)!?bku&
rad&Jz#3hllKg%|%`aP<(<7xN8iKipz~xy*NFA~;YHOdCd_H6f-&Wny3M-tgdfB+8UF0r{T|Z3GFOsgHkNnd#L@GbTToFBL75$y(?36K2zI)546+Aw zRdh+BgkP-O^e?WmCrUh=!HAQ~qvZN}i;CyE+uHk_=Df~oc=$Y&h8fEq8M+;dF5QY;W_PlPrDkBI=T+s?*Q|aqWU97WV!ztNqVWn`fVk7@*AZ z_TPB!t>%kMoDz&&EY1J;>0^NWTA5)*I6Wu8_am$PB0+KK$SP|O$2m%Vmbkppm3CpN zQNcc77er4=qBR9!(1j@9XZe5e5rWh zi_Bohqm_%_LX*>W*Cu-$H$Gv~H?%V6W_7aXcrv-^a#x+t-n^mxY{=`u$JFX^a{m?C zMVhMViJC+WEe+|I#y*e1WcnP37>A?X`O9#)iGsSQSF6CYn9$A!BOvTHF1j%0@tH0w z=lvGv*n94hoU9L7Uz9EqJ+l}xA4E-EF~?1PHte~^v)3bSWhkh+5m#7y=GFA!Qp{lQ z`a!MJ$q~NIT0*Q|Be^R}6dOpYe+k+D6Sk!5Sb&(*!cRF1*Aoi~_Q#^tNM14|!g zZqL1pcu?CSe9nN{^kg!0)VI4G$I=6KBMv7shU?PrUNb*x4Cc zS8jB5-G3I|=i6W6KOQ_73ujV-hJk@+x0+r}i|sLV+{B1DO2Usnwkp3;tGbE2n#Eu5 zvud{9XM}}G^WJ4xm(|*D^%glyHJi=uz4uWz5SMAJvq}%G zvxc*oN_buRQm^CE`+ZZV4mUQ#_k!Y{2d0N2Kk4{8z!WsJYF=$>%pYh<%^PI?c{cN5 ziDxesGuHUNJ?k}3-wD^|SNXyOI{c_$J#>k*>BFaIW?ohLa@o5Z%Rkm=CB9P*aQN?W z!;~OGVRZOadvQdUdorgvAp7G=Vt7UVi>s?jeIyR^_Z#u{v0iF-E=^tn>${?%wJ%(= z@mqLMugv_V+}g6SM;Zb+Qp?2k?wR!FcjIs`?o02CCd^(SiQ2Cs=_M)hu7AA-88xqldO%>o*%?H4>OAH3KIx@y@AwL@bH54-g#GB&)k%T z&x3gOmp$K#W9`hYmvX76jAOK)XY}30k)lxXkHd0H8g-I39_2{}BHGB&(wVWVxo*BM z?+W%yO+8=y{;MLmy@x3DuY!aMa7cjWQo3ZAte#!#Za67Yqh^!&VD3RzMkOOWQ2=s{ zLzfia7jVrg4yUeBG2ClKZ=8buB!lq4Zk%BtAEtf$1hUWhve}$`@3o_(v$e}5vs>B2 zWvB8^`|mIC_Y$`R{flkcD5D_-k&UDVc--F>n2`KEMZ@Ki>5Bk!*4XI&9vy|9By5jO z#Y|#fH0`L1ITY7=bZiizTD`;^;pfEmHiylY_qKsC zXCAs`u(oAKL&SgHRrNe%J&?eN2XrU-6Gcc$4)7Zp;6;)x{UGj|EyuM*Z^Sj!l(tWM z+f_}j?r`?F%yS872;%g#{W#C|wjpoz#ZmRs`G?Oo5MP~X zq#i`sy%EC<7@O?_MFx9x20Mg)j7S*DH=%vx^RDs#_7-heF|ww?Z8kS^rfoc~`8qXc zKh`F(_OpG!}+^cHJp=V_M@RN{qrO6FxBea_M6Z8wl{N7&9W zn2vpaYs&>D;hB^BL?Zh#`kB95$@nXY>K6rdGX;0~oq}%&XTIs`m1((y+%cR7hxr0l^D zZUd~|(v(*I@;u$dF)AV@-Sf<4#QUy|)H=POS>fCw2bM)B3TmuqVObwN4_m|coQyURgV z84>4!1T*DI<8z_Wr@V(w@?A@t&gcRNqcMxNqut&=lPL4AXEHF45SOEbNr|(QuO2y6Trm2|GDz@2#>ufjwO9MgwWQOX-5mJuvkdeB$F3 zHOGTIQyXWH2avFxW8|$V%U2(GP@+x$ z`0EUjj;!lw3(CDFa!b(ZGF?i0gLSe3srPy;{4m|q?|~_Bo8ER6N?KEUc@4uOXTT@I z(+ho%5+Y2@yB#14l`AVC{R1pRw5oCMx2R=4+yrm$d_KF~*aafz}Dps`;uMweh#L4eyiqo1RhHoF;e_%UX62J zHMO~yVJjXjSHZQIS7s;-i_m@X?p;42&MljQdAk69%t2pBM?rHpvS^NF9?~OLhQE(V zU^DE&XH5%@Pn#cF+(YArtqRmPpvv5N1y0k}&>((MnkZs1I^92RG9Klb9j zyVfddkn_uqt6PULZCxe~GTy0wQ?4CnvoF1+MWQ}WWeE@CqY6Z|LIio!6_Uktu?zGV z4#Fr*F+O_g;z>Ev{4*rU``#MDW$Rg7cS^*zRM;#-^B1nRp>U07TgDbgipOi}b{=Mg zg~zoKh#VS#LCYgXfJ6)CF76|l6*>)-&v(4bzC{xmI+8!hytt*wUQXe6Z|c*8`ox#z zfqKbr6EbG&0iO`08f`7FjOK5wYO&SR^J2bPN*`qwi!W5F^=4u>ODo0g$iK=CdU|8$12Kis_%@g zmXM1k`YGaT;+dsU@Z5*l?yHO();oCdi4T3D)y6Olby2vohQQ|C?%~q_GpBB`vtg@>Katj zpZnY1c2Jgl^O@@&U;~S;PlL|-kG;7@Q`Ml1wC5!XIdB5Izd3{FOH78mO_kNRVbhQS zI#Pv{yydDoJJ$^hxtF3Q(kY5IXJ9&xf1Cb99MSG^QeI3;O)u`kY6@y&F;t)hRCVT}BAHTIUN{ex4lF+ArYT3varxh4r{S6IH6HxUFzVb9;JDq6g>j2U5QDSAL4_f(AhYLln6^xH#XO4tLbk?s1`>(MtSym2@|SCwljQ77 zPpP!%&@i`LJ1Nu3FJhu|GqXcTKc?MaGRXL+y{K2j{Pg(`b+<77xPfq(*W2;>7fHoU zO|?9)4Ubj)kGHKFW~4cJaE=n_uyr@qtT)jJJbj4r$}r zYeGoI7NRn~sR-jTW&W|q>TOAbiu*bBkC4&RzfISRLauOAz)(y_d+(VnHgfGwlnNJN z@LOEt{T=UghC-nouhqDpw_Rem?usB%jPv&tGEZdiPs^wm6>_tY+%<`mBGeB1LNN$xqO@h3u4` zK_kEo&pIfUxu{A{aB)MEhfMJTQu~8gOEt$T5==h0i`oQSECYBO81UOSzLyFLodY?b z3Wn_H#WdMHbO%(yC$g>sBUwWh%U&igj*TQ0AqLQSLr{v9yWS@I`dB1!Lf&Ct#tXpU>M*JlfqsT-Sc8DYq{$@nQ9Z?TGix{%0;;I7$#UA0zhXd z5M59s?uH^mI#f3`VDCV!qZLZ9q)?pH$}4HCxOdq`mEn!kFh&+0z2D6Ll*)3bB_}NWQ^FE^QN{jcRik566 z$#@^%3~`htPQJl}Zu3oM@XXkb>i1Ba@8nywc|y#0l#C3!9(J{=sj%#tz1h?_^x&lS z3>n5;M$R7+<_4sk1gU+sYN)#u#$KyZC~_-Yk_6loD-Npm9~;`++!hyzkys z?!!&Hy(UXGj0!AA-rKN=a%M2^^}sH yJc7bVVFC}2DT4a+IqyBaqc{KG{*RdM@YK0us-8NsSC){A%1JBV&69fM{eJ*hui0<_ literal 0 HcmV?d00001 diff --git a/docs/images/centralized_Bb.png b/docs/images/centralized_Bb.png new file mode 100644 index 0000000000000000000000000000000000000000..0a9ffa175d08843193db2d716533d7b7b46a132f GIT binary patch literal 35065 zcma%j2VB!h^EV(wrK%{s1O-7s2%#gzP^I_Y1OlNZ5L)Oef+8J6q@N-nA|RmByGW3x zf*`$v(tDBiZt(7&yXU#*|Ga(-kj?&fXJ=+-XTGzOaBWRxDsm=rA|fIxm75B>L_{Qb zA|hhBQxMRy;J|A@M0Co}Q_(F#E;!5qY;3+SU>6iScmY5|RV&RnVSxF5ow420yj*zz<{a5`^&!iu~$< z^#;R>!T4cUK)bS?tur7>iO>W57X(d8j#xW;3=T9BI{hNR&eIz8i&$$#Pc2^$w6~Cm zga_J4*InIF-s@K%Z#%4qBL@Aen;@JY{>Q_fzHWBET9J0%j<$Bcy1RizdHmj)&?2M= zn1^(zi&`zUmrY#yAlLRX~@y1lK4maC|cF9HQVRmYmzD=UddCTL)&Y+^5= z=Huxs?&YWAW2hl6Zz?XP=AnvpLMe-=sd}SeC?G{S4Ffk}J9QgtZBy$unZ;bo4!J(!r%{eI|&U3F`THqkFuJg`)UUFVG3YuU7Jyks+q=t{9gNU21lC!Up zv#2f@SneiV$;ck&WU4DFrlsSgDD)|{B{1oL31Xc9hz4Tms%ceRD#JeBlKbYaHga*mFM_DEi69*3<6p@(x*veS~66LWUg)z@;>FmU!n!!%6Y+`%?e(sk2Pb@asWg7N1`V;xRgIlI zOpOrQ9=KIjw z4aUe+M;Qp*h`<$PV?(UBgPxHBT+UtJRN27EPtIKruBxS^gfKx{8-gynnjR=S9UFv> zwz!v;xHASWiWSloCOqtH?Tl5owKZ_T-qexzR`T}1s^OfG?kKdgikh`I=wRxE)5AEU z!1~nW)C?Sb9Q7Q*3Ov0q#)?K51F@UpD72lhud1Ges)~q~ArmKy+wu6lW)?V8f=j*KDi^8CR znNv~&FF|o7K_ELogE-pGNl{DgA!MYU?Sit*VID z(l=Gpa&go*Ffdisk@rK2xj5=!Rh-dU_RcQy5-wt3LVHIYRivqkv$&9*j+>gYpQ^gI zo4B#Ouc9JSSjSxz=j~*K)-?o%6&P6s6$vLC)`qYOPc2blGzwv8>V&dZHxRZ}()2*c z+hMeoyo^OmG_&8f|{|guz`t|oq`8iN7+Ql&je_OsgZ-VlC`m%k{wJzL>VUGqo5{?M!VP{d_`T* zc8*3iK6>unwsuB#XgJ#2PEXs*UIL@#Wp8b-tFI~SEnx~nsVa&aNXUt6feow#jz#eE zkCXBEm;~>CcQb?(QL7`&L_|;`6$OMoE^Q&*-~5rmVN2J$#p!M5{s9f1MDoiwAlA27 zAmN)%O z?%gB0R}wtHwUMAXJdnqJ=^lFghjRK_YkX=dbCdf}CIvaU6FGRPU+xyemR-w-C##R1 zxeQt}QYAcc*q&}=N{km@r%UFqr(K!qAgLzCSY81WAa-|m?a0W;M6hD#PxY{0IDdYj ztF3KsapOmK)7g)Q?Dw$2X!;)u(VCshV=}gnqLe0CSy~Pl?}pFX!INvR@bLv$=5|vl z@Jt3*;TGZ)?a5b2rF=_?F!iZ@s+N|Pn%36ivl9I_X5@<2h~ z4@bwgc8#=N7-~>#b`TzE(26q!>yZgQtX^2m_Dq)zwsf1chbL<~3HLYfIDROqG-mNn z4YXIP=lCk-{I97A>{g1N+*)6*XJTSPVosgG)4t5hJ3Dk5kSpJvlMD}KO_P$uIzCF~ z2VDiH+^6EX)K& zLH{ZAF4s43^$&8PjYkTALnv1i%8V!Ta!IN` zUsGJ^O~wQyPA%2H5yFSzTWE?E$Ayn9DC=6 z96?*z=-pZFb8B2nW2DtPf1cGsxZGLP1k0^%`2FK(tHt@lEEVHVaz&bH;s`c_p!_fH zDjX2C_u9594>XOuiz1+xDE#YbnQcOIGsssyggu!q5Gt6sI<<8BhIGV}sm8`ejL_v- ztw<)lSLa4fdz3}ct$hCc+2T=kbv0`2Gbi)Kn5(D`cn$M2Ue+AV*GeyyP32h;_6oa2 zi&}8fA2OD9G&mb&zLg$5;*oD>J*rndd;-cFpXO%xeVj&F`7esJ%s%F~ zm`j|$9<(oRS&b`}=hn*D__$nyohC0VC}_si9f%%&&x%|cx)=ZSL;RTK`?Knn8$&6?MMMmAxleIOVMe6(hkZs{N9?f= zGxhA??#f>QaVyWiz}{AF$Q1NS-E@loLBV%QB7<$5jDoH_+wwQ|ir$m#0nnY{zyr1t zEgyaTpATT?_o-9TJM)0j_i!t1r?|{_Pba$yM-qKcGVrIcC2P&IGYa`@J#iJtK$v@?0e_gVXk|SP4(j zn>QOLzkTfRJ&8CqKARalGAoIHrWiq5*Vfh5^0 zLY%e6St@*N7e!j{hMs(0+GY`igeN-V-Bo=Qv-k}^R+-7icb$z{w zUcj>Y2lX1zj`7Ej>GluzmM3R3{HZa+l^4WBL^hpzH}i{X8&A@(y=Mul8SqIE+Wsi{SbZ8K0phCYA3Q@6Ktz1p^gT4i{{&M?M? zV1QFoeXskjJs;ba+FxmkmfA*T1{$tpG~4J5?AQqqPF0S#&wN-(IqtQ!wP9X{2n!2~ z*G$T#``8xRTNYW?JiJC1F@^f_<%^r0on0C0^X%+qyG}QPI{Ixo70n@B?iv-Ca^W&% zQ^#j#XE)A1DhkhZic8-8WczdKoU}^WZ{A*oHA8ZA^Yts?syB#m~6PyI^egefsYB zeRjI);H4pn{=sO-*Pfnl9&sLPlKA1*x4h7e3q<~?q*tAZyyg3+I6Nu(yBUAR`R~Pb z*jq8y0ruY?N(^x+?Pu|KKS^dM{Zco|Oc?4EP;t3p2$~!k&Rf$RWIN+NGxQONuP;T! znN)VWl{@p)NbppgGU>9IurSZ@88JgVqM0ET&dbZ|ZfBRxS+v)lEOd8SIJHBSscxYe z37@X?_*$vUxda6^`yEl`Y1+#h-w^!f#p^`z7TJIBw`{ zUMna>T`^Qtrl<0bi{a~|j5MMwDcZV(I@iUsotvRjm3I-&MnlbZw0HZrH@kOqayG1$ z+T7wNvifF+ZLQ8~K3V^6s~W>ZH+J)7QQe^_TyHlfcwhd?@%9v0=>>Yr%*?zBoDZ&1 z-<=g}t|+T5Is2GBzuY|i@auRWtraza8cv++-qzx@Ruovd;o1%- zH+KthMfr(?)tKP-eT5cH!1%R9&Ff{*6cKm%B0n(oAPqT&sjLy1G79yer*dxYA3Nx$exzjt;7e6)gv3s*c!RMD4+r zS%a-k$znu4^~Ioje!$2yEPwz0{mhVceCEO4=n?;pM4~EJMeUn6!nv#pE)FzoQan2q z4TVJm_V19pvK{_zAD17ST$@?*ubPS2tWqP&Q#0WN+&5`Otj;Z$9 zTR{s{Aqx=)_+%;D*wPGbMI`@e>dqaD8&mftUfe5*1Dm);vVp0;!&X&`5m~lM%2HwI z?_b5a&A@SQes1oU!@5p_DWoj1ENUqzDEK(hfrNK#Zqlx!b3(qkhyv?)4R;VqZ|OFG zD732M-%0!I;Q7=4{fR-~D6Edu_>&b$%&&AwZ>M!rUp$nNm6g5o{+8_SRKjp?h(M z82n|ej8{A(EN0K@C#pw$&H(!jd?P-h*g9_552#hZYUtQ(u6pvs#`XjD2-fS~ltpG` znkxel0<}E56&Ow3hqz0B~i$X>QS%VXAj~5`o&pdjQRM;M1vUN zxPDSm>d~~PSK)rl6%69k@Z7ay>z^!T6WhdntyDHAZN=gCWseN|dtPsXo{J#r&49tt zVaw#kdGG>{Nq?f{$JFw4U({XPTpkg|lBw*0%+K$a5Fa6O2VK6oVaLvW1<$vE#Tm() zdU^?JV?*vIE4=0n8=O0JZ$0KSnRpZOAOx+bs91*e3b-7HE3RJ}t|D7bN=l*_k@eqt zwcx6pfTt4$XZG0Go9x3l#$JLdAdMpM=a&u($^1(?FZvHd_jCcfwHT&;dk`K>X{fb- z!i1`j)s|6yZ3k?ba#IVx%%pc}Ybn;NiqRQCKOI{~Q!+A^-dWt^nId@ZzPHPwn8haS z+8&P+UUmD^0`c1#mlMXeH$@8WOOC98f8x z2AQvTfI~fgUQJ!ya{%#vu<&hJ8AZ?%!DCFt)VjaOjhbqX@i#0qzk*}mLM)kpky^lt z-akJ%vO=S3b%sI48E*1A7dRtwDzU4Th^_3PDRnjx{CIK<*-5Pz57dhzr*k*O=_(_6 zjzlmKh#abtB6m)MF{7IFpLu+?89d-7B2*FLNEdyj$~}wVC-rs`_PR z?sD%%rVO~H3ysR%p48tbl`8`%I zaCbUMb^Mng#N4m8%ze#gU1pWQDqvXy)1g_>Vleyxf#p`DM zaQ|${sEXuZSL`SPxT2i?)(}$CvDO6c>os&(7o^WQfgTD<%J+A@lcvW%W40$nYC7%*uXsY@M)D0b;G+ftah67)2ePhqmjN} zU+<7BPc$+5ed$cUp4dnT$<~Q51h;mdKkM@YJzZVSqN>T?&N$&b`jV$r4KR{;U}d>t zbl!9S@bp1)SxsC@BM^_KU21;4j`4giR8<%hbaZ%7^D@F?wzFd4i9)G;N97NXkwC?% zSG$hg_CW}2KBQSQvoqN9`R6Agc0u}AQW0B;L7b1laaSCW)_Y4OTI5knMW?S(7-EYo zdL&@I!_q=FZjh=Z;P@<~i87lA-RXR;o97q!jEA=R4r0l2;?hNn^dw6&vq9IB#r>7C z5z;W&r)eLn3M^UI6G8bkp}f4jauD3F4D=S5k;(1>M_>$`AuzE$8q6NN)sLz%ee`)?;33f1Ffq&sU;q7b?)2+E)XU$BPZXJy zLX3=zCf8@Xw&yYBxYdgz6&~D{rS?=Z0Xr~)!Xt}}2EKmzau+yf-MO>oCCjT%YQI^D zpr5LmNrmzC(&m^gvOOa3Ht}Y= z5r!nvtu(MD>Ipa?&ge-IW60^es)=txW4C+kVyh&NZY_q;=y-bak}`yGQNb8v)=xyD z^-ZK0kY}3y+L==rzm+9?vZcM9BR%+LK(W%mZjImZQ(sewF5PO6ofRO5Ed}WkO9}Fk z`;v^D=bOWwh`j+_ZnY-Zc8@6zWes8Hb2B+B>#>_ZtNqhlwr8Tq1kT)IFdQX`R4zMD zn!!L9`qvp1ex@EghwhoIAI8yM8xVXf_%p8eUaWD1;Y^kpM?H*hj)ZdM{TrUC(>T4? zFD9vOkld-}eu{Vpc47F(t1&DVk6#Ks<_i6;dcU<=n}&q4=^1}Lk(PRdT*hrNvIxl$ zBF-~-|8vwtjioapt<#W(I~-rKgd3nC=(_^s|1k3G;5R(3xBAVE+t$WCWDH~BjyibS zm7H5TEzGw`Nqn{Vw|yVYzw&jwL)7&81qH00w)@598Zo1bnLR)H_E##l zDa&h~lK2E{J$8Z0`Qy9U#B5`0u>VL)YA&ME?E~)7&^Pz1 z7e6zHEPO4U$tL?xW=NgZ^_LHA!lezK4=d`Svl+}wT8(h$~Mf1H0ex9qB{knjHo0!AtiY2581DuzZoFpo#&ULmHK+A zy+3=-(PGnc;Y!S%lZyyt|Jw+NukPR3n96+vkUnA!FJuok^`4Zk0>1}ztTJ%bG>Zr= z{|$Sh?+s9Y$+7%{Hjop64Q?63PhL_H~uu_Pzw^8fDi^Ji`5 z3}g{u^nXy+n#_@3SP00!`9C_ne0lf(?sS?GZb1^lcl6t7o0PwDjF|yhNeW5kzk_%j z{E1Ysu~qU%wOdLIesQb?N;nGyD&n1{jXfe8iod!U}!C+#H^lC zup3Am?6cD+-jvyEnD4@)-%P=gVuNh5=28!nq#94pEzjo!-tF4YEg%-)r|(@&)br;Y zee2i1kFIM;bV1Qf%*^Zg{7}Cf42mQm<1Z>88=!ag&C<%--5-dA6$>crd&GZnUe5 z<|wz|sZrh0L^12#%!9Ty#lV+s(wG{a9X6(c|FSuXonk9Y5;!vhkKP*GiXHyM7SqLh zu4i;&EudkK*~ zEA)PzetZ{y$&-3$)l+Hen(ddr9a1*Ey}Lb|G<9=MT$)Q~C3!aU2{$fyF_V4093k`0 z`9x3|mdtYP)O`Ym)^}6>ziEvj6*SWw{&Fuwne}jfh@)k;C~Y7lUsNjMXn5cgDkLq=-mR6K zBxGptKg~vzNC?Z>9Bj(=&#uyw=|K<)Fx9rAH2=qBK0$5A599G0cWBD$W_BuI27TL) zJ%m=O5_w-=Zpg)=ow`HhW9=5=3F6ooS9hbQ}c?c=V|nKY+Poq2Q8 zci1K6BJSXQMwIwh*wE@gpT=IL2>#m1vYYLDA6q^)K&y)WK^`5@^(fcSuTwiLve$07 z1nok0M+>OLxpYPg;wB;(*SUSC5Cn3%CdG*N0 z(&P@%Jiy)5F%*4@M>-Rd5c0+;Ni$6G=5F0eysF)JBDvxaIeOWAeW{HaL5 zOSv{6D?78}^UQK`X9$qXS`^X2Itncc4stB&5S;q}jmQ=b;SekT6gVFOHjm!SCT)B|`Wb}JzUIvvrvrrTixG!Qvw5Zg9k!n)ikk@^j#f3)mx41^rt9G1$E|o zNY_20(ix|~2H8oJH5l8rG7CNGzTAPZtZ}L3%$4BM5!kVW;nBSRNZVdn^+kM)G_VUh z_al^9ee;NkiFd4KsU<}4i!DCnmRdc|q!evw$uE~E*B%=kO}(qrbHN)joFf}ZPXEfi znJZ0~G)r+fQpY~iChO#u}7`h|p5=-M^=Ftu57rvHd z)JFrU(E88Qk2wNTF&@&#a+LWf<04)nXer#6pNgG2%NUQ3 z#Mg568NBwrFTRix-deW#2oV(CM*m_pk9+U=;XCDbJljbxcu*yiLK6ww5AdZqpOIr@x=Ld`!rprJvfd!RgnhoR3CB{Kz45Kd z>s%Ck{*vi5l>fN145`*gxLa5gTt~sKVCQE`oI2a}x5IW2zUky$yxaj)*WM|}Zg7bq~77IoUyb zKN)*2bMMUIgCx~@G5nQqzf3{=-iWV?Hg_jBBFV^@G}9O8pJrX#ML51wvypc8Vcoud z=-7ctLO0oRZZ4B`d0&fyytJ(%M9MBb1p$olifU%cs8tU&#AVDoSYBs`HxEnLfz?!yl76wdmF?S1oj%Ld>we#cJ_e zbe(MPi<=h1W%PuG>!qbD;}1hW&Vu>iXv}*QH6~#p=f?A|_r|&D&R@5X`tXdy_NlZ3 z9bsAqc>Vyc_i|PE<{Rqt+OUp%rxn1Wk2J7bvB@Af)oEo;HmIq@6fIot?f&W1;!T(p zFz9K(NWf<(?Qpppsg-0gMdr8Cvt?`tzqMpVmTJ*ytt%>~1coCZ@~V7gTg6#@~}vv`0?kx&22A@%33-+753 zxc8fzn>zq@{Yd#IiK#_Q$mSR;;Ip4Qc5eHx6Msmeg2!FIOBot)2VoB!Ed?%2txN0$ z>7Y9Zf(uLN2-Xh6ZzVG{ll{q7D#{YM76A|-{BoYb3Qj;O$N}8JD^PzMVQ+do4xm!< z--kJvd}vb(C;AU}rQDGZs16GGGyW|m&$^-VZooE^!i z!TNv2N`%K?c&O>06oC*B1lxq3s;*BLAooQq@~*un#AYMHP5)y?5UFdFSDYL7NNpS| zKGgWGWJU!7V@eRE0ngB;j1b}e9@P-$1i|MU0>_Uz zgE87qa!2%j{>zsu0}n%Sp&Vz6V4M>o(;lk;9)Bv0bfGE!*?la4i}?t&lykyxwV&yI zO{=CE0}i@URK_y@oYaS4%%oDalU!vzv`9^>iQCHW z?0|e2AO%FV)09MCE~?#@z|P&ZrRz6(W4+G*7c&GbX5De~WKq%8;Qck>n*hEnfBBMp zd9;K9sMQHV*q>RrnlOB-HCwB1XMA#AcxNStQPW4+%|3qg<4uCC8nTIhDvhp{e(&V! zYS2koIb#aJS_6AHp`0-)02uq6EzmJW&*K#y?8jsR3iY=7?bpSY`@D1$bNxa{u_88+ z4>LBoGsba~Nn$r%=I75RuCBZh1^Iv{$$@zOuPFTL-}-dc9z3eFmnAK@>6=SqKb_2&-oTh$P9%R?3);N$o)m+PxdaOmM8<+- z0NTkZzGfBp)Bp$BneR0;*L$?=oiOy}3%0v&avoBVovrA5j4{NiEPX2oldF9(cKSI( z#*;CPrDD<#0v{+#I(o(QW;BbDvAgz>bO{S>kbc&gCm#WL^Wqc?SjCdEin6bn!#X@O z5#$*+9*&2f9=WxzdkQlG`LQ@zP{IEcbE-Z>Jj#G|^L)j zQ1YZF$#Zo=k!wwn?052EL09gHRonoQY1YutD6g$$1NT`5d8uw@AwW7$tgxu41++^q zMUQ>tWZzrwK7hB@xYAgGL%y5sAp_C#(1%3wPc~+DphPr0|ozR3%DgSq;fClKG zONLPEt=K(*|AND*L3bY_-t|k_#)B}&>c8N8NjKKGz_kZ2Onlwm)?n(P%>ThNx|^(- zHKBvdR|&Z8&W3XkN!GNQ@$WInpWv@?$R4FJOi>YC-9NVY;t!NRj1>ZHL_HF3!Hh=aY}c8j(j;j)}Ry@J}QK zOjT5{Q#~qXg)85hIOlu|Ffv#A8LkU`lo1T_J@L%IyC_cT5OBeqvq8QVveo=Q#$htWHcdZe|K;VIG+&Nit$U; zrK@ms#fFvQ%b6)7T&vY$j_Y91cGpiGe3DBT*%Mt(;3@UK8TTUiTkL$}E7`88tJ^J1 z7xE%fN9s3e%vL)W_=dpe(=k>VB;lB70RyddN7~+Wi7I{Rg8YnV>FHu>N&@aoKtUt; zh{M`9uJq$qsG?{(c3KY=NS^4PC-`FaF^MkGyO;a*O7Q@KJsn$U`NHQ(ZP7JR06L8gc$KdO2#MZN93AZsKU`Tc0z}sWAe+b<6sSYEml8F`p zpAZGC1px9tqOr!^Ko-D3TA}w^|4o`FmirI?!vAwSxWzyI^5xZlv3la{H8M!Dbq>!| zxavs2S151LK344J?7My+)z|+7UN4zM0fzqR<}kLa8yF;-+1mgZ8+iNb;8aSi-mOF! zF6>PI|3KN4CF=G{Plba1T*oix6D-5ZI{O$>wTo%$ZFvzYx zJ_Uc#T@U5;+$RAb^!u<^bv|~V*gW6<&Mf<$cI}TKvIt8)_7096DnO{kIpz|u{-J>Z z^2MjIzcdquk4#}y-6aWWhw1&@#qFdt3639qD*fH*zYA^>Oj0@39{aU#`#@OCfu`~g zZVNft^GyGXH+K+*-**7V`6YyqU%!3@seu1O8~zw{^Ca*O|L2Imocf0L*Eat=Aej?9Jh|vBBU5oh9UU<+Sh=}@{M22+KI?WNHXkg>1xS>Yv*e0fOn5({?ODq6~AAMSs zx^-G+J@Xx;x?IX?B^dyYH*1|YOLm8YVtD&I+1_hidndW!zL_fK`ka8ehtXIzEf-jX zr*74S9Qhy0U!_bgde=e#;F*02TMU>pnmfFfCjj)TuK3HOyb7})Jfu@hO;OC>7Ehnn zbGaRt%zcZTvPA4FFiVQe)2yMFbA?(Sver$B;>&B!I*V8{a7<}`4MUMF2b=D_8w&(j z!q_H?Gwi`pZDuC>p_Z9^QP5Gd?R8Swy`<>~zraN+vBf3=`a3UwYFv?ymH<)bP{QXI zKl^Q*a8d7(bd6DF=YKz%E?Xi3aHid)-$PX)XkS0Zf;_ z=)U@m`jsx3_aNI@>#O?!p$2#{|Cw7*5E2^3p8r5gTirT5!&jojg%M_$l3%X53KI+) zH*MHkSHp5DasNAN&*8r@F4xMJSK8lpVz3ux7i;1g^GKm)suJ8~AAe9EKn8(~%?B4X z0Rf2xtfY+XsbWi$;ds7xEnt|tnSz&{R%N@`-V>5H&Le>pgU)0jd_4N3Pe5qUe=-1* zpO>fN`xkE4r^w;58lWH%Aa#vbAsTTI#23kKm)F-z9Ru^stD`io zUe!6_r<#6T36DOiTmMK-z_gQ_21#YBKBdj~#DW{Lrz4r?8e%OZ-YUm;n?4d^JaSv* z3DSFXpI0->7^*5??rB4>2T6XrWqh=++b|zc8VyE}^C%oU7+H!WhM3e6*_QUBY#y;V z0PzjM*X0DQ^jAsM1-oeVfaD_Al_C&ysm~TV6)lzx3*D;iQy$;peRa9zh~9H1{$a;l zt&b(v37^)Q$gwxuA0n<(6lYGGeEM*Gl7?dBdf!UM*s=*=%h-X|maEN2XlXCo@Q=MN znn_rq>Pjg^n&zq#j&%l{ffh{c_zq@}pdhl=Vu#{s0wfSJh{hD>V^(mmBsb7$@oq{q$FYI;sSO zfYv(G7*zrln(~RCCkflDZJf4jN#h1t=}&UF7RoN<8T%PlHd>2-8Eku9kA+%;$O^zXzk(=wL?m#NZw zZGMRT@%HRp+ST_}mwQ|b&Ijb~z@h+Ln=FI}aNB2(XD3v_rF}Ukn8CH>WM zNV}=OxHCvg1pWM6A?DR1#(+Epv#7s_ZH@^issn3Kp(Lp zU)8lI8GIl1KyHZ4f9IwC9YUrEU?y1r*2jjIRGvF#5U1Cok0rewGxakN0w{IJ{y1ZXE?6U?Kb+gmg4@$!fKNe?vl(-c=ryq*7wqlqsWL ziJ4=|1m}PG#jFxu2=B=`SKmfXy_SF7K-ki}E%66xbzyzg==!_K8+U9v&DTTPD)>YjtNs`2Gxi{^GH5*WXUoakvel9pOLS>W_b0{Hg)Lp%0y( zpI>c`pig8ZV2T~*38?$OL)wh(k+~g|adK2te|q!)JAd{4FCJn@^6KN{yrLo_%fC1io6`P+nSG9|xla2(1P{M_(U4^8=Kl;X+K}+T26q-h z&8685iJD%*b*_M{0#GvElmcSpF>rlI)?#v2bcGNH#(=ULa05R>gm)I(M-*FeujDp> z_H=~g?f#wkWc8c_aF>^sTeF<(k;mXUcYq{=@VgD|W}E;3Z~~GHP(}0!nt_mvEpbq^ zIwMZ4l`dIYa6Ni60ObXFgF9E~p8PAE;su3ozR{C}Qp#ZxRdBtEfO5}@ zhBnw9q<#W-$p9R{0N{4q(c|Xd>H7wXYXN;R?3sa|k`cX3d;o}<7Grvq!g%#KLGqVa zjTcmq(kuz%!M$8?w=IN#VCze3cxN>|(f!rG(Dkr{@Z8=nh%3n;9vgweVG+??f36xA^F^g z96VC%f(AWP_b=)3*=C*ls5FM`JqLiABGf{h^*KHrJouDLx0Ml>` zW>DHrnxrnweC?L>{K-T>$UQRqXDKF)wzSsQ*XO|vb5JaV4xRZ@oXcWAIg|qnunRu> zE^zIkR%$5#bG~i0^6q>*yr>-Q>|_7x)6+w`*nsobH#D>X3i)ld-=7O2 z0dHfHsN^&Fy>=p$f$uDJ2#b+lt^mUnFx1f?octVJS35^RtRxC{$Q{HMowKs0gvU&%XIg+hnfvzb1tWl;9GKBfHl^B+5zzc@k)RT0j*5*ROtJw1 z=@3FL>BrKk&K?5300;sF++drtv4|XU87TB7G>j)aCd(H}f{v6=g|3{dXTCZUUX-l=fxj)0aAxQu@o9V$VYX>4m;p_-am;* zXJQ1IwSEvC5yIiO|0QqmX_exD^y$J;YgZR$TAUhGU6r$@RaRD7GSX-{w+)2uyj6~s zXU>Gq zue4U;mjZRmr+%m{Ey3{>vSr%_(@eDkB5Mx(1INu>177+1~VJ0%ToavDj@~4lqwsGufC6Tt#N( zgZx;}QR03a$z=bAl1AeRGoiZjnPJk%{RxUCITfLxQ0-QMdgxx*d4^jccko1)QcHd1 zvQKI$o~fXE*omf=s}6jzfoq2GD^zK}8z>f8h*GXGfe)eH8Ic3;)bFA@Nb0Jcz)h) zwsmlB87+e3Z2*^3mamY*^jApLuY_@5#+~SW)HE2gXqC2FcgnqNc6yq1LVP>pRZAa0 z)9l(*a)!qY4EjCHrQ;08qmM~&4?J+B!D2^x@`MH#N?SD7V%`d|eTjS>eVJl|^tv%a z1+hi?sCBF~s|y-b@HoMfw>((xYo&>E05CK{B~S%alP(ygr;EEkbLK^GbBn|iT@wE| zqy{lT|A9i(r(khaWL`5Imym0zq8*P~-C6jrT{~z5&n9*h z##Q+&e*g&`(B4{5Wq^Ao?eI$0$11aGAS*kYEa~y%!+Lj?-Vf2EQ&($jVuLi5)Z_ba zuEnrYYJkv)D@=&!*@J--@udP9f?G|0>6SoPcRIA|zm}Xw{z896D6U1UWThN>G6lE?eR} z4vkq2amg{U8|0wCRR+`~s7ilfdfA;gJoxs8*6ZffK1!^m<;XfXw?Umo+eqPWBaYz9 zAD&E#r^QJuz9=Qi5;2aShU4pL{hvJoSH_<{O9W)_bdWHv?R#LcOrUf??J=KDq)JZn zjpC~V^D$xb^WeK5K-<~ph7V9NtW`$u+tbFsjYK6`avuXU}Du{C0M_j+!EsEh8bp zqf?K2p_PM)@_ZX%z2!1kn~_&05=kfIg)%&({nJ(lfz|&;D2}Ssv0Jrv_jo1}-Yjop z^YE^xB>K06eOqt|;og_v5AH(-zygDUH#>}(_O@j3VL%@VJsW0AKjZWTdFABpXTKk5 zbEM8lLOpo|(IooAZElF7Ah>!=q9}xU5B140LGBC4RrvgNub((PoHqA79o@hqn|Mc< zHebZn&xF074bl&|-+@=?mld{)PF-@RgOv#qI7knAS^B)&11NHkQ z=#WSH;2TOfOJP`nLXrwPlcD&`%%ecJ??|qN&*uVCly^iBvOI^${uQA3I z=XH*IoI+7P3>g!?v$8=3ZYz6oJn!6=>!tsR-;DR{fguK?3`nHl8YXnhI*>#`!lAa* zEvj+VYH1(;eZMG!p|{~{pKc@9=u?j(qr*#5(Yt2J8%Ba~($d@@lRctdqSCMWn+LcR z&`f|DhA!-xkaeUt<5fEX^a6Psu60Vtsb5W(ED+~lqL(I5Ik`?wuE#KP!muVYTCy6p zIxr*LJKfmZu`oBc-F?gH_IWH1t0P%~`ttDxQ^<6Py-(3mfri_N*^DS3E_?-A262vA*_1h-aTD^D;du(hL|%Q{nYbpXYUw#PH< zgx-mk=S>)DSbR8Y7*Bggxb6$Xj{GLPtYT2xXNQKU+}4~Jr?DvZK*|A?5KJT-P6E_Z zy7@j=P({SAAeq6wX-7pIBl;pUy zG;3M`c=18|^-J4jKdOy}&N}BR(W^|ylx|+!i$SWKJ$MHlxv{^Kzo^R%15D`azD{QGD^1JEh4G1S$sa44PDGFLA9sp7JSTYx8yVS zh?Y+*-i@_)x5OC}r%*<0bYI-p(9U!)bxwZUuss;RcrcgH;qJzzGe0B`UfEvv9T_Y% zB~K<3+p`QyBX9vOd^Yyy`1f;{ODHpOaSnjF#K(3dnKi%PC?p6Fy5HQ)Y-X}l<$0tS zFRfVWxv7!&L46`8nJlK2=@@ag)EHwPJR&Yfbl}Oh#JB;vf#o2l?)0958*%*Abje48 zTnj{uKNRz~x!QF*;vFonnO%&kfJdAn(q^@YxVBQ?21>{?2uQhoYU+^iB;K1WuT4Iv$2t3So!2dgxo z!@E9Qp%i*bcJ;fAJx#}9{~MQv;0m>A-FN=8>9cv2_5V^PLhLgaDTTC0#$srO7qcxJ zdA1S)@wC!%El_nyms z+r{ao_^9aKX)i2%s87rwWqH7C8fWA=Bk(ST^5*@T&D+CXwVTz`8^a{~yOT-J0OUB%4`IWQ{N zhVM=;%7gQT1w00Bt{>!KUj{XH(3$4u(xVs&qllZZlomUp=Ts#O*tA+wX}@>cs;yVF zr~kA2nHuwECjFDn_Kju{rqPffi1Oke2OhI*b1}1Jvis**Q=&W-tdL~Nt?vjSGPur+ z4yiMwr{~MOd6J-@{)9LJ4YbwG?>RML$bm2|f~TB^5#mqq?z{MCQiqlUm#Yr3#?TZb zlXk;i?VGhllhElAfYP+TL#y+d*uc&7@mo`NdhU*FwLJ9|f=jgT97Ngu0})EB#NXxE z`7nW$2c-HF&_-w>tL5KH)=4;P%-4CN%aWTQ%piUBlo_B(W%p)X^w-!`9DiO9l%XZ? z93zZh1S0v2tj}D#Um3iST94LuTvxT#i%VL`t}Z7$XQ!Nf=A2S2W}6mw#00l{lHQ%n z$6t7W2pW*tV=Lz9hDza-fh^fn^(UIJ33hC=*TD*IQS>=af5*U%I%q89*9){`$be## zMiwU;?f9w&kMoJI_lNFe+O$Vu>m}0M zy@))Kz1O}kN#5wky2HvR_4JAcPb;3V?*&v_RPyhrxCQQWbs*7jhoG7(CTRK6YeLEy z7EBQsK^0~|Ur4r;3Rb}05J7Dp8adi9Y={&n75R4mux9qDzyZQ|7M#v6c{?%4Ns zj_m69N{aCSl|yJk!h2vnl+xXdiVt5Hp#2c#>AMc4P%)!*THlpNlvFVGWf@J!CE7lf z#I|K>6~o{+d?nhC4jk~7xnd@v*D#@xe#fx7Rb69R@0RB1%wJq~sm54#9Zv$F1);F; z@GeZ72q%_zL6ocYrShl5Y=HJbF3A2zDMa-N#Fz;Ecv%iV>d?7A*oy;QEvU6qWRf(; zi?Ik_`mlq-qOmy%BWAY7W)N_E0K^07E^7eEJ%Lh~j5s#Q@M4Jol7w2w?oe38V|2MB z3%=f!brfLAInWeX1lu$8n3}WSJ?y^WI=1nmB=E%gA?6sRdRpWwI&WnzZ6;1Rsr`Ch z0x7rKnw_gJ4;d1bGeAaLg+3_C1Q!=OsCGeY#`D;r6llf;3I(Nb3=(@9&JIY31X9r& zTB84WDr=DoLCk-PuIUPNwvl1%dtJJ8@*oqaCbRz8)4?CW<~lbHm+)0H$@;z8GQcSN z4Uu}K7Y+G`sDz1;x=)fJ?DTk_*yZoxqM`e%bwx!D7A)6gMBRZqj>5c$_|GUpkTN^s zupgy~+={#wmxst0$eSUB6Uqm4R?Kuxux|~O`0Z-LZGP`ib|@$?MCRFGrJZ=~ zK;J|K-I}#@UAx}VFuG<&1t`cdH7>G;j^xr7FVMHnWBNSwcNzU-I%oeT$ z0`b0R(DoA~sZr;gd@zFu`~p)s%#;5)YnhEro9|R}6v|J~C`3al8hpQ4Q;Aj%o~-BM z_|p6Ik>ml$V9tZ$Evsk8$F+JnLy2WuRTnRiGfWmkl^p?Mdp2@4P)LNT$&Y1RezTLJyl4>_?Sk?D!P95 zp4>3wi_HDdp)pdRVxo&451+idF9C4&SfOtY$b`*S%s#Wi2{1v!IL7<*6BI|52Lr(g zT-!3Da5W*9C}#g7u$0Kw`6cTi^-qSRio+VPG7_5Tm)_TT3i zn#)TNa8+*e)$|>UVSM~JDUzGt!C*f<@(=}7&;lr~q383lB2VKin`)S+bGPL5DwP?A2yLg5JGalmi;Yo($iz$q!c)bnmkN&S^COdZ8zfEyFYzta}-6O0QJD#p|;Md<{D7c0O6d-_i0WMwBK90u!OvxUO-!fZrc3(k>&6>IB&guU@=U zPl1(wk$wK;K!ND+<2@{Chw|kF7n6~}U3z5DrEd!(3_F}BEbK)*UIplA46xR0Z%BC# zV=zA0EFCjWu}OC{RD%0tM(#B(1~(W3&aU^iZ?c-cFg9a3|IoxaSbdT6v{w!0Z$AIW zsGrXTFTv;}rf@b@X>nQpFQlSau60b6Qn>p*tbO+)fP$2Z(krsDy|k7rY<4) z1o}{Ri1!6SG8*bP37m? zhZMw!mWTXmINP2=eHCCFbpY8y@*+or{&|Ow=E|&K?Y?^gPiHp&LoP#W`!nRA6DLQX z7hJ4XN)DjS(a`xN@D}ut8|U^;XNUVI>PxHL0}m9zNk!+|Yfmss5%y;;RxgK@4KJ%9 zUK?s|YdR2AIkF2cggQWNhd_xbpIKp=j_gH@n>bUM(3XrAo`}c%eLPFfH~c>5Ts|T5 z=w7UOgB~;>QuDl*)FS`#r8V88ZxgYTX^-@VC07mi4o?rsYw>iT6r(p5Z!iuCB{Gya z%Yqi#5n#>?AaWQG@mvNA5_@a!4|)v9&DM9a`O-6{Tk?UBpg}t@487%XNrsz3OQ`jK(T~Z$8!v{%Cr1v&P!N!*0$!7_0%<5St z_Z9r`r*#FPDf~j{72P-NwKAp}aatI=E2Ylra)exJrBJHQA95Jh;4dfXaCQZH#l!w! zpdXe_Y5tlBxYzk-okK(*@3wo;!Ef<9X|#TIHqEJ@!HHyTfY>;jqedGXJ&b?~p^~TY zHv}kB(FC{vg%CQZ#DYgLMnj0MuggCKA9O2ogwQDfBvrZ3=ooR({PN1(rUFWsF*;U5 z;4})f$F!a~X7R&ZI>2QZBSz#}akEyTMVz&TLu+@Vx%8ZtP{~<6qCDX7i*J57l4L-T z(8!ZZ6^95wE+W8%K!iy1(eU^CM0T8l}cXr)kl^u&x>F5pLLuM5VZ?Q2 zE@%5MNDB$>8C-sp@zI^#CQhVA3c!8hLP~Q_MHmHE8DxDaHbY)d)cBkANjT6CT||?0h47j$B7sbmy;s-NX=A)@5!F5j$O@{u;gl zDp19QdXXMyBZAayuPgPK{e>0N1_B%UE__Y0HdlA+g631N}4 z>719;ZFB6KJIfyXIU@V=MR8%lP6BSbmIRtSZuv0w;h3-EH*<0qe2S^NZrN6mEEFlF zUAr+D+b+T3>?AkcdP7U|2T~?yU(A;lY@W1AC`~?1LS5`;@&7olVT$D(LvUwVikFpq zE2?Xf(8p?DHN%oK$`iqKUjfmaFTt!&G*eJ-4$@lrMU*>JDRsU>=?VxKQoH0EQ4?^& zke>m#qkO>BAUfHa2u<)kWz%SMxFFcXv$|;|C6s{bRYKjA1P3#NQ1X;%%}$d-`#PpM zA#ob-S7Vcxx3jwNS%0Pa&88B*vQFKyt`$lK?yi5+J^9Y9R}iNGkW1BKKzGdYq)L_0 zx0}pBQXT>yjB%myTV|R(L62;1wMmq8OR6BcdT_PfLx89!c#YlX#o}w# zi?VK|P`j%^X9fwQcm?y=QNrc8NDZ;prT!^pM%P)ArmV*v9l;#f=#Cd-Jg)B7Aj9+{Ha!Je{sqj-y zwG?cXwJzmS^x&^_+I!?0-VS-*&g_%SrV#7!xv9Z+#n6Vmn7E;UB+$ybt~lX&UOKzXLvRE^t10Pa+pji7Re;3M_cl^C~2*> zU02TPGI7rNxP4m1WRG%b-66^c72aA%4el^SxCl5gja7ah?k@X>@3oQxt{fi8IKgMvE^rmy@mda1(rO%Lo<9NF#H5B3JdO zz9ykz8=Ak$Lt|Q~G6w?Qq6L^hW*FI8j?+3);DV4Uq*vdc!~skX>WNT3519JuvY%%q z&2;^PvgwP-q4>+g%mugAF|xbV0HcapT0Tcembpwi{J5z-?AO0GqH&)H{}}Uoo^x5+ zJrVWEiX<%K+D+JP_alKz#kLWI3xK@XH{=8$XS)j{`y&30mF_UWwKr=dc+y1zeeP;Y z!OYXs`6GCFBB(w=09tG)8Q|z{6+=DY^}YZ&sIOlau6m z%C79G`x{kMepcqLCj@&zjv zwAma2O3}$DNB4UvaC^0PsT-c_)R=8F>j@gDmbBfq-yG;Kx0`EAS6OIBZB5f-93<&c zM?b%%&qpG^dG?=o0rFV76AP5iCnnTQ>kjG6Un=+-OPW+N)RlRD+S}dD_RhhJR*&(E zJg}5JpQ!)hf_>e28WmwQ75V2I`^dT4=}+~9XLi+hbM}qrv-X1@cGMImNhv%$8sm9~ zVmzBqbB2rbDyGpU0^a0}PnC@wrkF;=9lX!5A9ynAkxoexjr~V5(IP=Shv4=>Zr`sn zxaXzU-3fgAhw9wzcK?!!N-EL8*$>ILE-kbHYPvp=BFa~94TgPRZg>AlpHQ|QWPP1| zzdXr-pRRMLZu%YJ&5c(`G>1C6a@Y|D@6{={>$TXa*;TLTkmvn-q&~Z{E*M|e0q9Uu zuF|&-(VlAeyVjlx&iojVtPw8}rF@9_QAB}mbT9bF%ZB#_Z#++RkKR5w9pg`*5bHOi z$viEK%U%7`F%{B(-}B(9pdMVsOEp@M&?GpxNOf}jVnOXwsK;&SyofRiMRL~JIGO(4 zf@oinv`8$e5#`Ps3L}F47O$5}ZiAa_D@Fzvu|HsHf2KcECsthD8eYpU$j6>YqqAWl ze8d_HW^ir6G<5O8^*cdC&ZT+PD6IDFI(AfDm81NTG<}0 zvC)F#iJf-rTzBH4*K*S+)Nrqx?RozyT<4^5SztVyxR)03e@2O5kLGfVLN&o~*cA6h zwN9}hy*g@o8Ls2mdAAb52ru$ zJQ=>agbVZGrTmeXxBD?8Z1uTaEoE-0eq}%Q60h;q-T6$Q#|+*ezV2DO@u9Jdb%nY&|U1`OX>Uc?RAi zI_AP)vy9kVrse^rHKI&rR%E0%f*B-w|9@ss?d5KSK^d8X)b0EA!I9`am&XL7h3J0J zM#OdO?vA4QbEiJaf%^5ogJiaC<&@4>Jl|!yPuL%IW!#)S((|RInnVM^A;L!>sle0( zuNgYpbJ-!hvsr%uJ{Vf`0~k{coA4kDUG-euP9rjyK|?jv3?i3MBhd=JeIlV?=%t19 z0txY>G12W6aRWU4W#8cMSX7`%XyFLf5$wSAzJF!JCCO!6)mJB^o;DU}Xg#B4MnzBl zV=ZeQr!oBAyCKrSw)Q*sHo|VSKi<^sj_&APr+v{G(l{e%yKwWF&c7EPZwa)lPF~r| zu#BsTB{;Ve_tx){b&;({aB^H8*xQV0Fe<^2Vj=J{g&+w$h3b6?Szcns>qKnu)yY4&_cshR6EjZ%2y0dk587^7_ z#&{rqRdnfUfUP~?-S@#sBTP=daB!jMs?LX51@lL(5HRMMz*_z)0txHPPfulwbQ3DE z0?aHu(X1+Lbu_CW=`AYS;2_=&v7^NXiJ2FTuL|qQ?>M0J6BzB;4J{b-jEi!velZt4 z=-{@srW^emVY2o+eSHNjeq_2?#?|rB`Ou4UxqD$oJ$-9>@Qr7K1HpLi!gDo4mo*?n z1P!LKb#Cz_^K;1L@8~x&W~ne(8)ZfGzcLDeO`8#j6w(kHFI)A-C~6=@A0x>nMptHO z-JBfU+(R1wAVO898SgJ1ki8lb9XsRMq~F*x*+uC=exnTx;po9bns;W{4>MghU~=mM zG0!eQcnY#JD;X3X_%O#{gUm3WAYlwWZ^NXjm)~~_+M?K`tP`1H7nGXYC{z^7Z?~!j z6ua}^XB?}=C5P0uepJLX#>TghPbC_vdNkJJu5ZiqOFG)_T28)E`w|S9*8@1i}BfJ*pz8Fxn}=_(Z8tqBBG|++yG< z((+MW)ea6)8{hNXOH1!jD*}z+Tjj$pw+iB-dQ#l~VVgIpky+N=FFOwg6A`+>fq0Q+ zi`5k;cTtRv(rAuo>=UMh_c={^r`jM*Q<$BEp*XiV*)s6AOcIu+24 zF8bl-Uui+0*Rk{W-L1RuCC#PS^KV~keVB)*^B!69!lI)&*pUg&cn!?(mb~+Z#r}KR z5U$oV5~17S?zZRdzmF@58uVQ0nUPHFUG8=EEy;gMH0DmgJZA{J!JVI-JujCmob&e> zhcUM9_vW>kg^D{gkd0vp_f%b)Ki;GsXn;7;S3X8D)a(YIpGrapo~yr=XT_WJ4p-xO z)}%8C&DZ}Gnz`6N5CC-$>EWbNdAht7)5 z7KQ@KBQkRC*AFKDoY|+yEIKfco#Fv3FhJb zT$W)&@5E}0ZeANedD%WW+Um1aDyhQ8OK<-7@dkN*PChVEs_boqY5m+D1z+4gjk0aR z;|B_cK3Yg`Hhw;cB~o%ML3lIU!^~A_K`r)T_pK7F-s7yXyddCTQR7c(BtJnXS}S|s zH`x`}g#QLCQabpMrrVol+-?VL@E>4RQ09b6E&WJR&LheSDl$IQwHFoMn}6)FO*f6? z%o2ES--Cj7Hq^c?2j(AKR2O1R`-d8<-2)9<+Y7w{^;h5dBW0`eYL^cx*U@@;d_Fs0 z%vx0^2SGS@6{dkN8()O7bn#VT)`(X{$XQ71mq`=cX z>iq`B?(p~ZqC%+VzuY2j1a-gO^%X@8^gzEp3;!Fo6C>OyD#%jrx&lAk?6+T!vrhXF zNAc9`0AQ5AgUareVu9Op*W6E=L?sUam`F~^=#JiT8g@9I7Hs4zRP15ofCVVQA?VT@ zlcDx*u#5H`qj2kZHH4h<8}dc=->0{F-g#NY=8JnQB>Z|BsnB@$-kpvmJO;H_sn&ha zSQkP4K?UkWY%mIHQ<5k1{nWx_-58$W#PCE5r>kqUo9gR7k@|tmRaPHS3A~& zGEgW{ec$}+Q`}t#b>=4sA5YH2S6o)$?-K93+QHtJk+b4NHgi#xdwuiDeg7 zN1>O=L8+HrMA1LgUqd`vP-8yISB5fLWj~rTV~;YLQsc8VFk8NL20b@}O$kYybFL0i zjqEQa`A9o1J&$MUulVjz;Wvo4*E-D(rj`1&g4)mL;QZt6f0xvc8>W1=*y{prPLq-{ zv@Pr)D68RC;}+Mn0WI!By9sk_b9Ogbdul4*AIKRf>KJ1r8ft&y(n(Lnl|JPtdk9{R zH|(+~Cqv8|5(KYT?8^F`WkaHapPv7~ZkJnk! z^5+*I!$m}IL8WAu^#as7!_L#v-%9|7zq|g&tItXoesprrvj=Mhg2265qFOt5?|xFy z?xj=u8utO$|6Z!)x6JKcvwZbot5I{A@n-e_sDLaxPCg)bK}$LnihFG%^Tnj#v2+DL zS~Qw2=oqp#$z&-Un0~F5l`SI!gBpg;Ni zlAUk=V-v{-vn`JAV30r);n+{{_df%t&#OXpT(Q53 zjymUH8=|9k?7M52)IjlDD=P;xeAW~YHU!+^lH^0(n_{oNg&hDU+%65ZIDdQ5ptGfG zrBD1$9Q>_=M|Qc0?JUG+B!@F!I-<;9pO~^ z2+@NWX-2}Lq9|i{1}%)*Gm)mLfb_ay=6VD|Na^Y?Q^5_9|?wU=(8YxwENegw*F$`<Y2dHg@LCY9ggwRkl;=nuHZNj^ zndU$lqc^7jP#XA}(@%5J9GmdglK$CUPVM*JUYIyqVif>q{Q4cZ8iqlAd2oD;hFGq= zaKQU0+bJ@7R!-iIQM%*vi~6H3t49oH?x&5XJRq9VxbDRx%arnjmP%x|QbAk;AcWgw zDGp)~EZYSDWWnN66jO7`d%hiC=Qg}g9>W_B%|hl0$vjdqI(MDT{S5!J9=~1zXYFMR zw;V>3uFwRpaafLHcNjWFnI{#-1Oz&Jrf*$ z<>ANy%u@kv5cL-x36OY&OWrhXjGHuI^OyQ-zxB4keC9p$;1>WF>R}98<+DC0YZkqDFh2jcAy-oiX&Q+69Zm+CGEj_s#v=u`8h zmV95LlXw8aSQ$3cl12{VzMm{EpK5TDbnA=8EzN zC1=EdyfJ;^79k_*AGycYTPe?ykb<>V#bFuu=4jg$e}=Do%ftIs&kQ5Tpor6GVH}== z$G3nxWEtD<4Mcnb)rASfm z+)fa#a^yV2bnE4MCA;Nki*>v>tSK?N-dIxdRKkAU%fpc@EhnE02Yi>n5OfH}?gIHv zPy*wEg69b}6_UQxAN}H0te!Nnt_OB&2(CsUcdm6t@D2GC?6L*>0Jpsvw4GKh(QJv8 z>eO$W_9mu^lqG?76O1%ezW6>~m|ouI|Bx`7SfGYE{JhMAKtqv|Yd{gG|3#@$GKJ6s z{kmbrZRN^n@NjV8QPI4!o{;Nt^5xph#*ALAoA+Z;rlG}K;^0Ao%^~>)04D&d@KjE3 zo+3_UHD8b(?YH=QF<%Yg5+UN3r^5-GFsSnnSl}Jv0dgYbg+k^AV7Z5HJRt1Sqm#;w%jEJpi1@MhC*;VyM4=^8l)%lW^-)3tOx=$P1G?|2wwnNdY~;ZB{`>wNr%xF!V0V zhu8>T#s2-%FZ-x~lEcV6M|{zAY<2_4=IC}4Q=rTND0}b}vF2w7T5j>pf zBl&>$s(59-kN?V%NU@z5yA=!AgKtQP#0W@gv*%?*k=FrLQ9`BKAY1P8H0;~V592Ea zRsJ@wf_fT&h|Yqc)*n>yZDsWkdn%V9kYNkIL4yWnYKaK4C>@Y@&4I4+n2N|gLa zy1Y1h^~(JT+irx3G4nSJh<5Qe66q@Xd`;zd`U3zQ*7(>!L+!=#Bow1F1a`TaTd*Xm zR1Jn7796K#(Yy3ra3T?(Ki3hXn3i(5aLFA054f-_`e0G?Og&NB1YipnqkmzGTCGy7 zOVNK}3l|7m80P;^Y{9buVGBdhpOr=Sow4ce@k*fY$V|u--;T3IDOQ1A^=7w3IJ<@w z{06idHOsS&JAn53GG+oVMMNYPdIGuofV-H;S+YkNpxZzq_ek&uKY57c4?fIFrA?Dy zB|=x0UavZ}V5}B>23z~P_CjeRYk1Y$^BA$Vp|%q)Mw zPO-ojihI=M^rJ5gU=@?dORrtv1ONo_rc;wiqNmYO#%jv0UjtXh0i*}tGj6Fd!7QuY^O8<5ZduY-y`)Z~A_nr{kD8yR?k$E%R+*0(Raps7WXiZOx-H4= zAnI&JHmu*>H26)*gbSU2*|W_&qRtoXy%qrQ>j0D|mj*2hbAFG}>_B|h zA_N%L11<#AK$O)}^<-(0N=@wUZbFEh0p%Oyc8JDwt#}JDd;qC_HS%m!o!hr9V`7-T z0inj{u_7O5dHmL_&ICjf2&fIPq|hQrA9)|ySORAQam7eFCdD~iIlbqHGZEK&`iYKS zFI(RL8x-HIy0-^Q&cnN$54bcQ1Aj?Hyoju~rKY3wc>niCMmyKqnG{+ExqUkNni#=kKo;b37!p|ELG;(- z<@FZ37i81rI6V=THTKQ!zl5mjmamvS0R~Un>+CyYVEarHRiYa*iC)@7DuJG6^!+Y~ zccdlM*5&ytepDC+WE9n!c!Iz2Jk}c<+ToH7ih-f-XfL5pR}Gz6X-A!B&y2NezV=bX z?Y`F1p{*{F{}hNb@gpC5^}3TmLJV~wj1kqSl7kP7(~@Vzr0Ts4A_fhd*J(&x3IQ$ zL6wDgnv7fm-bR2Z5PJRt2^x`H{%=4696uZh4Gh3I+}vTdQv&{B7|kJoa)F{RUIbUV zz?Y4>9B@jW^#Z_dFEbIUxIs{esr^nj!M1X_34R2<{8?BSXz-pyA|ix>+IxRN)9`*r z2}1(Na#&CfY|g}kf<5;SGmw~y6FGH6fgt8PBqg=;+!vGDulPy5-gCVQW;r*&Ug50f zmx0Qz;r3>U`GK8USf>So7(SS;V;RcJlMMxjl|%ZgqeJPd?P1KhWYFm+Ch@{SQHBfh zeX13H_k1paC4Rr}b&}N}^xWShL4<(};nq$jw*ww9C;#Vcbz4UnL?KiSXmdCO9zXt! z@C~GEsIjvZGtrKxp+Xw^Qk=rdiyL-_A2qEAdXS?$w<&f4z?=~R<5=Qx6F^1Pa(m+q zzl72h)S`oG1vOLW5qrKk66hAxq#?-Ea52DbfsT$!`)zZJ?WOZu0V=|(=h9-R>CH_0 z_?nnqpqPLJ$Of44&x!BB)HgvEg<=q#CshXW5W2C^gTp@w9t%gn*8|3D6EFkRemE^! zJHYV-*@}e7W&q}Ar7BZ10N5byX8S#8OIUXgVWbZciM;}RF}fs^zC0F%@k~pAC;=>5 zLQ0P5-Wz}UvM!JiKG}hzh$OIQ{|wm^9%=HUSql^_@qw3@Q-*-=&;-8$-LqZ$1}q4R z;S6efXeu<*_(5Q#t0Z7|b*7F^YX7&gmlE>d${yHRoCyhDf)aT6rB`2sp(R>Kv~VP? zVcnNk+9waQGG3P#x06DV&~fuqvNM%M&twTSqtG#)s7l1AkXAH94^(2XWjsFa%AmAU zHeDb|fHVu*nj#Mg%WgDRP_uZLjC)zDHCOcwB0yjGEqnIv=u}7X>}4+q72u&mvfnsr z^j|(5Uj}QZ&{7gT`*VpU{61|pdf@j z#^~Dl$sgiwSw;~Z!vWH1O6;l<=mjJ7=5|0~dtWrM4d%qseMbiQ-uK=xwg2DRfPnRg zGMbOwZ?$!BhrGiK;}pv}3mo(?cc*02IjI_15&|_TX@Ptx;wcRpOUy}dA$M(FbNGQ>#)>BY2idjN2iRwOz-nI1|tRH1~v&;J5X+{W1e literal 0 HcmV?d00001 diff --git a/docs/images/distributed_Bb.png b/docs/images/distributed_Bb.png new file mode 100644 index 0000000000000000000000000000000000000000..086c8b1ae4e86657f9dd7b3cbbe2a69a211c4aa0 GIT binary patch literal 26099 zcmc$`bzGC}+c-QxN<>r?1*B9Yl#UV79iv94bO|HIMvPKKMF9gv8Kr=TC@9h~LJbm7^9B1x(N(M z6$SoHqN4%tHE(ab!CKvRPgj>eKZ}cs$qI>y35iOXf?p+6#NgmfR7_Y*5^nwHen)p#-+$IZ zph7*bU326W*OU?#1xxWG9GyLV(WpQVUI|t3UB?&gdJTMn%i!G*3EnKhOH5Q)OzO`a z0T{46TvS+809@B{b@m28X+n2^zhdB$re}bw8!8xFgl_r+fGgV3{SUC(Qfk-`lw^Pk zQp3Z|$xH=^B>|^m;?xUyEyoJAv9fd^_{Fu(bg(|7VvfS`Da-pu#^x09a;o# z?&snq6)34`Br4`=jrF%u3H5e%l9u)tGj}pK@$(Z8m6Vp$QGu)Kxf&uweWj&kowa1d z%@84$*6LOkj;7v7r{F*<3AC$>v@FI))kjiWN|sk#Rmaf7)Gfrs!^_iJ+|3A!P(c_W zWYOYMaJZztq^6g;x2m>^iH4Vzv6Qu@o2rJJkB(-jhlR9Gh_l$^U~!1MpPq{aMjheeVE}#)GR7c0 zbY-LsLp+SoUb^Bg($}O^%p_z?{6m~vJv~j_&CSu4NVtlRuBf-8o3RGgBv1n7Wgw>S zCT{Mb@8@aYspaCWp@udI3PqUvxahiSS*XDg8fZ(nkG8rDT0=`Uz(7=A%NP|X6@=C> zca+ufMu(bu=<5aQi+KR?RYkgC^)1{)wOm~MB@D4%j#z(xYaPSeU#KHDzR+@%t7NQb*>gN8A zn%<7^5GS1gBvLb2^_m`924yT^uH)yf14tqX^w31r*a(BYrfFbm>>eWRt7++|j#WeY zifRT1s+t&DTDgb?1O`cqSy(zF{D2H0jhv;CKqpm!Vymhl#ofKIGCrDKNNH6y3rXt$ z11;z$grgNoOgtnAC1njqhI$)o$Y`R?1CiDqdOFU*KwxyO^x&axD!vx(M$U%57;6i0 zH77%D6bg_-GdKXNZVXn@)DweCIAM(~%~U*9G$b)TCN5H9n$GT`s)l-!X2FI!E|yLx zv_t@aLsMTCts1BT4{?!ye%y(W%XP%bqzzsL$okD+Mb%; zSXo0u@RToF-M}l6)~&l%F;v!&oVBi&0RJCfr)xR81w+B*-t& z0$f*-a56*%8c2F-T1mq7kpb=&=B8FoQW|>F8shqD9G+t*_<3j= zn7Qd%q0CW;09ij7ls3}K%P7#rSwhOp18aZ^7WXk#mB6|~2x9`ld!V$IKgJVnjxooY z>AD%J1-g2A>!38H;9~A($Y6MgnZCI`5^WCdbk=opb(AuOTd1hHnt}biecg0q{dJ8n z7(FXV6DL);fiA+y%-lc7GT2Q`RNGWr1s$sHsivo)6Kw6_=HnOO=o*BOG&TvALF+kt zhPJpvUh%wqgR}vZIrRt=qqH1cXuO(@9%}qSS z!W7}ErUTb8(+LW}g!&lis#}K0SgMGr10mM*v2yYZb~Q3E($|D`LtK+V!`0zPU*N}F zrJa18^(4{3DzYje7>uixuRGQt)C%K)baRnZaTd4MhXdI}I2o$KLv;Ntd`&&IJkY-0 zjz*3aGVV^nK^i_PT3$Y$9u@&uBVSojOLIe6OLrL)8OUKssDZuzdWr+L0l{+MfPn)T zD%Mqi!MI^M>MEwesq<-(wx(u#`-xUVi+yXz_c13fD$_W^ZjCq9)p=w-)SE1|L>_T3 z-)x6bNd!iItsYHmT754>W${$DBXDspJ+OoK!aJ!~#l@0E)z#I-)zv%EadB}Ar%s(3 za{v0&|Nex`VdYE#5t04!7cZQqzCM${COvtgXlie-I8i#}dzUwR+G6DX-OBQJck5&B zGXx)*Rh^+GM!l(nlW4N32=#Do4xY)Oy4PU+3b1y>ZTh5^d#n=Htt~Aq25E9qH}1b6 z9y1|(^HHB(y};6%ukAn1ZX~zN2%CvQ!L+qRx1`4eg@odN%J`;{+N>(k0(MmxAs-)~ zoVRb^7H4JMz+fi0 zbWEBkUmD=Db8s;1NR#K=rlpL6r^;g!`uh7<3&(augS^ zm@s$vD840mZI%&7ok#+aSo%dw6=jAo_HPI+oF#W&JsS|O6cN+t!hvJ)k%ewpe%m;`rupOPi zK#7u}1-rTOR;4?4@hMM}Oj!-OF2z2Ooa=3u8kwI(T+O+O;{FcS0`vtiB>lv9G#2z+ zS+1k{9!`Hqh=YS;Z8U5t=WJ7_{L*BUtD;~00sLO^^+!K^t*4|e_l~BP!b!ayYf}3(Yn0+Q?=k*dxzsH z`rQrMFDG6rYcmj)`6>Uh^UE*1Veh(f`;u%z)=(KLc^7fy!uG~O*5+tf;oT;#$m*u0 z@U36TcErtZYD_x6w>ho^-xs!gSEpfPgTJu9IPk%JCV@kKnjKZGqC}oPPo79rYT2{W zp?Js(rls;o)Hti_92aeySrQHzRz5 zb)-5(!miAh!^hK;K2IxLdja2JW^BA7K9pjy2p9SG#=0c#gzBN{?Zqn2T5MOgPQg;A zhvdVZh{)YJ2kP3P!4hvO@>1HBkcX2~Q&ly5F~{WG+s~4_4V9Yrp6O5wbz!Y%?!|ZJ zNeOgLrUXY97tDkafebvKW6xk@m!{kEsZhEk5RHnaAkc5EbsOdlT-%T#wBg~25xeXC zMUieMFd`)tj=9KZNIK==!ylbxw>U#@rz?b+Z*=G%&R%QFovpN1lTBc_GgTCPLRwKV zeQ)_AlP{5dzJXIIqFT#}Rl>bJ8{nF}8^|EZXOQZL7{Q~eSDA30cHNr=>>+xQh}4dt znav&r9D^P4(*j~XeK_NBQe9acj94ify2jt@RZ^3HT3K1)5tFBYV3~8<-`lOJ)TDd< zh|loo9UbaC+-ikPf=!eE*|&v-Km;C98aA9zoi1{19VWoP-%Nv}(#MEh9+I4ePo$-# zTlOk+RKmj*e#s=ToHj(U`s6j=Hh8OZr}9^uD}?>G<*Ez7BYWu+sSq{K-1_ zJ{2)_hcuBWsid~CX0knVELDBU*TVP`_ZQF6Qd(zhJFTqW5m!&U9~2eV&$d z3&>y>&z4xft@S&rg{X)ucQ4Mlc>c4bylh`XBQKwYCfkwWlJ==e{hDjs$jxOTboku- zd^R;Tb&Wm8cU9!Om0ym257(z11|8~(UhGuhuS@|cfB{D zL{X%#$#HEW?yZW1+gF({{%3uBeV^>cKRo>Y-49RiJziP~2IkfgNc?3%o3RW;3qysK>-hFq*#>R#ns0u_*-_x)JP6a`kmu+s3fLO-C z$dfkj@j(lis(IAIq}w;cC*t9UPUxq~bgtJZ(1-p0sRQ>vckY~rib`}7zJ1H$bR*TH zvW9e}$aJ=N&9eSRj-xy==H})bTZ&gUzB^yf*!ua%@KB)$#2zvseA{JzKKuGewfu#E z;ob4~$d~Ma^Dn1+Qcg+-8Q$ydmnr+bzW16D$KXS}!jpbH2}l|PX;Nv-hKSo#q8LaG0R<{+eFfm7|LaFIGXcf4Qk03U(ICuAsw%Pq{Kb@^DxN@4?Hk34t1&@lbVK*{HwFAvU3UcRiqT(wa`6WN|Bdwt@LJL%Js4&=Nu(oEk8 z;e?*pj2^k0BFZQD{G3wny!PGJV&2(+c`>RAFDjzy5ZsU(CKrv`qav)gZ>>&wy1M3! z5ykOs5a^V|9a`M#J(8l$=Szjv=eFH1N<}ZX^|O8I^S;$}C)_>P-7#`!MXNsGa%nuu zuputl7ubE1WfY);fuF{$-Ff2v9=5*2v&6cLh}t2oaNaycTRKvOWr*+vX6t%?{~N3B zxouJB&-d*2X;w~j5)w}9zp<-&5^z?Ff#L2b*XuMhsc%bp9>wPfgoB0NN^cubqQqj)AL0w=; z%SNQvxr~i(MjkLAz;x!&$ESm94`$5B_639QrpoKegvV?;{xTrM4Tub9a(4|-KFHGn zYYxc&b^PP4ypQ9{!S%oTY*3J`eh4E?2C7&-cn6H=8n||fu0HZ2C-X+DNbSj;og@Jy z1i0%(+M{PP3#)&}jd@=(f8)1Z0i-hYWN9I7>0K)zM}f>}z5{v3erg^0PHc@t zsnY+W6>FVZ*LibdV)_)yATn28o&5T2e@rGJUB+K^1Fo$b%3NAgQ=>((IDSy#D}-eCI`f?$_jI$MmY`<$MAf|U}?Ys zQV?UV0QL8c^j1+(`7-adZ~!&_QNKfEny_{8ZRn-AJ^#VCFgZDyVXwx>8nfUIbXNtW zDc{>A18QYWy77SKi{2TH4H(Q(T-!$d4O(iY(_CCj8{4j~t|2GQcPwMOa}60H7I(H* z*Q%BXft~#~d>UJ6Sw!qF1B;hI_2p`W#{>h{y*NQjQOTk)dmmjhd))#l327hmhjX&R zFFhT@hpQO?&|1e}RX{B%2tnsLC+PfrAC9-FmE7PUO>&LlnO9<>NjPJi$PX>9@C(5H z@J1}Rp{O0hj&Fc`%eTAN$*~dsZKK2+siQ-`06Zlzd^2r`JPRk{x`T$-=lZL5tMn~b zs1#aTTkTAs014faByhcdyOMPq7@(6oY1VI_0H^Q-BuiaDU#oU)0)RRAxO2N}U zVZO7YpqgsodqpAgrpP?-?8E7HCs|lDQ8nx4iM6d|#E7A(FrHDmi|5W=XJKKfZiA@ZM>ozVc(b@>Xc2LQL_-`_hRxFuwrEL{>O0FbgzwcM zAArG30%_=^Ara(D;~OZe%Pex3dCRiGTU|HCSR;P7iBLOkAKL(|_3i$Rb5!8S<$H43 z>Kv&8Mc8`T&+gawTMU?z5%ZHWHgw${x6*i80dw-+!-psSgz^BG%*?u1e42rF1y>^+AP!SFL&WKxO0s(<{ zVa;UkQU~1vjE+ci7&oU`?Vrh4Y%R1qn_uX%Kb-xUiNFIR-{{S^U|@3r_N-}X=)JR6 zA&^x}7YJg?^7%pZb(=FWd}H3q+SYb{Lt<-d3k}kf>xFM^$_$o2+&aqf0_2QqBov6< zyR)YMW9j?qnX5d!!ZWh0b z9-HpWLQi~-M>ocuT1&%@J_nh|GYfbgmptZPfkSP2fn8PKf-Ei^(9_{tY%C4AfkEWk z=g*oMSfX6e>`5qR2_L_CdU~`b49bA`wzmcOz9xPl*>Jkem1eEq{DA!J^&4^{z_}+` zGMvusw$i%47Uvj~WUun2Tu*(jR0M>Gd}nY}s$=+oXjDBd4k=teHq@D=HA7yXo4G4O z(8kVxA1{56cs0&P+!1G>bEKH2GHmbyioiQ;iAMoN$W46DUR~~gn(6|~{@(H9w!DKC z^w8%2rlWs17jYcutD$$UV2`ocWZBh}R6ZJuf~_6>oACmSj=?DIU+278B6BhOsdoUf zEkW|)H4_upU=bTv=cq&f2?ZHIp`5v*`Mq4<<_EuCQ1NN1pBl@IQR)9C@jKJ(l-#iB zmvF~NAADFuVgS63|9uV8XUfcuR^{U`is>7BbooH^(F}7N%LYg8!`3MNg^Kn7s>4`E z2CknB-e-{b#YULy8Gv9o)!%%ZRDVESN^tQHk)R^LKUmkXJ8HeOn6Ya}X^lP6V3$B4 zE;)cDU9kHd%8-UEyF2*g8k6;t>QPlDj@_1}C9Xf)PKz{Q7qSrylbxSE8_|KC*_k<> z^;bg4SuC{#Y{YLyieoT=Iv9PmD{=f$5v@et3YISBPw`@RoK%o@aLJuc9d<}r<7(8Q z(IV3NZcnL;yxoo_`^f4HQ}IOp{vlqW3~W~;s#tcvuYYrUUE%Knj!QcgV9N`G;`^Lr z#79kXA(fCy^X3vNOK5fJIsWd&&YIOJ^1`m#-_;x38m$$Vk`%J3Qxr1xlzsF{yJdr_ zcF3F!pbSDqh~BGnYa~QQ;^%pUzTvvxJ@6GyQD@IRwf&zkED-&UI`a`%^<& zqh*2pI_hpxzh@s=6G;~ZdqsETex1!RSUBz98xAQGmHz^9;x&aIh}ipxm6L#6s>dOA0C~)S?$;aj$O6x*Y}{wz2f~mIeJ|x5nhC!| zgou+<2mi*k(Vn3PKaF@Hw7Qr>l)MHBOxzNYsA)4fa&VooRPa#U)v00k$edjKRJX{e zbhn{H-?#JY7cOkAxiaql@R3j$xss8-JzrQ9DIAbhOmSktv|<4bbfCbh&d70nP)LOX zHjWPnry@`hlkS*=%M-kFy;(#}JxEKZkHc;}UNV&K%!=%86|OdsCBqeV_go&$XFRgy zca2h)H`9_If%I~R91*%%lvZ@>gK_||W%EjG?N$1yWdsL-f*6IGRn{SJ5KA5{rYR*$ z8(ey#kV-8{8hCvrR_Rb3_c(eLmweUX&7B-~?APqHTNznOTttJJnJ4#uPE8DXPK{4Z z4{%7#oc4CITk3d5W%y+;q&fNojdn3Is1#{PrNs{;UT2ab(Tv%=%VZ(`3!O18LIosB zd9eebKUKHMhqo0^QzB*0U%!;LN}GYManQAj5HuLi?r8OY0^?s&5D*f z2;?Ia-^FEw9e*?QJV(w}te61{u$r_j6&-!$eJRcr(VIIsT$Q&F8C>|XC#!qYmg>S_ z9(icyy>pWb*PHCgo;>8TP>j(-6pTPEue-|VoO&5V5l2s!PVUSnJS%E_Bi+9ER41{A znf$9jwkR^egr^^N2g7!DVeR{SXGrONM=H%WqA%T-l#<#yyElfHGyadYenM*{iS)U5 zo?nl_a7Gwl9`tHL{CE(m%0zgP3+lT?kxxvxp*7<$Y;kq)bqZn+0}jcxjHuy;wHyg6 z=YPVnmXVQ&%GH{sSJ=58>pNI1a6yi%0%Birx6vU&9uw)M{TQAMgoK*H$nCqZZh+n! z^Y7bDFB>~jWkGuWn$mGw0TAv*TI{)H1V{yfD2YAvI3yokl=zK=rdo~;S}q*-!F^)j zHcApp@&9noO9XxzBHF&~DdEe|=78cXq?ZN&<7m|EA+Yne6;kblCkj-31I^0co#E~@ zpujO$|MNQ`kNJ@DGnJ-^{e-6eTS}K$kKCWV&@<@y4)zC@KY(3_@k>kx?+{lc4+?{V zww6bLumIKw{TJYrnO`#}RwA1|cy?bLP_nqma2KfK$-1&kYOu#2fK*^ftmrG#!7j&G z0J{aEP%40=eE+${x0Ns4{~H#W6Ax7Ww|!Nb&ffs2sl7@g)a2g~4*jL(WmBZUZ?)EP zYi+>}oIvn5z-4HOs*Dxk^1o*V$@O+0W+q?gs5H}J+g@tf+vS)-;yLWMJG^o>G?LLy zG%S{#PigSy$~ITJ#ll7n``+*3d5$=1InF%eR@hc+-tJm-m8?n?xpX(YeiIPJt+I~i zT9`Xi?G)xsrN-u3XlnZO4h|lUiymogiZ5KR3J;1P;H6Q)&j`C+PJ@J1)S42XqN$lY zB!c&7oisyt&$>@P zutRl4?m8FF;ut2!rUd8&zrhdHMgLfWkv<0|nyB7pkF+9Z4sAyCRyIy-_9Ypv3@tHx z0OKvI+|?~4slj#0!vb%vbUt|Z5eKTgL}8`zR%GPBgOIVsPsjnJ$5?tmH4V ziLZQ`!40{Q?!04oNy19VQsJdTO~&2ju&FUT8&1A7lB&cdC-^LJvC8}6?#wjp>*crM z>kiX^9mO-k@(LxBW`81#JgLI2msRrzp$CC10*nBKU#9jcSjEF;U*dLqtKKn~S!@c; z7uqS)!-Kt)O-JhI3JWcN_HA&cA_Yk4;duMw)rvWy#JR}GOsbz1AD1c^7oA?eJl&aL zxQIl{yGk6d$DM0Y;$i&7N-wqjc?xe>YV53A?M20%dIP1qxzH0$^SW1aO^(pmdP^ySZf)~2H;{J#j58=AEu~o?6JW2zdnAI#1>FjHBLo02D zE}g;ilG0rnrs(V=XY#hG6qQG@{rYA3Md<}kBHry$Ym^~N_4O8+HS(@Tir|pRDB99D zFHwC;nq1}hEVlU13s}=Xlz)56AiB8~3zIR*J{%Qu zjJ*C_vwqW!b$$16)IN~FUB8LeV*j>)Q1ntL8Is$nN88heT1tNP4s4qp9oQnt>HP3sfArG%o7iM@2XwS^BkZYS# z?0n|-!{0Zp?*ggUg1?Q3-CKG;Nx^tuDPx%L%#riw8m6~umtL%)8GJ^Ui^hU`b2Q?7 zb7fd>0@}u>`#!%p(N@#0`O4EaDsUOmOi1$>;3u)_nIGN-=QA@Rxf|UJD8>G{rPt~} zGf$1PQ;sRqqha?Fdn@d*oy}xKaRB+PZ`wexUhI(gEhac&o?Kz+ zv+2)aD;EA?PwaIfR`_jqQxo~uk_OCl5vF5U_HBqS{&aETBxC!=%4&1C?+wQ7rL3_2 zst83;wESi;(C_mLdB_13z7&*#S|qqH4Q*_Oad69HYJW)wix7z8EWc-2M$VJMNEU|M zfweT0nP|Q&F5EJO4+?b)CAb`)t_d@h*uxwpQ1_Jy7538;Ha7I`Qb!s-;J?o~RIAcS zxakEmeTt%sqT>BVai42=a;Vjg*yKRqK8oFekpOKKgjuKQ`zX4)8(?3qbM5bMPe8D5 zz_yZem<}s$=5kfyySm%0u147KpTYkk??;C1iRl8o01^xln1Z)G6R^Bak3&?R4pct} zfaH%*8O!dPAMjtQlbUfwydzAbHI(2r^5Q#&?KNN4e9i9I4W==CLwHthn!E)na#i-U z7lHOz`_|-H9K#z_{u6}@Bsryfzha28@Kh!HJTqjEes@%2xnbfbeQJMB(cX{Bb#x~0=g%Qs| z0H91;$Kjp9Y^&fTFOLvDGe;iGtr}fB1B;X`Aui~4Wh~{rNXuAyxi7o&e5hG;#;D zS7+o!M4sHTe|#3ooUdCaZ8|HHk|GW_+%8$l?F<+rCZmWPM}*rxNRz0Pu*GX0Nc+i#6WpoqyKoyI&9-b@I`R#GtAM0sEsu5&VUu*9(>V z5hm{!PV!|DYMYjY-Vs75DU1|IL$aicH7H_a<;R6lB~`kSvYAtj|H2!0lHz`C@A8jr zj_a(o0ynqT9QMD@iNU06wjYko?S#>>b|@$k=U^n@FXopU(0^R=ZZMn3fpt+rvJ%d~gY<=UEW3+|sbl~X;31cU9aP4zmp%o6Q;5SZ%H-~IJxr` z&~Yv(>ZH<-Pk4%l$2Rj*^*H_L?3d;^>q>pfX^jN9VXr%&{eVdq>?(af7RcfaPUugT1;)yb=w2#d!?YJ2Pa`Ni|K+gLaB|(qCnLsS%VcG(ar}_z}9OHFO74 z_)9>H7+2!Fc}z7~DD}L|#dNAywqW}ysQ%qcMVU#)F(>*1 zzx6Zx!*;c3)_dSRbuA_;tGnPGa z%D|@9PmE!&w$5_WN@{-0*V`3WF5>;_?GS9c92BoPF*yGV1Y0NPQ}TN?zqJ>n1=>g# zp4b`w`mp)HBQcJ^Ua|a3mnrthRaVv1FnL_DYl&8b969<$xEjnf2C~)~uVLkjwHfHS z;O{rI!VX6%?AoHZVKgiyNAIW8aDr1GniERPPmcZYKla--YUfu^L>a69p0M|i_^ZHe zPyf9B{yLsSzY;-preaXQKvNA4d)*8d`y-xqgpd!{k~$JfTEf2?yYo?Is^ZjOXZcO2 zl?{K?f{;sOIJ1%MLW! zp_Mly%2!;=N{7j#b4lqK;qVQ~r>oBtUC-@y++jL3V&E^%Hb`M4M7W~@Gi3rG{6<`n z|9+Y-s)dq5^vg09wllQaU?;mZM60Dv_mOR=PPO{(ss zr;y(Ek3n9Z0zPw~@3GH*{fQ+>@-Uhd2ZM#vw(*twz6ReM&k}1@x`?#EoKa=7Z7qJm zR5~vNqI-*OO-Bpm)9zf>lO3EPr*S|4g`?M_GZlHD;8(Gd41-|Gh;Y3`$$j4k1T+WG z@cgjZo~`}Xrt!#Qb>zcLvs7U-XBTB7`0Sjla+i1CXHm?SgsgeoL-}-|wOQ}2h2n41 zWgS26C~2O5>KgM-A`{WyhLY{}t(r^k0YyP-roPB}$EdDb{}AO>k8@KXq;^pPc=P;Z z$5S7nzS6cQPzTwx7i$SxI3=1HYh6*#>P8>ry}kLkGeT# zkA!i$^BjYzovvu2hO#62ig0g--Q>)p*BK>R720hPLay5! zZ#WW7AsBQ$pfZCO``K}PsPT1_-%-NN!^=Y1K+n$dd(8$%C{`qbB3#*#H@wf*Z*?cTJrcO_&mnE?X)|K7gfRc&K3HpS8mdyb)aLI{lXdUpsr8}>IwjUYWctLPX}Mm zpP-in`|G?a>Ap(wZ}$6dI{yD5k^LWY^cx9UG>-pcDPVO*k1aGi{Trx9B-=TatgNmq ztZ6GeC%f0m$C043rJk|zNGRD`_Kq?d|g_)rjqJ^Nn^AWZ+ar@Gr#XF#2#WAUj# z)fcY`P>zB23hO_4OTX#-|5h>i3JN`-au*Zv{vt~&6S=;lsjOS1?^74Jw21Ro`2zeK zBPc0&xN102_xIi+iqgoTM!hhTI$0Sg`52$)4I|;6B&)wkR_F@M(-P zRKOVYd__S_V!|DFP&+|VEUlk_k3z*AnkEM z$z)BC9A4Vz1p)!ii(?757wf`F@VaAI9Lk=91Su>`$FUuJ?rGF%3qp9Ji}~# z>pTwXIHu+A6d`iWdglIb=K>^_7yOL}H4h5jtEc_|_R)+a zg{M04N@^f$rW`2oPPVvyXP^R8pgg#n|CLHE!8&2;&Ovj;7WIjoBlii#$jI{=RVL2z zBl3~LBes%vO25|LiU3CidvnTqM!$Erhdf$$!l04?8XQi5>$!#MpzU77NGAE z4+_xH2c?PZU#kX{q{T<3X4{5$cmm(vx zK2w;)VOk2lMegrO4;Y@If~wq5wW!$NS`4%X6_IGB#@KO>`rtPGZ!BAur7(GoX))0JrD&XVcw@5xkmwd< zLh9k3;m#CBi+&eR%`cfj*S8f z!`FA9A}b>>iKk>~iDNdAbiJ@wqQD_hwSj`7M_8D=kC)dAn=;?W*;z(ER!V(Gbnz)E z;Ch(R(gy2Q$6ZC`joh@@23TGUhOLD!>&VR`_m_nZ0Y5;H|FhmN*eh&01zg$%A^&kG%BRSol}5V|ZU7_LV&f+s2go&+n(}{L z=QwPfJr(&V5P-H0f_5k>esw5`^ZU04TSq~O^%APSnV*syAnFwW*jBa&okc--Ms-oK z%ZU@~fo=IEB*Ur!i$VDrARr~^bdR0wpQXZ6H3#zI4vgo3g8Qb%kWdU~J z6zG75rj)u&*QUSVXn|RHTsG!%lHwTPk<=d^fnzBs64h0niDu+@wsbX0z3Mkh>b-L? zP~XhG2+8Ffi00xzBMikm(5J4KFfo0Hna>4f>!o(^1Pb&W8zoG5=rpfMj5LhyM2S$j zh>~J4zgRH=8$hpIl#Gu^g2Nc}UQHu%M_L9*C(TrL-~pNql*R|r{^7r*eS2o}@g&}X zYRmwv>Fu3AAFt=bukh~q`PF#PYz?|%Y8FcKn!=c+g}a}8PoyweCp@UqO#iA~ZZexLHHmDPYaKnUtq`!9q*qml_ACjx$IVdKKJxQdHq8B{JtfOfqS zkP!}>>d1-jj&lVBoYr~uwq@eZD-noua2dWdNRk+1!fzRNH5M-AVnv3av$YGGZ*d*Y zBxDpSf-&k8Xe9^Dt1YANdSd@bktG-s0L{0P2^Fr+sD@enjTqx6v(unQeAzyR@m z0+U$7Dth_P(u$p?c;~NgrJ>bh-VZa59=TsX#yf#?dv5yK}>qK2){IC6*>-*l?jI#r+I*f&kA?c+qafby) zE6ZfQmJdcbZUPE{K8@zl%$^vCLPgO?2M1z}JznJ6&+Kid6Bh@Vec!b8|I@iZy4=K{ z{LZs4KkN*5m2mkhBE0z0l82Yq!|-~X`Mx!u z6PapHdsJhbM}0srs)^x>z#gxXy$1iYy{;z-Z{jo7v`vS9Y8Y4Ix#B=inuC6Wkd#1( zR0xfp4~BfEif7Q4xv+oe#Szdok^Al!8&g9ijdsa4&@m3+XdgiUAae}>f;3Yn=OKnI zBi0F9mtYU?(g~0eU`B!lh+cSj&3wGB(19+opPBbU7#lYeOTdnE_Jww&tEQ&)*T;3R zNiz})aJE0qp-}%wa;hz>4(JfQuis>5;HVe5fYuXLK8n$O<4gQjP(AhkJ{n`>-#Gvz zJLnI|Ly0>lU?P5;#cTbbYh3RtT;QNbPBVw*fWPY*pbw|qw~x^|91@;JuSVQ>|e7Xh}9s*OT{4vBw1=rU0e3{j`$ znC!znL$<#5%OqsAJ`sAf8;_(quz%zZ(3#HP>T=TkC^v?WSkkb-{@O9b6Jz+9z?Ie)G_=X5x^vY_u1!ah4*1ScvU@FTpF+%T}i zjv0`!StG~MISI4tYR{aCxmWdR#?+81yE0s@igFJ|2m!+)e?PeE2wOR}?DaAk67_b9 z(4Y*y$aGGcVDkLHw}pL|t^oqQHm@`Hy_ao#)!LVg3=Z(m88^t{3#t)+gW1q^(N!OVbWpTDkq5sM@1{XF8d_wo%H3@ic_Oea+A|5T?jZ2Y<3=SR0 z22EeCP=7nr^QV~a-yl}G!?x#Tmc4GLg%;^WzObQZTXK|`(QpSA|elW?-Npd z`dc#!;B2i0=VwlROr0yCw%LfI{gSy1a6NL8)E7C%eujVf?S36x)ab%m8;bWkAsO_e z9cO@FkRwkfJ$(33WZ*IWV7SLN=YYI#uqk-ForRaK?#qljw*_nk%dNY4FI8DHr%SJ5 zfij`J5W1jiR~WKIVD@LU#=feF9iw5P(>upLckp*|D*qx+v6DZtZzIvBd{& zgz1gFzow(1Q5IR(SM+*D(_Y{Yf)evIc9@*ch`#dXyNBeZc!})?YI(V20Cayk+rAVM z6CPsA2d!P@hW%9|@8tQP;()yDQWF@@g5D1X9J|ai!Qvi?6}yuI!B*n9|YuHo&c;wz~c8ql>>eByz9F((5N ziynFi&3m30@hU5e!ou6rqXlLCzr<7~0AkB5q)GW@#$X2ADhG+CT7v?Js5-tLZC9dG zct-P!I@KM~$3G_xaQ;w#tV<*?LJA7l#ngGOsMG^PG&c!MVKIR)$~y}hI`PUTsWUtWG@YR@hEY~oJB#}DW0v|P7f<9Wg%|Xjm!Lc> zGDXJl`%$QWKTb~KJ9sO-x=S$fZJ|M+BT|Nkxk#tnlZ`dSM?_H2*D12N*K0S2K(Ln@ z+9tCf*yZ-10nnI#V3-FNmhS2vrGGg+uu*N-y*AG(WNgelbu0HIF%RCx=?(S3rEqw@ z_x$kCNys*J%nJb~q#_LZ^9O-b_{O~41|;j>kHhP2BEJYXn8$_rCX1ejhHD(UQz@1a zGo{aWyh0-DMG=esGn0VZH55BDJ7uGxcS}o3I!jBX8<#eIUIP4i-t!;`40a8U*q~~7 zpFVLvdFt_w!WI7?flkH)i~2jBWT?aL{KMR@J??|39^k7WBX;d|L{g62;MG?(>E#7| z@9;q*FXzIT%}WS3O{yqC|JuJ(ex}p-j$h-l!eFWabmYbipU{c5@#PtR(sld0t*^(M z#+Ut5N+C$c&c25aT8Gtzv>ec=5omJOfT!*fxJ;DPQXKI?zmL z9z+QU?mr(f*TYx!LY`KN95|nLyVRE&55CY(*mY^zyit46)NvH_`fKI4MqY+*J}If) zw(Hy&LS>MDEtDDV2F&(!75?0-qr3yM7XBa?Uf%AGVa!m^1k|Taf&gyk2yE5#`MQ+e zH+xa3<#bQb&6j#T{3C5lL8vwxQq%K6`oLR!MC0R+MK3vI%!x`^4aoV_jI!i_4nC-z zvOO)OytS%qHpuSnin*m3nMHTua4^lmP$zKz6f;J7s0e}1p74HQ0}wGlql9Vga@W=O z+8dHF9{LeJmOY>L+1SA0h^gr~SJsQeEFKf62xWE$($%EJCra|fr#cK-KQ4;IK_L&+ z+&Nhr?d4jZHZ!-y(HMKnX)uj3#*X#a#At92`ifM;piNn3dtS)%SM37i=MShiWBEoD zg@6yeS8sy%0qunCF1_w%@%mr$yV&Ag`;R-XOEu($*7v6DjlRj31CCVL@IQu91wlXi z3B<_VY$?C~S#;lVmrp&Wy@{lg1H#O;@9B2MB{5XqwPEJ_@>^dL-E>MO-Q~sHL}#M$kldy0E&z>?fJB& z26{@5>XdIS(mBs8ffmXqrXPeZL6aL`EX@DW*V$3V9ucbW^&vU$H}jBhSH6({c>QC7{oboX8= z+QDVGZBw4tO@^pl^8RY*EdED=>Nm?Pm_zNKKcbYqWYa7CO5c7d$1h(S+3B_V?H>& zoO%D!<$xUph%jDuloEHghS&IljNgsz$;U^YLZ6K6<8)$-4Wel{`aGXP!=SPwq%rd< z1(z$A;o@eM((R$XPg~XtF(J_7;yNE+T-@tlo_@h4mFt@lM{DG*AS;GQ@qLE%J$l%>z*+ zesXX?EpgXrXP!rl*^M!g;QD!a+Q#ij|EUpaXb{e&j4+-*pvpt)bD!@AwY@HPVf_+*;(qlb#0eMKkD*`ZEnXeOug zNy)O08PtUS zz1T`e?vcVl4g(0@e7ZO2@n=b)uN6|LnNtmyq!e0aMI2Af*}N*f1cUU19VAjP+@mb@ zuA-?*G-4cUSHlCi3!%vvJ2=O@diVFdHw?yk1p0pgUKc~;gU|-j%)u-JiUKmw$RS?( zk^o2^o*f4icoBT0uM-mbH8G6Z#}{8i8WL2igk*=@I)TU3Z_aPlD}OJoH2!&TC#gE6 zI{VG6n80L;?VjX&v9iIZfO=Q?7lNVIb?`XVHr_RIsV0W_(y7~xH(tN6tI2in#IsB> z)?T@A^ImPc1ci%Wa$5}qy4iZBR9GP~1jBh2-`jz4=~cs9R}C;Xd=tCZ_MVto5w_ls zo2Ztz>b#r~#8C4Ak+1~5Bxy~27qhH$U0$?Ft$SF=o7D)DzBsfm_8^S?EOaEFyjW1|f`6hnoCLl?n>Y23IVplD8E$S^RCf zX5O4X0%Qb|llad?b0D+24UNDjX9GRCU2%;c{Lcal!h%)06Dcp;t0Z&;@||$2?o}2x z{|`KNU%beNQ=gXKSH@7g&$@!yiSdX1K-)n0rtjqRh7fxuzmNyy;%cV_1WLzsZv|c9 zx{_B2NY2%(A^GX48y_?v@m1!k72wE;c@3NdH8=UqzEuMA%0-~cl$ zU*+zAx{t?{Sza97jp(Yr-rHcnUSJ_Oh9gnRucQuTsIRLD5^YtL#)n0hB!!I|Ej)iF znb{L}dKS%w3Sf|1@P#bWGJ@yV_$(v7#ofD7FLETD_ZGcvcwS&=t!?-2X914I0E^s5 zcDP@+44E=`ruko1bW6Rl$K@j~iH`apx zcW?nsJL;#`jS{K8jn8&Jggp~q#JN8H;Ib^Fyr9TG_@Cyz*Eax0LTY+;w zE;hw+q3uzouud4WnLyERTAfL2F2dHVHTPin|0(6nWGYi7A&QI@BJ(Ud z#?wg(k%S_dr;H`Zm_p{6bRt8>EhJ+qk}(;#A+v3s-SzBqey?+f*L~f4Uw6OS|7h>; z_j$h0de(Zc&)Uz*eq~PvxFT6;A-fO!@7v%u;+HMeoG+jYEi^ISqJl?H6~p`Wr(a3x#(2m20nxt}!K(|5>TcSCl&OIW9m zdok{|Z5-@R5|bg@JOw|xQclrl*b95VadHz!xj4XVG-Hc9RO$)4qhW*LvWuQ#K4bK{ zBlaOk!aFCE^g@-p?AQEYcW{&yWj1#OACSom#k4AJ{HOH@Zm@aEo(jrtn!h-qgE z&zzz1Q;!GE!lT2x+Y_x++K>J?f_YYqq^kWSHPuOU)JgWTn>En%u#UgywjNJ;6a(wK zj=MQ42WRN(Ev}mQ` zpQR;)#k7cjZQ*X{JIWD+bM9s&JYN;2+cW7rZDOnr^GD)VdTeZNjjURGfMw=q)jN_X?_2u+%obDF}=a@Pdq^$pJl zY%qg`A%}BO^Vg@Tzx08VtwXcL{Yr!Qo6}Oz2-RvzVy==pdc8(-F4Kw3(8JZNk6Qop zJl}&% zcw=lAb6hDgIyX2-)WH(082-VHFU&2N@#a})X-}cp9YFgQwp+r9FF_S#|tZLqjJlfM#uH}J@`sv)h?hY(X&PC@7t1r<;k>=`zpmtyBsS@q0zeeBV;1`@Au?-A(j@{Ih)nn47}zi8-5CjStv9wbQ2PA54%k4pc@ON5@Q#Ilhia7WW!Ns zqH%t+E-kgmWr+upgeIwd^Vr)NFuhj;2Y561UO4R=)aARJTu(X_kcYiQ{H=8WWe$gy zUccLX^ZFUVFiZ~(nJA+GPt^lU^)5=51a}~FHSc5G?A;E_wy>^)ypUT6PGH8~DX};L zB3RDt>(L--o(sv*(%~?d$Zu=#Ke_jES8@Qp=LA{7V>ksXgdsAtVPFcZMg%H|6$Vq) zTJjVK6&g!taA9e>vaH${_4p>Kt5S7e$py1w2ZZN57ABbMjLwU}+3Ez@17`rLCahT^ z0u)LT&*0yyGtM6(FR>WDo+#>6B8Z?0jwOBr!39M664AN~?P;`K-_TU9TA4S70NC$J z9iLusy0Chfp(5Xg7D^gOG55p+au!)Q`;yGY-2WFmbBI@P%QlV zE$;vLF!jFEzO`VMQ2!y^=cW*yjZjN%X<<#n?U{8y zAxPNqB7IC8eH@L->tqksXQTFdD7#jd!37wuqxF6Q0Ui=Z&37a;q@2{9uEUR*0wVK} zH7~waX;E-5^8GuFG0gr@ZTEE8+C8yP))Ve0Qa?dy{nWI~ zFS|F~7qZ5o0_>zpV zK7Cr0);Q$GHXOq>L)KJY8}~sfEwHH$Yo%mTtvcR>*w9*1P=^mKueUHzZE@C2h0Cg zhU62Q1)@SaRIA8EMT}Fw+ck=69aviTBr_Feo4V5n`9R{@R+Pzb<;2^~_PK>X&pq8& z-YoxYMeis+^1`u{fNE!o4^x*C&F(`JFgkyoNoHYicjB_*l}Zdfz>`1FO+1(El)J8P7_XIIYgjvckcS$=psbTgYZkK`ci zv&FoB_Swpx6bDPkQUs}3jDCu55!fsnhl1uJ%I0H-Ew;lZ@KRrJ6$x__`y0SR$;w%3 z-LGbr-k9#YYu(c!XO*KlX zXmWU2K4E#t^)0*f81$R1LmojEwQ^yr0bJR@@6tpV5* zrPOWxhcfpS>1j?qGIG0rA(ns64P5^dls>F3%%Vi>{2gSY7d`6FSegoSOTrqoeqLtc zt8X+XX;HIXXj@oABmy%W|G(`fV>fK21j*iPmsfkfYY-{ znJ>Vbk;%$q9&|R)Xot>sA1l3;+BZCb{L&_NnD6~Q6-+_m;eIYwkBW%y3ZCX_IhRNcsKPNk-yuBcHu??y+fu}mBrYiLrTy`BT%v1(Ri=mD+o=MlQ)H(hRD zxH!cg8my+%yZY$f;@A4k_%#`3Ir~A;XN=dPBkd`tFM7A=pUW!_lM$30D2kJ%4bK=T zFng9W+OHbxT7?h}7kqC1XIYz=Ek12@q(L=3yK?X9&v{w)`5NlT?rZAz$T5uJD%~4S zcgiafl@FbH_Szh8aaD83m%*@oPWzy!4Z5rf4Eq5;%z*l&yZemWUjVk_Y6Fdl+drOP zdfHOb&~WSWbdM$-$O{zkn2z2@@r$qdE(6!Cg<;F)*ej??)ab*Ht&#nKS0ZF>1;n&~ioWDJ$I%1RtQL1#ua=^> zQ1{H;>v_H1D=XtA%yC)Dt^{J0X?y3XD^Jkca!>m0WO--+&$(rRgJ5#ITQw}|oX z0CqBK9OZaxyKho%9AF8=5RpyBzRW}eMd>sI&om@(K_-NwR9NNXK#L#z z*W$DG?O#5jr0Jt|aV3-IyFD{h5I?$*gu*moie|SwqLNaP&w_05jcEI{tzL6;)H4Wh zDu79&d5wo`x5H-AEe^&bhn90&{Rb*qKpol<6)_>qT|!L=AibadZIyi0bRG5DKzv2) z(+9o!zFl=;hhf*$ZW<9~$x_vmAZ3`g$QiCPhLa_crA+I|QTg&+o+oZC9+7U*WFwL@ zxOGhkLRsLld7uj#5u5>Q)!;zCIbtH@p#LVSyIC}HHfFp(``e-DU zAnF2>$}oq@P+X$Z(~mc7e!Ssf!FpAtiM4K!H8Kp}&Ur@*lt1AF`|@H2mk8D+x299!M z1fB9*tTEP?LSSj9i(adp8Yd);#Q`{Sb;d}E@ST8BnJ^JYm03Y*`t5B(VpB&8isHNH zbdemktKmxoqciY(iB?iJ7#6KRR_0@P*&G(}JLr|#Vx8k>Xrh1E5Cl1;3x+5!brDD~ zjOy6TiR05P8zd2fdqwOD`&u!v7}}E=%hVGAb^gH`Yv`w0)SZsRO=n1iBg+XcBZ@ev z^X5UY&$pB(ln&w5K5$84u=DWPG<;X(1u_L!f zvVf|9V4KPirGn1;DuM^t6efCrOF_|oi?>O>3p_pv08pg;#ODUp$M1_=rJh(!fSouH zuf%wdmLPfI7N-7}lGlJlR?zc3Say2~Hr7~3F)V_D5UIPn-|A+PW4f@xdeMF5gd(4N zE%Nr~o&pMi2{LI$kL*(z5ac}l-X9kI6x9BZk_DrB1jg)zjM{OiRQ2ro4Fon-dn*3M zmHU6YRwdMVChNQ`!pzSF@gWyno(pUrd6RCqZf+3G26Pbk%+^76h#RDq%C#BBB-PYTBQZ=Vf1LF&u_WH!jiM->_|@s2KnKl z5n_XU_tjW_W4P6Jl^(`1^`7Y9Z0hO#Se*lF>|-P;SN~`5Y5H%$r#Jgw;A0T2^}QB) zQ=#n^u(I>tFb~m?i&eNHgeZl!b497H+?b+y_`8Usi{t~%xA&h-mc5?L8D{p!uCi7= zlhr(dDj#U>s1WZn-dbMwYTEmdW3Mid^{4Vow@K!?z2k7h$ykrr=MJHcMy7~$Cd!7T zNL0bRDg}t8Uxp1olwg3gwY1%f~SU>B8`7vH2;7ie%oyum_jd zkDdOo!!z^goOCl>mQ!A}+CGfZrlHk&q&eE_#CEFg04s3@@7K8^ILO`2+8Fn-x(3$O z4$j>;>YWY29lTlZ5_#_uS+8S;QDQ9URaLOBy;Eb>!r?l zV@`7OI>)dBKGQ}XJXdc*%vac0W?>+~`Vlq#vL3sPL>Lrkh_|lXkG8<+Vh|xHO7nPJ z7o-c71w{J$%dD1Pga$_jO9-BevN5uXE3r?t{(+F?vAX`em64dorxISL=dQwCgOw9G z3qe~+K?!YzFG8fRsxQ(5VdOfGs=>f9U(;cL4_DKD&BXtaRcBdjji<|NtIP!=@5vQ~ zd27QjstyqcQ9yBUL=MpayejdCa&0;?GvW)o8YA5j$G%j`rv1u?&GY)%ID6oz#) zH3}qzx1!n=>TFu#=)pk}A@o5lRhLf#kYtqfGX_2g!)QXtoQK)Wg^Y|)!45@iRPlmt zpY8`S%H{{h0WX#E|G!$y@?IbA`GDA0$uRsLkF&8v$-_`~oXNtW60=y6P^`!>z3J(w za(GqGQ}TmHMS~S9U#Q&KdRmdtC(-e7^aFjcM?mDf`ld;ww1@X+^YP@t(Qy0fPxC79 z5{Pjb(3{Ap_UYZ6Zf(sGDIe!c&JC=$krg}|s>s#r{pkyx!pf8&1*dPlUNw2esrC)U zy1^HlRtEgLshiAg@9?)n8&V>2+_OGe(NO0I4C+?jw}=~7&FPRYoi(d12@KNA^wfm= zyyMRi4KpLR%sOJiG=Q7$DAp0*&1%(Hq8wW9G^A0$4xIJQ{^Ev}%(-R|2)Wj)+D|10 zGJBWLq4DIn->%rS%KS6@O387S&J~KHNsEwaMgIBq=337+V?qOt2RcSxt6vOD7R#eN zy;4ExwGAJRKI&`T_+^}3-n#GOqs!aMITVBmSKgMmGZ=~Blk=Eex2jwv%Vb?fe}Czl zU3(a?L>^^MN0i-XPwH1{DVybZ8-a^#-4B_d@9MYLQMUW6#kchhs-D@%kA*a(q{#2I zbbY*8xbs3Z-XzNnZ+wNn|M#2~x(w%P?h!k+8)c$!m#%ISw>2*(ksY(^uN94|Dejzq zAF|73DMqnR9D6>Soghcr$v_->RL2qZ_Y*oCzMC5tcSwKLNn?HeUeB}zH`8`pjFWDV ze89-_8AMb^{}97?-}+sy) zeU4e}9gK#Ip+-r|n5;92w9A7sx>LSn*6ily08Z05xJlYO5;}-kb;eg6di_b-yAUR# z9U|Y@ADp!N#b|Go-4|m~WApv-y8=9?qji}Cf^GS+tJE^b9|0_SAj2Q?k-KGzqxI5`Jn>Pl+?e1f=ugh*l8$h3mEY{t@!I|(vty2tUgh&5&yx+W zp?{lBpQruXwKZvd+L4hmPx#k9H^&HH_FBp*(i4B8?la#Vr0TOYVJ8IX*w?T{``({Ep1-VP7n|;XrIT-Rs9g zvzy&^Z_lUEi-CEah?}{`EhUn}>y6L$p8COyr>|-Mrk)~4UVQc`. + + +High-Powered Computing Machines +=============================== + +libEnsemble's worker processes can scale collections of computations across +small and large resource sets. When running at massive resource scales +on High-Powered Computing (HPC) machines, libEnsemble workers can +be distributed much more flexibly. + +libEnsemble's architecture lends it best to two general modes of worker +distributions across allocated compute nodes. The first mode we refer +to as *centralized* mode, where the libEnsemble manager and worker processes +all co-exist on a single node, but through the libEnsemble job controller or a +job-launch command can execute calculations on the other allocated nodes. + +.. image:: ../images/centralized_Bb.png + :alt: centralized + + + +Alternatively, in *distributed* mode, each worker process runs independently of +other workers directly on one or more allocated nodes. + +.. image:: ../images/distributed_Bb.png + :alt: distributed + + +.. note:: + + Certain machines (like Theta and Summit) do not support distributed mode + + +.. toctree:: + :maxdepth: 2 + :titlesonly: + + bebop + theta diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index c87fd2280..91dfd3e58 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -2,24 +2,15 @@ Theta ===== -Theta_ is a 11.69 petaflops system based on the second-generation Intel Xeon Phi -processor, available within ALCF_ at Argonne National Laboratory. +Theta_ is a 11.69 petaflops Cray XC40 system based on the second-generation Intel +Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. +On Theta, libEnsemble users will mostly interact with login, Machine Oriented +Mini-server (MOM) and compute nodes. MOM nodes execute user batch-scripts to run +on the compute nodes. -Before Getting Started ----------------------- - -An Argonne ALCF account is required to access Theta. Interested users will need -to apply for and be granted an account before continuing. To submit jobs to Theta, -users must charge their jobs to a project (specified while requesting an -account). - -Theta contains numerous node-types, but libEnsemble users will mostly interact -with login, Machine Oriented Mini-server (MOM) and compute nodes. MOM nodes execute -user batch-scripts to run on the compute nodes. - -libEnsemble functions need to incorporate the libEnsemble -:ref:`job controller` to perform calculations on Theta. +It is recommended that libEnsemble functions incorporate the libEnsemble +:ref:`job controller` for performing calculations on Theta. Configuring Python From 6bb9c33c6219106ae81fa9d62cd3b412597960af Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 6 Nov 2019 12:05:42 -0600 Subject: [PATCH 431/644] Adds job script example page, adjusts formatting and diagram size, clarifies information throughout --- docs/examples/gen_funcs.rst | 4 +- docs/platforms/bebop.rst | 4 +- docs/platforms/example_scripts.rst | 33 +++++++++++++++ docs/platforms/platforms_index.rst | 44 +++++++++++-------- docs/platforms/theta.rst | 68 +++++++++++++++++------------- 5 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 docs/platforms/example_scripts.rst diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 5c3fd2495..660b1b583 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -7,13 +7,13 @@ Below are example generation functions available in libEnsemble. See the API for generation functions :ref:`here` sampling ----------------- +-------- .. automodule:: sampling :members: :undoc-members: APOSMM ------------- +------ .. automodule:: aposmm :members: :undoc-members: diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 6c419ba87..8922c30a0 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -2,8 +2,8 @@ Bebop ===== -Bebop_ is a Cray CS400 cluster available within LCRC at Argonne National Laboratory, -featuring both Intel Broadwell and Knights Landing compute nodes. +Bebop_ is a Cray CS400 cluster available within LCRC at Argonne National +Laboratory, featuring both Intel Broadwell and Knights Landing compute nodes. Configuring Python diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst new file mode 100644 index 000000000..9300b8abc --- /dev/null +++ b/docs/platforms/example_scripts.rst @@ -0,0 +1,33 @@ +Example Job Submission Scripts +============================== + +Below are some example job-submission scripts used to configure and launch libEnsemble +on a variety of high-powered systems. See :doc:`here` for more +information about the respective systems and configuration. + +Bebop - Central Mode +-------------------- + +.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh + :language: bash + + +Bebop - Distributed Mode +------------------------ + +.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm.sh + :language: bash + + +Blues +----- + +.. literalinclude:: ../../examples/job_submission_scripts/blues_script.pbs + :language: bash + + +Theta - Central Mode with Balsam +-------------------------------- + +.. literalinclude:: ../../examples/job_submission_scripts/theta_submit_balsam.sh + :language: bash diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 28861c59e..22a8e5380 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,46 +1,52 @@ libEnsemble has been largely developed, supported, and tested on Linux -distributions and macOS. Although libEnsemble and most user functions are -cross-platform compatible, there are platform-specific differences -for installing and configuring libEnsemble. +distributions and macOS, from single-nodes to hundreds. Although libEnsemble +and most user functions are cross-platform compatible, there are platform-specific +differences for installing and configuring libEnsemble. -Personal Machines -================= +Local +===== Users interested in installing and running libEnsemble on their personal machines are encouraged to read the Quickstart guide :doc:`here<../quickstart>`. -High-Powered Computing Machines -=============================== +High-Powered Systems +==================== -libEnsemble's worker processes can scale collections of computations across -small and large resource sets. When running at massive resource scales -on High-Powered Computing (HPC) machines, libEnsemble workers can -be distributed much more flexibly. - -libEnsemble's architecture lends it best to two general modes of worker +libEnsemble's flexible architecture lends it best to two general modes of worker distributions across allocated compute nodes. The first mode we refer to as *centralized* mode, where the libEnsemble manager and worker processes -all co-exist on a single node, but through the libEnsemble job controller or a -job-launch command can execute calculations on the other allocated nodes. +are grouped on one or more nodes, but through the libEnsemble job-controller or a +job-launch command can execute calculations on the other allocated nodes: .. image:: ../images/centralized_Bb.png :alt: centralized + :scale: 75 + :align: center Alternatively, in *distributed* mode, each worker process runs independently of -other workers directly on one or more allocated nodes. +other workers directly on one or more allocated nodes: .. image:: ../images/distributed_Bb.png :alt: distributed + :scale: 75 + :align: center .. note:: - Certain machines (like Theta and Summit) do not support distributed mode + Certain machines (like Theta and Summit) that do not support child-process + launches also do not support libEnsemble in distributed mode. + +Due to this factor, Theta and Summit approach centralized mode differently. +On these machines, libEnsemble is run centralized on either a compute-node with +the support of Balsam_ or on a frontend server called a MOM +(Machine-Oriented Mini-server) node. +Read more about configuring and launching libEnsemble on some HPC machines: .. toctree:: :maxdepth: 2 @@ -48,3 +54,7 @@ other workers directly on one or more allocated nodes. bebop theta + example_scripts + + +.. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 91dfd3e58..f884d9451 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -5,12 +5,8 @@ Theta Theta_ is a 11.69 petaflops Cray XC40 system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. -On Theta, libEnsemble users will mostly interact with login, Machine Oriented -Mini-server (MOM) and compute nodes. MOM nodes execute user batch-scripts to run -on the compute nodes. - -It is recommended that libEnsemble functions incorporate the libEnsemble -:ref:`job controller` for performing calculations on Theta. +Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), +and compute nodes. MOM nodes execute user batch-scripts to run on the compute nodes. Configuring Python @@ -47,22 +43,32 @@ following block: (my_env) user@thetalogin6:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py (my_env) user@thetalogin6:~$ pip install libensemble +.. note:: + If you encounter pip errors, run ``python -m pip install --upgrade pip`` first + -Balsam -^^^^^^ +Balsam (Optional) +^^^^^^^^^^^^^^^^^ Balsam_ is an ALCF Python utility for coordinating and executing workflows of -computations on systems like Theta. Balsam can stage in tasks from many sources, -including libEnsemble's job controller, and submit these tasks dynamically to the -compute nodes. If running with MPI, Balsam is necessary to launch libEnsemble on -Theta. +computations on systems like Theta. Balsam can stage in tasks to a database hosted +on a MOM node and submit these tasks dynamically to the compute nodes. libEnsemble +can also be submitted to Balsam for centralized execution on a compute-node. At +this point, libEnsemble can then submit tasks to Balsam through libEnsemble's +Balsam job-controller for execution on additional allocated nodes: + +.. image:: ../images/centralized_Balsam_ThS.png + :alt: central_Balsam + :scale: 75 + :align: center + Load the Balsam module with:: $ module load balsam/0.3.5.1 -Balsam stages tasks in and out from a database. Initialize a new database similarly -to thefollowing: + +Initialize a new database similarly to the following (from the Balsam docs): .. code-block:: bash @@ -73,21 +79,26 @@ to thefollowing: $ balsam submit-launch -A [project] -q default -t 5 -n 1 --job-mode=mpi $ watch balsam ls # follow status in realtime from command-line -See **Additional Information** for the Balsam docs. + +See **Additional Information** for Balsam's documentation. Job Submission -------------- -Theta uses Cobalt_ for job submission and management. For libEnsemble, the most -important command is ``qsub``, for submitting batch scripts from the login nodes. +Theta uses Cobalt_ for job management and submission. For libEnsemble, the most +important command is ``qsub``, for submitting batch scripts from the login nodes +to execute on the MOM nodes. -On Theta, libEnsemble is commonly configured to run in one of two ways: +On Theta, libEnsemble's communications are commonly configured to run in one of two ways: - 1. Multiprocessing mode, with libEnsemble's MPI job controller taking - responsibility for direct submissions of jobs to compute nodes. + 1. **Multiprocessing mode**, where libEnsemble's MPI job-controller takes + responsibility for direct submissions of jobs to compute nodes. In this mode, + libEnsemble itself, including all manager and worker processes, runs on the + MOM nodes. - 2. MPI mode, with libEnsemble's Balsam job controller interfacing with a Balsam - backend for dynamic job submission. + 2. **MPI mode**, with libEnsemble's Balsam job-controller interfacing with the + previously-mentioned Balsam backend for dynamic task submission. In this mode + libEnsemble has been submitted to Balsam and tasked to a compute-node. Theta features one default production queue, ``default``, and two debug queues, ``debug-cache-quad`` and ``debug-flat-quad``. @@ -108,8 +119,9 @@ This will place the user on a MOM node. If running in multiprocessing mode, laun jobs to the compute nodes is as simple as ``python calling_script.py`` .. note:: - You will need to re-activate your conda virtual environment and reload your - modules! Configuring this routine to occur automatically is recommended. + You will need to re-activate your conda virtual environment, re-activate your + Balsam database (if using Balsam), and reload your modules! Configuring this + routine to occur automatically is recommended. Batch Runs ^^^^^^^^^^ @@ -247,12 +259,8 @@ Debugging Strategies View the status of your submitted jobs with ``qstat -fu [user]``. -It's not recommended to debug compute-intensive tasks on the login or MOM nodes, -and if running in MPI mode, may be impossible. Allocate nodes on the debug queues -for the best results. - -Each of the two debug queues has sixteen nodes apiece. A user can use up to -eight nodes at a time for a maximum of one hour. Allocate nodes on the debug +Theta features two debug queues with sixteen nodes apiece. Each user can allocate +up to eight nodes at once for a maximum of one hour. Allocate nodes on a debug queue interactively:: $ qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I From 692eda8e6f656a1914441c6b4f4499065ef03a14 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 6 Nov 2019 14:41:59 -0600 Subject: [PATCH 432/644] adjustments --- docs/platforms/bebop.rst | 2 +- docs/platforms/platforms_index.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 8922c30a0..c2ae554f8 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -113,7 +113,7 @@ Debugging Strategies -------------------- View the status of your submitted jobs with ``squeue`` and cancel jobs with -``scancel [jobID]``. +``scancel [Job ID]``. Additional Information diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 22a8e5380..9092a9110 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,8 +1,8 @@ libEnsemble has been largely developed, supported, and tested on Linux -distributions and macOS, from single-nodes to hundreds. Although libEnsemble -and most user functions are cross-platform compatible, there are platform-specific -differences for installing and configuring libEnsemble. +distributions and macOS, from laptops to hundreds of compute-nodes. Although +libEnsemble and most user functions are cross-platform compatible, there are +platform-specific differences for installing and configuring libEnsemble. Local ===== From 051ebd496109380bfede49ffb5ca6ccd8da13802 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 6 Nov 2019 15:31:07 -0600 Subject: [PATCH 433/644] user_specs changes --- docs/tutorials/local_sine_tutorial.rst | 34 +++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index ff9a7b8ce..186e2b2bc 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -20,8 +20,8 @@ launching and controlling user-applications with a :ref:`job controller`, the History array. @@ -30,7 +30,8 @@ parameters are stored in :ref:`H`, the History array. Getting started --------------- -libEnsemble and it's functions are written entirely in Python_. Let's make sure Python 3 is installed. +libEnsemble and it's functions are written entirely in Python_. Let's make sure +Python 3 is installed. Note: If you have a Python version-specific virtual environment set up (e.g. Conda), then ``python`` and ``pip`` will work in place of ``python3`` and ``pip3``. @@ -75,9 +76,11 @@ An available libEnsemble worker will call this generator function with the follo information. In our case this dictionary contains mechanisms called random streams for generating random numbers. -* :ref:`gen_specs`: Dictionary with ``gen_f`` - specifications like simulation IDs, inputs and outputs, data-types, and other - fields. +* :ref:`gen_specs`: Dictionary with user-defined and + operational parameters for the ``gen_f``. The user places function-specific + parameters like boundaries and batch-sizes within the nested ``user`` dictionary, + while parameters that libEnsemble depends on to operate the ``gen_f`` are placed + outside ``user``. Later on, we'll populate ``gen_specs`` and ``persis_info`` in our calling script. @@ -92,13 +95,16 @@ For now, create a new Python file named ``generator.py``. Write the following: def gen_random_sample(H, persis_info, gen_specs, _): # underscore parameter for internal/testing arguments + # Pull out user parameters to perform calculations + user_specs = gen_specs['user'] + # Get lower and upper bounds from gen_specs - lower = gen_specs['lower'] - upper = gen_specs['upper'] + lower = user_specs['lower'] + upper = user_specs['upper'] # Determine how many values to generate num = len(lower) - batch_size = gen_specs['gen_batch_size'] + batch_size = user_specs['gen_batch_size'] # Create array of 'batch_size' zeros out = np.zeros(batch_size, dtype=gen_specs['out']) @@ -185,9 +191,13 @@ inputs and outputs from those functions to expect. gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size) - 'lower': np.array([-3]), # lower boundary for random sampling - 'upper': np.array([3]), # upper boundary for random sampling - 'gen_batch_size': 5} # number of x's gen_f generates per call + 'user': { + 'lower': np.array([-3]), # lower boundary for random sampling + 'upper': np.array([3]), # upper boundary for random sampling + 'gen_batch_size': 5 # number of x's gen_f generates per call + } + } + sim_specs = {'sim_f': sim_find_sine, # Our simulator function 'in': ['x'], # Input field names. 'x' from gen_f output From 63b60ea1992384b8e1a4df059d720662226ceb5b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 6 Nov 2019 15:35:26 -0600 Subject: [PATCH 434/644] Reverting to one toctree --- docs/index.rst | 132 ++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ffeed0f44..a6fcab64a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,73 +36,71 @@ libEnsemble is a library for managing ensemble-like collections of computations. * Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. -.. only:: (latex or latexpdf) - - .. toctree:: - :maxdepth: 2 - - Quickstart - user_guide - tutorials/local_sine_tutorial - platforms/bebop - platforms/theta - contributing - FAQ - libE_module - data_structures/data_structures - user_funcs - job_controller/jc_index - logging - dev_guide/release_management/release_index.rst - dev_guide/dev_API/developer_API.rst - release_notes - - -.. only:: html - - .. toctree:: - :maxdepth: 2 - :caption: Getting Started: - - Quickstart - contributing - release_notes - FAQ - - - .. toctree:: - :maxdepth: 1 - :caption: Tutorials: - - tutorials/local_sine_tutorial - - - .. toctree:: - :maxdepth: 1 - :caption: Running on: - - platforms/bebop - platforms/theta - - - .. toctree:: - :maxdepth: 2 - :caption: User Guide: - - user_guide - libE_module - data_structures/data_structures - user_funcs - job_controller/jc_index - logging - - - .. toctree:: - :maxdepth: 2 - :caption: Developer Guide: - - dev_guide/release_management/release_index.rst - dev_guide/dev_API/developer_API.rst +.. .. only:: (latex or latexpdf) +.. +.. .. toctree:: +.. :maxdepth: 2 +.. +.. Quickstart +.. user_guide +.. tutorials/local_sine_tutorial +.. platforms/bebop +.. platforms/theta +.. contributing +.. FAQ +.. libE_module +.. data_structures/data_structures +.. user_funcs +.. job_controller/jc_index +.. logging +.. dev_guide/release_management/release_index.rst +.. dev_guide/dev_API/developer_API.rst +.. release_notes + + +.. toctree:: + :maxdepth: 2 + :caption: Getting Started: + + Quickstart + contributing + release_notes + FAQ + + +.. toctree:: + :maxdepth: 1 + :caption: Tutorials: + + tutorials/local_sine_tutorial + + +.. toctree:: + :maxdepth: 1 + :caption: Running on: + + platforms/bebop + platforms/theta + + +.. toctree:: + :maxdepth: 2 + :caption: User Guide: + + user_guide + libE_module + data_structures/data_structures + user_funcs + job_controller/jc_index + logging + + +.. toctree:: + :maxdepth: 2 + :caption: Developer Guide: + + dev_guide/release_management/release_index.rst + dev_guide/dev_API/developer_API.rst From e71dd824e8c43792620cb1e41b72ffd723a16b4e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 06:06:48 -0600 Subject: [PATCH 435/644] Splitting html and latex toctrees --- docs/conf.py | 9 ++++++--- docs/index.rst | 22 ---------------------- docs/latex_index.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 25 deletions(-) create mode 100644 docs/latex_index.rst diff --git a/docs/conf.py b/docs/conf.py index 7ef1e3929..00aaf27d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,6 +87,9 @@ def __getattr__(cls, name): # The master toctree document. master_doc = 'index' +# The latex toctree document. +latex_doc = 'latex_index' + # General information about the project. project = 'libEnsemble' copyright = '2019, Jeffrey Larson' @@ -205,7 +208,7 @@ def __getattr__(cls, name): # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'libEnsemble.tex', 'libEnsemble User Manual', + (latex_doc, 'libEnsemble.tex', 'libEnsemble User Manual', 'Stephen Hudson, Jeffrey Larson, Stefan M. Wild, \\\\ \\hfill David Bindel, John-Luke Navarro', 'manual'), ] @@ -216,7 +219,7 @@ def __getattr__(cls, name): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'libEnsemble', 'libEnsemble User Manual', + (latex_doc, 'libEnsemble', 'libEnsemble User Manual', [author], 1) ] @@ -227,7 +230,7 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'libEnsemble', 'libEnsemble User Manual', + (latex_doc, 'libEnsemble', 'libEnsemble User Manual', author, 'libEnsemble', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/index.rst b/docs/index.rst index a6fcab64a..9d5246845 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,28 +36,6 @@ libEnsemble is a library for managing ensemble-like collections of computations. * Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. -.. .. only:: (latex or latexpdf) -.. -.. .. toctree:: -.. :maxdepth: 2 -.. -.. Quickstart -.. user_guide -.. tutorials/local_sine_tutorial -.. platforms/bebop -.. platforms/theta -.. contributing -.. FAQ -.. libE_module -.. data_structures/data_structures -.. user_funcs -.. job_controller/jc_index -.. logging -.. dev_guide/release_management/release_index.rst -.. dev_guide/dev_API/developer_API.rst -.. release_notes - - .. toctree:: :maxdepth: 2 :caption: Getting Started: diff --git a/docs/latex_index.rst b/docs/latex_index.rst new file mode 100644 index 000000000..59f78a832 --- /dev/null +++ b/docs/latex_index.rst @@ -0,0 +1,45 @@ +.. libEnsemble documentation master file for the latex build + +.. image:: images/libE_logo.png + :alt: libEnsemble + + +======================================= +Welcome to libEnsemble's documentation! +======================================= + +libEnsemble is a library for managing ensemble-like collections of computations. + + +* New to libEnsemble? Start :doc:`here`. +* Try out libEnsemble with a :doc:`tutorial`. +* Go in-depth by reading the :doc:`User Guide`. +* Check the :doc:`FAQ` for common questions and answers, errors and resolutions. + + +.. toctree:: + :maxdepth: 2 + + Quickstart + user_guide + tutorials/local_sine_tutorial + platforms/bebop + platforms/theta + contributing + FAQ + libE_module + data_structures/data_structures + user_funcs + job_controller/jc_index + logging + dev_guide/release_management/release_index.rst + dev_guide/dev_API/developer_API.rst + release_notes + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From b83f3855e47b201d57881ee112f2cda713b48fd6 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:09:22 -0600 Subject: [PATCH 436/644] Moving duplicate welcome to welcome.rst --- docs/index.rst | 39 ++++++++++----------------------------- docs/latex_index.rst | 18 +----------------- docs/welcome.rst | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 46 deletions(-) create mode 100644 docs/welcome.rst diff --git a/docs/index.rst b/docs/index.rst index 9d5246845..9a95c8ee3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,36 +5,20 @@ .. image:: images/libE_logo.png :alt: libEnsemble +.. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue + :target: https://pypi.org/project/libensemble -.. only::html - | +.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master + :target: https://travis-ci.org/Libensemble/libensemble - .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue - :target: https://pypi.org/project/libensemble +.. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master + :target: https://coveralls.io/github/Libensemble/libensemble?branch=master - .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master - :target: https://travis-ci.org/Libensemble/libensemble +.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 + :target: https://libensemble.readthedocs.org/en/latest/ + :alt: Documentation Status - .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master - :target: https://coveralls.io/github/Libensemble/libensemble?branch=master - - .. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 - :target: https://libensemble.readthedocs.org/en/latest/ - :alt: Documentation Status - - | - -======================================= -Welcome to libEnsemble's documentation! -======================================= - -libEnsemble is a library for managing ensemble-like collections of computations. - - -* New to libEnsemble? Start :doc:`here`. -* Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`User Guide`. -* Check the :doc:`FAQ` for common questions and answers, errors and resolutions. +.. include:: welcome.rst .. toctree:: :maxdepth: 2 @@ -81,11 +65,8 @@ libEnsemble is a library for managing ensemble-like collections of computations. dev_guide/dev_API/developer_API.rst - - Indices and tables ================== * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 59f78a832..7c801667a 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -1,21 +1,6 @@ .. libEnsemble documentation master file for the latex build -.. image:: images/libE_logo.png - :alt: libEnsemble - - -======================================= -Welcome to libEnsemble's documentation! -======================================= - -libEnsemble is a library for managing ensemble-like collections of computations. - - -* New to libEnsemble? Start :doc:`here`. -* Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`User Guide`. -* Check the :doc:`FAQ` for common questions and answers, errors and resolutions. - +.. include:: welcome.rst .. toctree:: :maxdepth: 2 @@ -42,4 +27,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs/welcome.rst b/docs/welcome.rst new file mode 100644 index 000000000..dab1afb1c --- /dev/null +++ b/docs/welcome.rst @@ -0,0 +1,32 @@ +.. image:: images/libE_logo.png + :alt: libEnsemble + +.. only::html + | + + .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue + :target: https://pypi.org/project/libensemble + + .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master + :target: https://travis-ci.org/Libensemble/libensemble + + .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master + :target: https://coveralls.io/github/Libensemble/libensemble?branch=master + + .. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 + :target: https://libensemble.readthedocs.org/en/latest/ + :alt: Documentation Status + + | + +======================================= +Welcome to libEnsemble's documentation! +======================================= + +libEnsemble is a library for managing ensemble-like collections of computations. + + +* New to libEnsemble? Start :doc:`here`. +* Try out libEnsemble with a :doc:`tutorial`. +* Go in-depth by reading the :doc:`User Guide`. +* Check the :doc:`FAQ` for common questions and answers, errors and resolutions. From 1abd69807beb88d9bc371a562ddfe37043aee10c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:12:05 -0600 Subject: [PATCH 437/644] Removing duplicate welcome from index.rst --- docs/index.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 9a95c8ee3..f02175da7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,21 +2,6 @@ sphinx-quickstart on Fri Aug 18 11:52:31 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -.. image:: images/libE_logo.png - :alt: libEnsemble - -.. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue - :target: https://pypi.org/project/libensemble - -.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master - :target: https://travis-ci.org/Libensemble/libensemble - -.. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master - :target: https://coveralls.io/github/Libensemble/libensemble?branch=master - -.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 - :target: https://libensemble.readthedocs.org/en/latest/ - :alt: Documentation Status .. include:: welcome.rst From a58d13b1af502126dcf4cbd1258f9f7a6f50d0ca Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:20:09 -0600 Subject: [PATCH 438/644] Using xSDK wording --- README.rst | 2 +- docs/user_guide.rst | 4 ++-- docs/welcome.rst | 2 +- libensemble/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 297ee22c3..37ec8a6ac 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ What is libEnsemble? ==================== libEnsemble is a Python library to coordinate the concurrent evaluation of -ensembles of computations. Designed with flexibility in mind, libEnsemble can +dynamic ensembles of calculations. Designed with flexibility in mind, libEnsemble can utilize massively parallel resources to accelerate the solution of design, decision, and inference problems. diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 78ac30302..a9fc1b5b8 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -3,8 +3,8 @@ Introduction libEnsemble overview -------------------- -libEnsemble is a software library to coordinate the concurrent evaluation of -ensembles of calculations. libEnsemble uses a manager to allocate work to +libEnsemble is a Python library to coordinate the concurrent evaluation of +dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to various workers. A libEnsemble worker is the smallest indivisible unit to perform some calculation. The work performed by libEnsemble is governed by three routines: diff --git a/docs/welcome.rst b/docs/welcome.rst index dab1afb1c..160e69055 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -23,7 +23,7 @@ Welcome to libEnsemble's documentation! ======================================= -libEnsemble is a library for managing ensemble-like collections of computations. +libEnsemble is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. * New to libEnsemble? Start :doc:`here`. diff --git a/libensemble/__init__.py b/libensemble/__init__.py index 1b5ad41a3..c4a9bdf34 100644 --- a/libensemble/__init__.py +++ b/libensemble/__init__.py @@ -1,7 +1,7 @@ """ libEnsemble. -Library for managing ensemble-like collections of computations. +Library to coordinate the concurrent evaluation of dynamic ensembles of calculations. """ __version__ = "0.5.2" diff --git a/setup.py b/setup.py index 82f74fb8d..b6439cba8 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def run_tests(self): setup( name='libensemble', version='0.5.2', - description='Library for managing ensemble-like collections of computations', + description='Library to coordinate the concurrent evaluation of dynamic ensembles of calculations', url='https://github.com/Libensemble/libensemble', author='Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro', author_email='libensemble@lists.mcs.anl.gov', From fcd5db110be1b549ad613d55abc0d0d8662b506e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:27:55 -0600 Subject: [PATCH 439/644] Folding xSDK wording into README --- README.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 37ec8a6ac..64ed05d3b 100644 --- a/README.rst +++ b/README.rst @@ -25,9 +25,10 @@ What is libEnsemble? ==================== libEnsemble is a Python library to coordinate the concurrent evaluation of -dynamic ensembles of calculations. Designed with flexibility in mind, libEnsemble can -utilize massively parallel resources to accelerate the solution of design, -decision, and inference problems. +dynamic ensembles of calculations. The library is developed to use massively +parallel resources to accelerate the solution of design, decision, and +inference problems and to expand the class of problems that can benefit from +increased concurrency levels. libEnsemble aims for: @@ -37,15 +38,16 @@ libEnsemble aims for: • Portability and flexibility • Exploitation of persistent data/control flow. -The user selects or supplies a generation function that produces simulation -input as well as a simulation function that performs and monitors the -simulations. The generation function may contain, for example, an optimization -method to generate new simulation parameters on-the-fly and based on the -results of previous simulations. Examples and templates of these functions are +The user selects or supplies a function that generates simulation +input as well as a function that performs and monitors the +simulations. For example, the generation function may contain an +optimization routine to generate new simulation parameters on-the-fly based on the +results of previous simulations. Examples and templates of such functions are included in the library. libEnsemble employs a manager-worker scheme that can run on various -communication media (including MPI, multiprocessing, and TCP). Each worker can +communication media (including MPI, multiprocessing, and TCP); interfacing with +user-provided executables is also supported. Each worker can control and monitor any level of work from small sub-node jobs to huge many-node simulations. A job controller interface is provided to ensure scripts are portable, resilient and flexible; it also enables automatic detection of From c60da93611060a27196865d43a80f81d811b6f66 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:30:42 -0600 Subject: [PATCH 440/644] Removing badges (which weren't working) from quickstart --- docs/quickstart.rst | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 240804990..0152356dc 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,23 +1,2 @@ -.. only::html - .. image:: images/libE_logo.png - :alt: libEnsemble - - | - - .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue - :target: https://pypi.org/project/libensemble - - .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master - :target: https://travis-ci.org/Libensemble/libensemble - - .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master - :target: https://coveralls.io/github/Libensemble/libensemble?branch=master - - .. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 - :target: https://libensemble.readthedocs.org/en/latest/ - :alt: Documentation Status - - | - .. include:: ../README.rst :start-after: after_badges_rst_tag From b3fb8485e89b1dea90236521c5d52fc7f897d28e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:32:08 -0600 Subject: [PATCH 441/644] Including image in quickstart --- docs/quickstart.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 0152356dc..a470df97f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,2 +1,5 @@ +.. image:: images/libE_logo.png + :alt: libEnsemble + .. include:: ../README.rst :start-after: after_badges_rst_tag From 561e498aa42ddb43638d7c7de28ea8760851ff5d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:43:14 -0600 Subject: [PATCH 442/644] Removing multiple blank lines from rst files --- docs/FAQ.rst | 19 ------------------- docs/data_structures/alloc_specs.rst | 1 - docs/data_structures/gen_specs.rst | 1 - docs/data_structures/history_array.rst | 2 -- docs/data_structures/sim_specs.rst | 4 ---- docs/data_structures/work_dict.rst | 2 -- docs/dev_guide/dev_API/developer_API.rst | 1 - .../release_platforms/rel_pypi.rst | 6 ------ .../release_platforms/rel_spack.rst | 7 ------- .../release_management/release_process.rst | 4 ---- docs/examples/examples_index.rst | 1 - docs/index.rst | 5 ----- docs/job_controller/job_controller.rst | 1 - docs/job_controller/mpi_controller.rst | 1 - docs/latex_index.rst | 1 - docs/logging.rst | 1 - docs/platforms/bebop.rst | 12 ------------ docs/platforms/theta.rst | 13 ------------- docs/sim_gen_alloc_funcs.rst | 4 ---- docs/tutorials/local_sine_tutorial.rst | 9 --------- docs/user_funcs.rst | 1 - docs/user_guide.rst | 4 ---- docs/welcome.rst | 1 - 23 files changed, 101 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index be0592545..6a9a52aa6 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -6,8 +6,6 @@ If you have any further questions, feel free to contact us through Support_ .. _Support: https://libensemble.readthedocs.io/en/latest/quickstart.html#support - - Common Errors ------------- @@ -17,7 +15,6 @@ running with multiprocessing and multiple workers specified.** If your calling script code was switched from MPI to multiprocessing, make sure that libE_specs is populated with ``comms: local`` and ``nworkers: [num]``. - **What does "AssertionError: Should not wait for workers when all workers are idle." mean?** @@ -28,7 +25,6 @@ manager but no workers. This may also occur with two processes if you are using a persistent generator. The generator will occupy the one worker, leaving none to run simulation functions. - **I keep getting: "Not enough processors per worker to honor arguments." when using the job controller. Can I launch jobs anyway?** @@ -41,7 +37,6 @@ Automatic partitioning of resources can be disabled if you want to oversubscribe Note that the job_controller ``.launch()`` method has a parameter``hyperthreads`` which will attempt to use all hyperthreads/SMT threads available if set to ``True`` - **FileExistsError: [Errno 17] File exists: './sim_worker1'** This can happen when libEnsemble tries to create sim directories that already exist, @@ -51,7 +46,6 @@ set to ``local``. To create differently named sim directories, you can use the ``sim_dir_suffix`` option in :ref:`sim_specs`. - **PETSc and MPI errors with "[unset]: write_line error; fd=-1 buf=:cmd=abort exitcode=59"** with ``python [test with PETSc].py --comms local --nworkers 4`` @@ -66,12 +60,9 @@ build of PETSc. This error may depend on how multiprocessing handles an existing MPI communicator in a particular platform. - - HPC Errors and Questions ------------------------ - **Why does libEnsemble hang on certain systems when running with MPI?** This may occur if matching probes, which mpi4py uses by default, are not supported @@ -86,7 +77,6 @@ Add these two lines BEFORE ``from mpi4py import MPI``:: Also see https://software.intel.com/en-us/articles/python-mpi4py-on-intel-true-scale-and-omni-path-clusters - **can't open hfi unit: -1 (err=23)** **[13] MPI startup(): tmi fabric is not available and fallback fabric is not enabled** @@ -104,7 +94,6 @@ variables:: Alternatively, libEnsemble can be run in central mode where all workers run on dedicated nodes, while launching all sub-jobs onto other nodes. - **What does "_pickle.UnpicklingError: invalid load key, '\x00'." indicate?** This has been observed with the OFA fabric, and usually indicates MPI messages @@ -114,8 +103,6 @@ does libEnsemble hang on certain systems when running with MPI?" For more information see: https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe - - libEnsemble-theory ------------------ @@ -135,18 +122,14 @@ to ``pdb``. How well this works varies by system:: from libensemble.util.forkpdb import ForkablePdb ForkablePdb().set_trace() - .. _xQuartz: https://www.xquartz.org/ - **Can I use the MPI Job Controller when running libEnsemble with multiprocessing?** Actually, yes! The job controller type only determines how launched jobs communicate with libEnsemble, and is independent of ``comms`` chosen for manager-worker communications. - - macOS-specific Errors --------------------- @@ -156,7 +139,6 @@ Resolve this by appending ``127.0.0.1 [your hostname]`` to /etc/hosts. Unfortunately, ``127.0.0.1 localhost`` isn't satisfactory for preventing this error. - **How do I stop the Firewall Security popups when running with the Job Controller?** There are several ways to address this nuisance, but all involve trial and error. @@ -167,7 +149,6 @@ Job Controller executable. We've had limited success running ``sudo codesign --f on our Job Controller executables, then confirming the next alerts for the executable and ``mpiexec.hydra``. - **Frozen PETSc installation following a failed wheel build with** ``pip install petsc petsc4py`` Following a failed wheel build for PETSc, the installation process may freeze when diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 4131d6c83..ec942f6e7 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -18,7 +18,6 @@ Allocation function specifications to be set in user calling script and passed t 'out' [list of tuples] : Default: [('allocated',bool)] - .. note:: * The alloc_specs has the default keys as given above, but may be overidden by the user. diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index c2cfd8c8a..cfd99adf0 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -31,7 +31,6 @@ Generation function specifications to be set in user calling script and passed t :ref:`sim_specs example`, where 'x' is defined in the gen_specs 'out' field to give two positional floats. - .. seealso:: .. _gen-specs-exmple1: diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index ab0c7a1b5..04f5955eb 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -11,8 +11,6 @@ Fields in ``H`` include those specified in ``sim_specs['out']``, ``gen_specs['out']``, and ``alloc_specs['out']``. All values are initiated to 0 for integers, 0.0 for floats, and False for booleans. - - Below are the protected fields used in ``H`` .. literalinclude:: ../../libensemble/libE_fields.py diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index 02fca0801..0e593eea6 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -5,7 +5,6 @@ sim_specs Simulation function specifications to be set in user calling script and passed to ``libE.libE()``:: - sim_specs: [dict]: Required keys : @@ -17,7 +16,6 @@ Simulation function specifications to be set in user calling script and passed t 'out' [list of tuples (field name, data type, [size])] : sim_f outputs that will be stored in the libEnsemble history - Optional keys : 'save_every_k' [int] : @@ -38,7 +36,6 @@ Simulation function specifications to be set in user calling script and passed t * The tuples defined in the ``'out'`` list are entered into the master :ref:`history array` * The ``sim_dir_prefix`` option may be used to create simulation working directories in node local/scratch storage when workers are distributed. This may have a performance benefit with I/O heavy sim funcs. - .. seealso:: .. _sim-specs-exmple1: @@ -49,7 +46,6 @@ Simulation function specifications to be set in user calling script and passed t :start-at: sim_specs :end-before: end_sim_specs_rst_tag - The dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding generator ``'out'`` field ``'x'`` (see :ref:`gen_specs example`). Only the variable name is then required in ``sim_specs['in']``. diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 2863fbd6a..14730b2e5 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -6,7 +6,6 @@ work dictionary Dictionary with integer keys ``i`` and dictionary values to be given to worker ``i``. ``Work[i]`` has the following form:: - Work[i]: [dict]: Required keys : @@ -20,7 +19,6 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` 'blocking' [list of ints]: Workers to be blocked by this calculation 'persistent' [bool]: True if worker 'i' will enter persistent mode - .. seealso:: For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. diff --git a/docs/dev_guide/dev_API/developer_API.rst b/docs/dev_guide/dev_API/developer_API.rst index be0798b55..622ecf80c 100644 --- a/docs/dev_guide/dev_API/developer_API.rst +++ b/docs/dev_guide/dev_API/developer_API.rst @@ -3,7 +3,6 @@ libEnsemble Internal Modules Internal modules of libEnsemble. - .. toctree:: :maxdepth: 1 :caption: Modules: diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst index 98f1b6f84..b7dca74f3 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -10,28 +10,22 @@ The package is stored on PyPI in the form of a source distribution (commonly known as a tarball). The tarball could be obtained from GitHub, though historically this has been created with a checkout of libEnsemble from git. - You will need logon credentials for the libEnsemble PyPI. You will also need twine (which can be pip or conda installed). - In the package directory on the master branch (the one containing setup.py) do the following: - Create distribution:: python setup.py sdist - Upload (you will need username/password here):: twine upload dist/* - If you now do "``pip install libensemble``" it should find the new version. - It should also be visible here: https://pypi.org/project/libensemble/ diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index da6c73f14..c2f941461 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -17,13 +17,11 @@ You now have a configuration like shown in answer at: https://stackoverflow.com/ Upstream, in this case, is the official Spack repository on GitHub. Origin is your fork on GitHub and Local Machine is your local clone (from your fork). - Make sure ``SPACK_ROOT`` is set and spack binary is in your path:: export SPACK_ROOT= export PATH=$SPACK_ROOT/bin:$PATH - DO ONCE in your local checkout: To set upstream repo:: @@ -36,7 +34,6 @@ To set upstream repo:: git remote set-url --push upstream no_push git remote -v # Check for line: `upstream no_push (push)` - Now to update (the main develop branch) --------------------------------------- @@ -71,7 +68,6 @@ This may requires a forced push:: git push origin develop --force - Making changes -------------- @@ -81,7 +77,6 @@ you have multiple packages, to make separate branches for each package. See the Spack [packaging](https://spack.readthedocs.io/en/latest/packaging_guide.html) and [contibution](https://spack.readthedocs.io/en/latest/contribution_guide.html) guides for more info. - Quick example to update libEnsemble:: git branch update_libensemble @@ -93,7 +88,6 @@ This will open the libEnsemble ``package.py`` file in your editor (given by env Or just open it manually: var/spack/repos/builtin/packages/py-libensemble/package.py - Now get checksum for new lines: Get the tarball (see PyPI instructions), for the new release and use:: @@ -115,7 +109,6 @@ If OK add, commit and push to origin (forked repo):: Once the branch is pushed to the forked repo - go to GitHub and do a pull request from this branch on the fork to the develop branch on the upstream. - Express summary: Make fork identical to upstream ------------------------------------------------ diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index 86274e1e9..54c062e94 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -31,12 +31,10 @@ Before release - Documentation must build and display correctly wherever hosted (currently readthedocs.com). - - Pull request from either develop or release branch to master requesting reviewer/s (including at least one other administrator). - Reviewer will check tests have passed and approve merge. - During release -------------- @@ -52,10 +50,8 @@ An administrator will take the following steps. - Spack package will be updated (:ref:`Spack release`). - - If the merge was made from a release branch (instead of develop), merge this branch into develop. - After release ------------- diff --git a/docs/examples/examples_index.rst b/docs/examples/examples_index.rst index d4a4dd89d..10b7003fa 100644 --- a/docs/examples/examples_index.rst +++ b/docs/examples/examples_index.rst @@ -3,7 +3,6 @@ Example User Funcs Example gen, sim and alloc functions for libEnsemble. - .. toctree:: :maxdepth: 2 :caption: libEnsemble Example Functions: diff --git a/docs/index.rst b/docs/index.rst index f02175da7..440dc149c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,14 +14,12 @@ release_notes FAQ - .. toctree:: :maxdepth: 1 :caption: Tutorials: tutorials/local_sine_tutorial - .. toctree:: :maxdepth: 1 :caption: Running on: @@ -29,7 +27,6 @@ platforms/bebop platforms/theta - .. toctree:: :maxdepth: 2 :caption: User Guide: @@ -41,7 +38,6 @@ job_controller/jc_index logging - .. toctree:: :maxdepth: 2 :caption: Developer Guide: @@ -49,7 +45,6 @@ dev_guide/release_management/release_index.rst dev_guide/dev_API/developer_API.rst - Indices and tables ================== diff --git a/docs/job_controller/job_controller.rst b/docs/job_controller/job_controller.rst index 9b0a891b0..2e6b59e50 100644 --- a/docs/job_controller/job_controller.rst +++ b/docs/job_controller/job_controller.rst @@ -29,7 +29,6 @@ polled/killed (or through other job or job controller functions). .. :member-order: bysource .. :members: poll, kill, workdir_exists, file_exists_in_workdir, read_file_in_workdir, stdout_exists, read_stdout, stderr_exists, read_stderr - Job Attributes -------------- diff --git a/docs/job_controller/mpi_controller.rst b/docs/job_controller/mpi_controller.rst index 559a13a57..70385d42b 100644 --- a/docs/job_controller/mpi_controller.rst +++ b/docs/job_controller/mpi_controller.rst @@ -19,7 +19,6 @@ See the controller API below for optional arguments. .. :member-order: bysource .. :members: __init__, register_calc, launch, manager_poll - Class specific attributes ------------------------- diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 7c801667a..742e29d17 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -21,7 +21,6 @@ dev_guide/dev_API/developer_API.rst release_notes - Indices and tables ================== diff --git a/docs/logging.rst b/docs/logging.rst index a5bdd53b6..c0a244e10 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -24,7 +24,6 @@ This boundary can be adjusted as follows:: stderr displaying can be effectively disabled by setting the stderr level to CRITICAL. - Logging API ----------- diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 5ff711620..2a42413a7 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -5,7 +5,6 @@ Bebop Bebop_ is the newest addition to the computational power of LCRC at Argonne National Laboratory, featuring both Intel Broadwell and Knights Landing nodes. - Before Getting Started ---------------------- @@ -16,7 +15,6 @@ users can charge hours to a project or their personal allocation (default). Bebop consists primarily of login and compute nodes. Users start on the login nodes, and schedule work for execution on the compute nodes. - Configuring Python ------------------ @@ -31,7 +29,6 @@ dependencies:: conda create --name my_env intelpython3_core python=3 source activate my_env - Installing libEnsemble and Dependencies --------------------------------------- @@ -45,14 +42,12 @@ following block: (my_env) user@beboplogin4:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py (my_env) user@beboplogin4:~$ pip install libensemble - Job Submission -------------- Bebop uses Slurm_ for job submission and management. The two commands you'll likely use the most to run jobs are ``srun`` and ``sbatch`` for running interactively and batch, respectively. - Interactive Runs ^^^^^^^^^^^^^^^^ @@ -60,12 +55,10 @@ You can allocate four Knights Landing nodes for thirty minutes through the follo salloc -N 4 -p knl -A [username OR project] -t 00:30:00 - With your nodes allocated, queue your job to start with five MPI ranks:: srun -n 5 python calling.py - ``mpirun`` should also work. This launches a worker on every node, on which every worker can perform stand-alone calculations or launch jobs through the job controller. @@ -74,7 +67,6 @@ worker can perform stand-alone calculations or launch jobs through the job contr more MPI process than the number of allocated nodes. The Manager and first worker run together on a node. - Batch Runs ^^^^^^^^^^ @@ -101,13 +93,11 @@ for a libEnsemble use-case running on Broadwell nodes resembles the following: srun --ntasks 5 python3 calling_script.py - With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble on Bebop becomes:: sbatch myscript.sh - Debugging Strategies -------------------- @@ -122,13 +112,11 @@ a bash session on a Knights Landing node for thirty minutes with:: You will need to re-activate your conda virtual environment and reload your modules! Configuring this routine to occur automatically is recommended. - Additional Information ---------------------- See the LCRC Bebop docs here_ for more information about Bebop. - .. _Bebop: https://www.lcrc.anl.gov/systems/resources/bebop/ .. _LCRC: https://www.lcrc.anl.gov .. _Anaconda: https://www.anaconda.com/distribution/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index c87fd2280..d266a347f 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -5,7 +5,6 @@ Theta Theta_ is a 11.69 petaflops system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. - Before Getting Started ---------------------- @@ -21,7 +20,6 @@ user batch-scripts to run on the compute nodes. libEnsemble functions need to incorporate the libEnsemble :ref:`job controller` to perform calculations on Theta. - Configuring Python ------------------ @@ -38,11 +36,9 @@ dependencies:: More information_ on using Conda on Theta. - Installing libEnsemble and Dependencies --------------------------------------- - libEnsemble and mpi4py ^^^^^^^^^^^^^^^^^^^^^^ @@ -56,7 +52,6 @@ following block: (my_env) user@thetalogin6:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py (my_env) user@thetalogin6:~$ pip install libensemble - Balsam ^^^^^^ @@ -104,7 +99,6 @@ Theta features one default production queue, ``default``, and two debug queues, .. note:: For the default queue, the minimum number of nodes to allocate at once is 128 - Interactive Runs ^^^^^^^^^^^^^^^^ @@ -163,13 +157,11 @@ for a libEnsemble use-case may resemble the following: python $EXE $COMMS $NWORKERS > out.txt 2>&1 - With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble on Theta becomes:: $ qsub --mode script myscript.sh - Balsam Runs ^^^^^^^^^^^ @@ -219,7 +211,6 @@ for execution. This is an example Balsam submission script: # Activate Balsam database . balsamactivate default - # Currently need at least one DB connection per worker (for postgres). if [[ $NUM_WORKERS -gt 128 ]] then @@ -250,7 +241,6 @@ for execution. This is an example Balsam submission script: See **Additional Information** for the Balsam docs. - Debugging Strategies -------------------- @@ -266,7 +256,6 @@ queue interactively:: $ qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I - Additional Information ---------------------- @@ -274,8 +263,6 @@ See the ALCF guides_ on XC40 systems for more information about Theta. Read the documentation for Balsam here_. - - .. _ALCF: https://www.alcf.anl.gov/ .. _Theta: https://www.alcf.anl.gov/theta .. _Balsam: https://www.alcf.anl.gov/balsam diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index cfcaeba66..d1f627131 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -25,7 +25,6 @@ Parameters: **libE_info**: :obj:`dict` :doc:`(example)` - Returns: ******** @@ -51,7 +50,6 @@ The gen_f calculations will be called by libEnsemble with the following API:: out = gen_f(H[gen_specs['in']][sim_ids_from_allocf], persis_info, gen_specs, libE_info) - Parameters: *********** @@ -67,7 +65,6 @@ Parameters: **libE_info**: :obj:`dict` :doc:`(example)` - Returns: ******** @@ -114,7 +111,6 @@ Parameters: **persis_info**: :obj:`dict` :doc:`(example)` - Returns: ******** diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index ff9a7b8ce..aa71d28af 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -88,7 +88,6 @@ For now, create a new Python file named ``generator.py``. Write the following: import numpy as np - def gen_random_sample(H, persis_info, gen_specs, _): # underscore parameter for internal/testing arguments @@ -109,13 +108,11 @@ For now, create a new Python file named ``generator.py``. Write the following: # Send back our output and persis_info return out, persis_info - Our function creates 'batch_size' random numbers uniformly distributed between the 'lower' and 'upper' bounds. A random stream from ``persis_info`` is used to generate these values. Finally, the values are placed into a NumPy array that meets the specifications from ``gen_specs['out']``. - Simulator function ------------------ @@ -131,7 +128,6 @@ Create a new Python file named ``simulator.py``. Write the following: import numpy as np - def sim_find_sine(H, persis_info, sim_specs, _): # underscore for internal/testing arguments @@ -148,7 +144,6 @@ Our simulator function is called by a worker for every value in it's batch from generator function. This function calculates the sine of the passed value, then returns it so a worker can log it into ``H``. - Calling Script -------------- @@ -193,7 +188,6 @@ inputs and outputs from those functions to expect. 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') - Recall that each worker is assigned an entry in the :ref:`persis_info` dictionary that, in this tutorial, contains a ``RandomState()`` random stream for uniform random sampling. We populate that dictionary here. Finally, we specify the @@ -264,7 +258,6 @@ and run ``python3 calling_script.py`` again .. code-block:: python :linenos: - import matplotlib.pyplot as plt colors = ['b', 'g', 'r', 'y', 'm', 'c', 'k', 'w'] @@ -280,10 +273,8 @@ and run ``python3 calling_script.py`` again plt.legend(loc = 'lower right') plt.show() - --- - Next Steps ---------- diff --git a/docs/user_funcs.rst b/docs/user_funcs.rst index 6c8d60211..f912ed73b 100644 --- a/docs/user_funcs.rst +++ b/docs/user_funcs.rst @@ -6,7 +6,6 @@ libEnsemble requires functions for generation, simulation and allocation. While libEnsemble provides a default allocation function, the sim and gen functions must be provided. The required API and examples are given here. - .. toctree:: :maxdepth: 2 :caption: libEnsemble User Functions: diff --git a/docs/user_guide.rst b/docs/user_guide.rst index a9fc1b5b8..960fc7864 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -23,7 +23,6 @@ The default ``alloc_f`` tells each available worker to call ``sim_f`` with the highest priority unit of work from ``gen_f``. If a worker is idle and there is no ``gen_f`` output to give, the worker is told to call ``gen_f``. - Expected use cases ------------------ @@ -64,12 +63,10 @@ to support) and plan to have examples of: point is ruled to be an optimum, a different ``gen_f`` can produce a collection of parameters necessary for sensitivity analysis of ``sim_f``. - Naturally, combinations of these use cases are supported as well. An example of such a combination is using libEnsemble to solve an optimization problem that relies on simulations that fail frequently. - The libEnsemble History Array ----------------------------- @@ -103,7 +100,6 @@ include the following fields: * ``returned`` [bool]: Has this worker completed the evaluation of this unit of work? - libEnsemble Output ------------------ diff --git a/docs/welcome.rst b/docs/welcome.rst index 160e69055..4979ea952 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -25,7 +25,6 @@ Welcome to libEnsemble's documentation! libEnsemble is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. - * New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. * Go in-depth by reading the :doc:`User Guide`. From 3d4ee5a78ed084665fdca5e6aa814415ee20f899 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 07:46:40 -0600 Subject: [PATCH 443/644] Removing multiple blank lines from rst files --- CHANGELOG.rst | 9 --------- README.rst | 7 ------- libensemble/tests/balsam_tests/readme.rst | 15 --------------- postproc_scripts/readme.rst | 1 - 4 files changed, 32 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78cfc6265..9d268c86a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,6 @@ Release Notes ============= - Release 0.5.2 ------------- @@ -27,7 +26,6 @@ Release 0.5.2 * These are unchanged from v0.5.0 - Release 0.5.1 ------------- @@ -46,7 +44,6 @@ Release 0.5.1 * These are unchanged from v0.5.0 - Release 0.5.0 ------------- @@ -80,20 +77,17 @@ Release 0.5.0 * Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. * TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. - Release 0.4.1 ------------- :Date: February 20, 2019 - * Logging no longer uses root logger (Also added option to change libEnsemble log level) (#105) * Added wait_on_run option for job controller launch to block until jobs have started (#111) * persis_info can be passed to sim as well as gen functions (#112) * Post-processing scripts added to create performance/utilization graphs (#102) * New scaling test added (not part of current CI test suite) (#114) - Release 0.4.0 ------------- @@ -111,7 +105,6 @@ Release 0.4.0 * OpenMPI is not supported with direct MPI launches as nested MPI launches are not supported. - Release 0.3.0 ------------- @@ -133,7 +126,6 @@ Release 0.3.0 * OpenMPI is not supported with direct MPI launches as nested MPI launches are not supported. - Release 0.2.0 ------------- @@ -150,7 +142,6 @@ Release 0.2.0 * Killing MPI jobs does not work correctly on some systems (including Cray XC40 and CS400). In these cases, libEnsemble continues, but processes remain running. * OpenMPI does not work correctly with direct launches (and has not been tested with Balsam). - Release 0.1.0 ------------- diff --git a/README.rst b/README.rst index 64ed05d3b..18a6bfdef 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,6 @@ A visual overview is given in the libEnsemble poster_. .. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 - Dependencies ------------ @@ -75,7 +74,6 @@ For libEnsemble running with the mpi4py parallelism: * mpi4py_ v2.0.0 or above - Optional dependency: * Balsam_ @@ -98,7 +96,6 @@ PETSc and NLopt must be built with shared libraries enabled and present in should produce a file nlopt.py if Python is found on the system. NLopt may also require SWIG_ to be installed on certain systems. - Installation ------------ @@ -119,7 +116,6 @@ If necessary, you may install all optional dependencies (listed above) at once w A `tarball `_ of the most recent release is also available. - Testing --------- @@ -163,7 +159,6 @@ available online at on Travis CI, Balsam integration with libEnsemble is now tested via ``test_balsam_hworld.py``. - Basic Usage ----------- @@ -188,12 +183,10 @@ When specifying these options via command line options, one may use the ``parse_args`` function used in the regression tests, which can be found in ``libensemble/tests/regression_tests/common.py`` - See the `user-guide `_ for more information. - Documentation ------------- diff --git a/libensemble/tests/balsam_tests/readme.rst b/libensemble/tests/balsam_tests/readme.rst index d2d1ac3e5..b809af293 100644 --- a/libensemble/tests/balsam_tests/readme.rst +++ b/libensemble/tests/balsam_tests/readme.rst @@ -48,7 +48,6 @@ To quickly reset the tests to run again use the reset (python) script: The file readme.balsam_tests.txt contains a brief explanation of the tests - ------------- General Usage ------------- @@ -91,8 +90,6 @@ To clean: balsam rm apps --all - - 2 Register job/s ---------------- @@ -115,12 +112,10 @@ A log will also be created when run under hpc-edge-service/log/ The standard output will go to file .out. So in above case this will be job_balsam1.out which will be staged out to $WORKDIR - In this case 4 ranks per node and 1 node are selected. This is for running on the parent application (e.g., libEnsemble). This does not constrain the running of sub-apps (eg. helloworld), which will use the full allocation available. Note that the user jobs (launched in a libEnsemble job) are registered from within the code. For staging out files, the output directory needs to somehow be accessible to the code. For the tests here, this is simply the directory of the test scripts (accessed via the __file__ variable in python). Search for dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) - To list jobs: .. code-block:: bash @@ -133,7 +128,6 @@ To clean: balsam rm jobs --all - 3 Launch job/s -------------- @@ -151,7 +145,6 @@ For other launcher options: balsam launcher -h - 4 Reset jobs ------------ @@ -159,7 +152,6 @@ A script to reset the tests is available: reset_balsam_tests.py This script can be modified easily. However, to reset from the command line - without removing and re-adding jobs you can do the following. - Note: After running tests the balsam job database will contain something like the following (job_ids abbreviated for space): .. code-block:: bash @@ -178,15 +170,12 @@ Note: After running tests the balsam job database will contain something like th 183c5f01-a8df-... | outfile_for_sim_id_1_ranks3.txt | libe_workflow | helloworld | [01-30-2018 18:56:10 JOB_FINISHED] .................. - To remove only the generated jobs you can just use a sub-string of the job name eg: .. code-block:: bash balsam rm jobs --name outfile - - .. code-block:: bash $ balsam ls jobs @@ -197,17 +186,14 @@ To remove only the generated jobs you can just use a sub-string of the job name ----------------------------------------------------------------------------------------------------------------------- 29add031-8e7c-... | job_balsam1 | libe_workflow | test_balsam_1 | [01-30-2018 18:57:47 JOB_FINISHED] - To run again - change status attribute to READY (you need to specify job_id - an abbreviation is ok) eg: - .. code-block:: bash balsam modify jobs 29ad --attr state --value READY Now you are ready to re-run. - Theta tip - Interactive sessions -------------------------------- @@ -225,7 +211,6 @@ At time of writing theta does not log you out of interactive sessions. But jobs To see time remaining: - .. code-block:: bash qstat -fu diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index ec861b81c..ce160f1e8 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -12,7 +12,6 @@ The following scripts must be run in the directory with the **libE_stats.txt** f * **plot_libE_histogram.py**: Create histogram showing the number of completed/killed/failed user calculations binned by run-time. - ======================== Results analysis scripts ======================== From 2fb02f89e103a427e187568588d8668bcff8826f Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 7 Nov 2019 11:31:19 -0600 Subject: [PATCH 444/644] adjust centralized diagrams, add xml source --- docs/images/centralized_Balsam_ThS.png | Bin 57918 -> 64098 bytes docs/images/centralized_Balsam_ThS.xml | 136 +++++++++++++++++++++++++ docs/images/centralized_Bb.png | Bin 35065 -> 39907 bytes docs/images/centralized_Bb.xml | 112 ++++++++++++++++++++ docs/images/distributed_Bb.xml | 95 +++++++++++++++++ 5 files changed, 343 insertions(+) create mode 100644 docs/images/centralized_Balsam_ThS.xml create mode 100644 docs/images/centralized_Bb.xml create mode 100644 docs/images/distributed_Bb.xml diff --git a/docs/images/centralized_Balsam_ThS.png b/docs/images/centralized_Balsam_ThS.png index f6ff9451d904ea2450a687108b54599cf88123e6..8ab646c8afc72dd2f4dd67370c3a4744133a01d7 100644 GIT binary patch literal 64098 zcmb5V2|Sc-`!}w#g^EH{w#pKPF?OLDjGe*Qx2$8IVK8>tQnoB9*@;p_h?1>{vS&+{ zM)n9}%aZLqhwl5ipXc|yzxV(DG}D-CuIs#x^H{&f_dKt=NG;_PH0Nl@$jDBpswnD` zksaeFBRhJXni3pghgkk1BRg*4sbu8o>SOQdVn@aeRXF^MTSV9agY)EuDsqd6;PH4t zTMsL|pd$u+13$T9Y#r?#?Q9Q!77>O>U4uZb2}__L65LQ21OomO6%~{a7BM+I-wJK# z_Gc_jjGv>6ixsyBLQGH?48^BuW#j1PiNQH=L*d}Ns+*^s3-|^@*MR7x zK_m=(G;GCi;z~$Sh>^RoinEG?s4~hEE8(jyCV_G>@kBrzofVa|be+U~jkLU>K-nS) zHAgjD9Tm8SpQE$3uL2w+g-2>@iD7Xj2qo|<%w7d5s_W^eCGMaip{|B8c6H+xfg7VW zywr^CoOCp8(e@fn&aO&MXah7xLCM>}QC~q!LIGSU2Gg-}Ru@-*iU@1yh&U@N*$6|` z^bjf-F()xCWj#LyeI*G=Z$%d;v^x}PEfW)D7?Np-pJY+hyiu5RT5PZRS-k#I=Q;|dMZLS^=<47UF@9QTrq6`<09 zQ7m5F$Pi(#V{Ia;h*p+RQgD=n>mocwoM4V{gtn1`6WY)VYa{H9LWy|@d!Qg*t}1?# zFdq#YN0b{>OUHmTAy1g2kB1@J*h54Fi@>Sq7#J#>co?f2Le)hPF1mJp$~e5CgaXnD z;Vp^B;v5lHA~+Q<7o>rKv$Bym##UWf-AhSN)W<USu(zSQ7gh!7tpULp zJF6LKYby9COF`ZAot;VAH9@O5DcUGtP~dM>VRvn`hMTR5jV`c1!WQO))$o&$a`bhv z@|HAk1c#6wUi!*-8)0EXOBdn6Hjt>wEVFzZ2P{6xu`(d?AY_L8~KpV2+!xN2JIV|A2m^>IEfCNME;9TgV?eF+CCWj{r%hBuBBQU(wk zv;oY=KoV+b3{gTzXnHCcc-tB%=*6jVJWH68to?VX{vI%Sl`!2@bCuxNjQLzPQKs6j~Nq4jM zQimYX9zGCvM+HqgRfw*as;8HZmZFW0qZ7$k>JVjLYdb?nJ7W_qn2n;i3*68T?TUgx zoRLT;D`jC_1uR+@WJb7#wvvsl0vh6HP0A0Nj$S^_wlJ6p(oV(M*B$Mt=H`HRQ->Jp zB9ygoJ}4y-BPVfDxSFG)k&3ILy(SD{Aa3ZWq-QIpWnd`buB>1p1yfWM({NKa)Wf22 ze!|KcCU%l|DH|~@9fSd>L~uJi-oV;H%?4#9ZY<_#rQw0FfeHH>=&4HqWm>za8yE;% z8N+e5T3WgqcBH@oFQm1St{y2FA51HaICt$ zo*~j%NzL8CMA_ccQQR7Cqhf*+_jB}y;0%x+x)NAFki;-pS0`6jNhcqSjk~qBrm=z{ z5>zXMs-FQ$OUcW`Q_@7u4x#0VaTKvvH!*R7i6Gq(sv=Myv>w{cL<()?Z6&6lCt_=e z0T&}}kx*TvCXms<+r|SfVQXY)Am*j!=`4bibQQM2i6~0ynuuX-ppNzu$|xOGEu5&7 zmZk&LRn0?11!bfGS95l=GXi(F2Za{yXdor5qNN5C)pEr9g6~qsK03a}zUtm4cySFM zZHyfP>jslD5m$8fC50OlqAa3-b(d7NF~q2${k-A!_6lAIq==HA70yRZ#8A~s3@0Hf zEN-A>D27G&D)`ttSljwRU6ph&l3KbhNH0-!H*qY)$`R_Oqy)n{>iFWk@glA!*2XG! zws;KEM^w#N(Zo?x+*Qn7LBZP44`du*4uqjT~QD-^zg1PU=&OEdtGgordajqUD3Yg9sSg5HRwknznWm{V;9py6SPhd#AmB_@+S3YWP zy1lc<(j`2T&U*8veW#s8$;XK;pC2PBQdt91#IB#|T{AeoDEV)8&EM>I!ke^@d_Pio zszcUk(VRMz6TDO=-$v&aR6o(q+Ph(IKRxm2(IctZ+1W0%t!+(4LV~*_6uJt1Vc@S< z*ngTV{ar^)Oic8-*y*tL_IAAJxei)zzVDSQSD;Tc9<2x~aqnph^>r=2I|-o{q$h`g zm$iOCz)s046IU|?qFDfp9DDRf_R-HmtgNiD;9BSDjx-79hVbJXI;c6Jm=Aj+S8GzJ z&U!n9kc6WSB`>ga5KRpQcXg_hNhkXn_UhHEGgq!Ss`EL`m0aQ9Gh|HOR2O(3!De!J zKIf4i#rN;ur!aX@nM{o1CX2iAxcgOpem)wBn#b_*&#>EIa};Qg(jLeb|NZ`h2h7Qt znS{@l%9yc+2;0@^4%_{`-N~<|HhFb*nPxRU39nwiPAV=Iv=(L|!%CW)o9ETmW{{5e z4GbjNe2h!j-QGw7b4Y4vXb3u{4kj5z2*#xj9`_RQ7yUcc_=}qfNV`!Tl*Bi$d`v-V|d#h3K5|%^PZwFq{;>E z`8dJH0-HbQCj7Fnvm$3UGe6Xj#{*qGJ+VNqaf0s*TMvQI>HqP}t(?zaz97JX47p=% zJ6n!^{4RCpnG*ejF*kEcUS8fi6;UW~Xr0ak#ADxXlc%l=Llt>1Uc_#kKE8f-~Q{mllm?$^Ia7tu2uPQL#ti#_r}a`tOTqNq9+H@ zSry`pMTxnIlWO^H{kz+SwT!v0hzG%Fd=xi@4aBLDT7bOV>G)H8N~!#3sEu^yN{KXVKeAaC39- zY8Cd2m&@4s)n*2)wD8v8&mH+5R*vpEU2B1HWSwYx&bJ^bUq7O(VQ9$1vs;W`jB?~- zX`nf<#b^38J>c4@-&jrS0;G7_jy-%@Tl;N%?j~^Xt5dS0+Bu(n`59RkQC|aI6LM}+ z|52L41B1d{s_HY9_8sRuc7J;jWbTqthx{BUF->>KN=z&rkP-bT#HqIJfW;0e^liqB zUMjkApzc8!@cQU$FXo!?AL+@sR$2v0unlTGq3!(^48D7h`=u=S_4gW4*v@0k`tRut z4n3MOpMsWIquW7Lz6?Y7`t4hXeNl*xQy!rgu?+mhWc8>*hE}K9UZkgASZO+IxkNZleuGX_Oe|r3-sQ{W@3rRTq+k5? z!hyGDTcaF1qY%v|Z&|D;4${h<7a$4;=iS%6yu51gdt~%yo@!-`w;Mi6x^FBRRA!-c z>KN@wE&n?-X+(Row~{mL6PjEQ%fo9wB`)?jh@S0`G5q=E#YAl{)h!m^-CqN&Fm5q@ z+^>Z1?ZQI(i;^XGuJ1$i%BEs;y6S!gn3|p^ zvdNKsPyv6;EdJ_!+7)P*E-ln2Q^ffh*y_N{$!{5&Wxm?f{h2!Vw+xULtO?KjwijOe z66cR_+7tYKUPB;_UvlQ-!Y|t0cJkQs$%^Ow$lp+|ytOqY*CSKKuJG>AW-Myz>i&tc zA$vm=&NcWoGI}UU1U0;pSs~i2u5v-x6rlyfDA56FsyfKWa z=l&SSc8QcVI0Dxgs@x|iiLTs=m9E3h)cmV6LT^Oc{SgMpaKVv)x`TlB4`%A+zxn3* z;hldveJL{DC^zJ5kk#`WEVHA{;fs1tsP(^AC(gz7e(l^Njp_Tu4T3)rQfKPxTcxm1 zNER?u@$?i$ry&`Yn1w7f{QUfUFi@`UFJGFCkF;yD@2*iRwZ^#5)gVba`mNRe3v$i4 z-Hd|HQ2#(h9_An^=cnLv4T{ZL{AaXOjA;1x_V)6stJ79Zo>3)NR*L7M_^;h&Ol@!1 z@K~MJBn)e+0GARDZJlg^63)v_yJuW16TW!aWi#F)%QsyXxudH~Ci|zgt!Y znh@21i(5%!?4?SJbyv#6cxM}H8flIO_L*4re(*EX|KneaJx&$o(-{TY^1Jsp;xgi2 zGNWuoSgZVg@lRA7JI*gFE1UQ1+1>fU^2nOsbA`48#pZdXrS3Njx$3qTWjt0URm(7= za=ks>$4>AY7MpSD>+4U}ZV=A*r6;CIU`_+xrFYaFkLR@v-W7we(2+_~3^BXk$rmHR z+#`m9;Y z{zYVzq$C|L@Us|{Y?sx}-safzzQn%cC%lo%8yo1I(SYO^b)l_aUYKsD55rbh+JvBl z0g&FPkL+5pqqH)lm25?fvr6vWz03Mpw=YfHbq?n<&=5h(+V%R?t0jUr+OTZy5#>EKt0-gR7XL^shnHck>kGrI8T)VE zNcK9vpa7m+j6E7wT`hArHkR9?D*X(*phb;?9}&n)bwQV}Je%Zxr!QP+_swK`#TRwa z*w}d2EmFVG077VFzi4D)(&|%JTpYJidFJ%?`hv~3e7WUg+Fi53tcwh$ldMa&qG*bj zFSpzn^LJlFRQD?!R035CeE!p?^;QJm=f%a-wFduMFM~vKBTv z#jpNQ(5^;KN@`Gaat?@-ZetG&FBws8BX8atJ$YzR z{M7_$B{b_l)ymX7`eT0H*33O;bQ_ckVFK^NHmOk|9-d}vq!m?k60f!h!O+aC-DmF` zeslpRllS!L#@6|-3XQ+FuNzK0z9(n>Vn>XqSG6Al;MlTu6;)>o3Nfifsl<5uGrn*} z>-onep#!frQ^q@24E0WDJbm(nhalq#Z`4JC@W^}qJZgD!bbk*Y4Vwa(GG^dezf*KD z8sxQ@feHZOPRVWFT29IDe`OK)OFtSGvFj`HC?;eJ$EcKW*1{*JYvH9s_r>O)^*bx- z&8zbQ{7g(tHLo+`-}o$Fyi{$St96_J_hmYS~U-v$&N4>q6Y0KF$%S&)EC~9u!uEp~acS4R-Q2Usr(Jt*lJs#-! zX7XbFiSNu^VrM%2tU!UNxVXu}=I}&RYq5DX%e~oOex2JO7dt`(3beNFm(icR{1PuJNP|GF8G6nr{8>T0hcFxnfOES~}Rq?A{lnz$!#U(+=@ znzg#y(Aap5l4;%J{d}RF1cq>Vpu|$5dvq7~wehn-iA9GG`-aXe$NHW<9sjbnzhASw zmT8GTU(#yyUIics*z6;UNNH zi9wGIuBv*^f3^BO@~s+=+Y@9%eU%99a}a$Z-1F_5=5+g`MxVNR{kJ4%-^>7sApk%D z5Pb2#nLYNHM;CY$C1FkYFSxbq$qXUEt;ldD#YaMV7cQi)(*rBP7S=x*(W zM!mnuiK*MN&w@35*(Rw!mC#lfeB5iMH7b8Q2Wbz>FE3Bp+}sq-#Nlw40D`UDG_Tv= z+YWk7;SaARrKfxMW&+rGdQ5NE*86UD1rT~M%}{oB_qK10!Jl+AKFcFU^DP)MWuI>Z z@ju%zd6Pv`{QN)>myLHBVfn$7r%`T%nif_h$AJnc;Y6?27P56>^j9%mJ{(RpXH)x8$WQ6&&u z1wLLd)nuhC>v5{I4}UTV@f3UCHBJ#UO%uc{6uJQorO_6ZkVqnRaRV0qv7~Yx05a>Y zmc0Z^442i$;8(BUf^Avh753EGgT7zVOtngjua$@&!rkEtiou3F&fy zsb=*-85;}yo?o}qM&!pRwCf)Q|60l@g)g437vP@+KVL1s238lQ(t}S%&C(pqqo_js3zsampsH7tjnw5WShfZGoc2bH&%r@D{`Dn!Bs=Ax7p(Mnq znGqb?>W)lAx2Jel4X4_8jOfl)&F|>FVX`-ISqvyL?8W)r*fXfUh~E1Qz6mgf2(BQj z-?wh@Ej%9af2M-DJHEEI2EfN7%ay=)lyTfpXt@`mCdPJRoPIT^*OYM<>dQ=<`-5D* znsNyme;WSluX%G7An$_)&D4^rD>wIInOz&R$Kr?->D|-U_rYK0D37M5=96_2%)P?T zKUdD_B+B}eufjXUUBzauwi)W{2@}nDe`adXVbgf$T~}>rR@1$&yD`~EZ=EUSE- zk-b!K=gId{k}5kseUfyVz42_|r>d{-R{YrjvYl=ML94Z`?XE@1Ya_beUMN3P)CV2V zK}t+~MnqFAKfSNg{JXKMhcf7PR(^QO&cV2>D#+vM1$t>zOrYJZl_`(a! zUiCe)^Icc*q5U9-_6yCnHW9n6M;;v&CG2ve0km<(7d*1N^CaTV{o++3wW_4X6H)S% z;Cz82V^h7nv^#X|`p$gnyRI8*)Pr5r{A7|0KVjT^j;qC*hh)i4EaPxo-MQ16WN?N z+dJ8I=)eGC;nN2L+_?s`>)UjpfEG9e3ruGxW@Z9N{YbiQn0zTLABqnSxcKD(xRuv|$t%IN#e+~n#pH~)KUCv|wrN2J`c-$GG+O@>~0g!74 z*WRKteVeb05+#UDHK?kI zokm(^Xw(=PojCXcLFF6iNyMScHl%~=;Mb59*=@4y)YOp z>~<(#6*M2Vf`A zL?wW#M)Kw{Gy}--AXAV8dECvIvTt8&LG0tActDd=UoStqxHw(&g^7ks)8Ai4hd;T7 zD!H>$t5fsi6D`e=0uq=aY5Qcy@_8t?DZzh2&re=BHqv`8h)kaLV^>EfU(0xm&krh)KxdirfMCzpuPP+RAmXBE-o%v=%1TILPNvDQroxF&?8yOF_|3NqdR?=M1l+epE(3B zO1X zmQDF3@HbvRzsSjKi3-*4h)e{2`}U1lXx6l`hCWl;t96|I8qV6<`csAT0OhM|b8~Yl zc6Ry4x!QFg7+ze-hmX8L-=`VMKYWvzJwIWFxu{c!X>-DfKtPXLz5j3nvu)=rK4 z1fqFc(Cp1SpP=qsrI;CKXXi2WlkY%}jDPD9uyY^VgNaqRne@7j!3ZpftC;cAXV0Sj zUR1z+KzF1C^8l@UX=y1luY79Og7w+#y=>YsKpU%KvBERH-ds=lq5$oA!fE(r$;+iG zOy8F;BI~idnJF9jW^%5BFCo3OvPaAN{)X1&-dt51GFw+cNyWeW_r{lKK?; zbrpio#EE8`?F}L?W!NwH8_!wY=ATJPNtzGWFaQ9vipM7=!s#e&AU0=PXR?AbJ@NP? z(17Ub>RMOOKS)l7uxE)mKX7bcM`r~hMn*G*=aEKi}il;2fHcWi!gX(kYZIdO#BphG>G|4mW;Htq|MfqsEhsm{Yy|_ zg6KX_E(H~}4XGT1EZ_Un)6;ek1?Jfe%=ijMlyv&O!6oaA_`@l~LQwkp5cd7;(S5g4 zdP0qF(y}z*)R&1bY1KfI-1;7Nishd15M@c-@8?bMPkYxNX*A!mW-0*<>-%ZfOL^JW zzWc(9XWiH#>g^k}^{ReE!mc#|)4ooteBxn97RBOB?^A8l3)AI}y*!6`YaSHMvpG)- zj~VJvzHjOfslLrb&NSmcYQiPTLT3vyRSeuFNcrYldW*V%N6ULsxM`yunfbO=bXO)f zrhZp^qLc)ZSL;?*Jfb7w10TGU2P~uu(AX*d%phpy^3oksIpDC*{IPQ!p(Nfcbmcxs ztR-2Ni`)l1UUtqOrFS@qnP2A)r!R%n#=s9xaXV*4f7 zvQYW>!{?WAW`(Kha>3E(r{{g`nwmmewZpLivxW5HpoP75bzRctHOQ>ZHh_s*tpjY1 z$<^?AdL1*f5nfX5^Bty^FKFK>j3mY|WoEiJUHaIWA%AO(B5wZc`>vyFtNVMzeUBh# z7niG!j*f@<@hHjTjz~L2OJxwhcyL$s*F4;a=M;4lBXYl(QtMowE4WsvZpv98Il3Rn z1P=?5d5~faKCPB)vgP+SZJM`tc6R~yasB#;HJ8~NEZ4+x+(~<*I^R{Es-WGsci)eY zX>7vs@uOJ19WGvMu9oKLO_x{FAD!pF7A3TcZ9mJ>X&$2x94y6&$(KhiBJaD4tle;1 zqyiOhsaz5O#*6M(Pi#3|GhAV(4nHYV^wpV~B{XVvBV>k2giPJkxu6b)`dM z^j#OQ=vBD_7XSC%w-9|(5{XLTQpmg&?RRfVD?1GG_#V6z*0kb9a=jbR`rqeL% z#?-FnPwPudyVnSSQghGT*w}deh9GBWKn<8HDd(B$s;Y#+a-Tdyd}9qL`kUp0zEN|yXigtI=5b$BtU3=4XF*}U}|NcTNC;drx0D6q!!!B&E<)(A@Ol)e(1iMP#EiyQP3;34Eb*LWSLG9 z;1m{a3nQ&trKYUOP-)+|O_IxXxbp5Rn?plhJCy0>XnQyC@C`CfX1WtUQ($&U=^!VZ zxKae%E12W!TQ7kLGE04ZyJauH@Bb_ZYHO}uE$hndep6j-x;qbI^1)?pZg;{NHa5*T zcG*@R7r-;eY*@U(hJ0vhGTuq+?(WX3s`C9je0#SlfTQFpk)!sE__5rTC#l+BQGA}6 z_i;R&ja^4qq5f~QgD*&XQXe9@-^TO^inuV@D5}f&pRFih z5&MA+?EkHCK9+iQaA-))n@G?ET~NzI%OJr74Cq)`drB=E2f{FjGVP$hL9BXMzCt`>uEPr0!#L z1scQePLH~921O}DPDh%}QR;_5v^j6FsUjDJL`Y1MkOl`meSTn{s`}f8|9KfdKR>0{ zF&=WJy^Z6XW-eF4zYNUbKP46~L0__h{@{4s4&7t53*<~6DU#p5m6loj0oo_Jya-#_ zTYv2ZpkEKxgCMwG4eT9}Kb97f6>|T3TbJh30fpdJ!+;Xx(x#S zQ`IvTY42MS|0e%g6cA+o&Erc2WH;OPLgLuyrAf_jz!obaS(2Kcn*aLo2>P!vY_vJG z8>co|ztZi{<&j$k-PYkg8A=1F4>%XO_@(c=$AT!ukI5gq^8R6#73ywSc4&4(5|TQk zgW6~RR#r#{sRQIJAd9^9|NDoV>d39-Io2|dga`K-^U^l!mZ>Mi?xbX}WW#T8k-@l32iQ-Z*TQg7k^OiTaC+inDE*3~y@`b;% zC$9;9_s1gis9Opze?>*N^+*} z^e!j(wS_vZJ(tvJdRe<3)~;XRCBI%P8NHnPHtID1uXpdUV})5%JfdR zEC_gk!5IGgU{Ck9cpl?87@pf+ufNP2afZzLaRY0K@#?&F^J>~&Wy2WEZmU6~Q~!e3 zo5WRf?RmnT?C}dZ^heVemi~5^8^|Lk-+#$oQ82v*8k?BT-TM0Lu6_jF8B3ld$Zh?O zyZ7v`k6P4c#mD)16jnS~6V2k3?cn{iUy&|SyElBhG0*y^z@&lmLUw2?ko5ytdRb!! z>j@sK(N1vbIn{-_j+m}CRZT^td)$BV1>qF^`e~jI!qOWR+u~_Qk2KNnt<|S#6UA#s zJ6=eLZf<|ly!TTS$8s-A#te~r$#3RWsY) zOBD*s7PFq*u89Y6#Y570g zKO}pc{A-4mp{?+$;%TDB?a@-Jvd3K+2xw{UkWJ6|d+8-~AKOu%^>rLR;$v zGcJ$4r|HD!mc;BV%tpk_m^kgXY|4^m=Cjw3fM-Bm5MqSigdQcxe@EYhl7f?(U+Wez zo|Bw%2xhpsQJk&R1-o)ZMM8FcVbP$%$34JzNDCKgWhNEY93)1XgrfZKYZcmZ#s%tI( z`C?xQ>0YC#PyO>;aePmiUzv}qSa&je^3{C1Uz6}vF~aND)ElRCwpgi;Z4S!4(&yk6 z<8D*w-?{9;34xXM-M@N?jiZ0Ht~O#C-ObX^zQlGq{M`+iAN|$hz>Q1?_L3n9->|cn zNg1QrhSwWoTQd`@-_BJ171q?F`+k>#37DeO|G}It2Eo7Gy1+yH)chts+Hhvjf z;lKN&meUJ`11XX*f~dg@D6q!o#e{(p^xl?zej>#`BmeFK6R(PB1lDViHCI4Z%>W~> ztz|yXqdBlApgJeS`e|-qCbqs}o{l(dZo3`nutOJqZ0EYC)>$ELs?%pw0+&y(m`??* z5X*lh=KcIcl~?C;Tq-{02Kap{u$s1|D_8h=?RS~W)FC0>pH-D*FQum>C5sHNom+x4oB@7>?Sd@cQdadpWvMnQ6Bqcj{X@iT6gX8;EO_!k8+j{!} zNy{Ut=(A9{GaEW%O1W3UJ&|KFijuUyH&$xDDOCpFY3{lq`L0VL@+5yiBsChgZ&CO6 z-N>1SL)Csh3S?IJG#F;OU3*k_>z?7lY1EJHg8P_&yUTAhc%Yi8`82_?QQ;Q#f ze}F8IxA%^@d0Qk2J^1PbJ&N*LHzS;!X^ivxu7YR~Wy6smc4=zS2PRA5y!HBXS%^@*;Wq5rxA_zfGdn9LXBBt(yLImJIuM*kdzwSa;#mQz{03Yxmk zsUb-jt8q6?D&)t=mMpsITik+6&8WuYeK&Jm-m=S z6@1@xx&Qa*%7oy@qgA{@%rg*+Ret950;5!cm403-mJ;I)UC6}NHwy*gtVUwEc23c>p<6q`GJXNO?ZfaYS$W%{{Jj^2f%=XT%c#XAms~Raa$TX5eWaE3`R(j+#=B4GFBYKUpFig(48y+vOGtX40pVxxZsJfQ{H78*>RiXYhR{2cVLP@{)UOm1yN8$2qJ8nwskJmeH%l61lZ-jU;euaoQp4!O4U1CeYbdx z&$y;Cbz|YAybR^PR1*q-&$#G;7P9FZy2c~+Y6CF%%|Qq!Fw#NPkc0d}&h%7OG~%uO(eDQDo8yOM9tK& ziSkE^2X_8@GCgfo58l4>D9*<}z?ttgTQS1#{ssgtkSQ@s`%m)PExSlcUWCT*$3LW3 zjy$t8P&|2=Ctl55Jg&?BG%)=?Z`~E>`MP__l5&>{ zBpD6q_(#odS}<(}B*lz45!(Ez>7`$%ZX|X|w6F{>eu#@u5WIa<&}{vI1FaQEJH&{V z!JATsvm}Zirlb~R5A67J%lJNm?P@U#3b10Ilnf%Py`Cm+aB_N@;Q3jH=X+yCJh1!Q z)cnUDf-#((1xRCjr}XV1NfR0cv_bh1J+NBN7b|UK1|jWB=+QB4-|BcFll-R4I>fZY zUy5t0`~;)au3KW$u{%cIp6r*0;J{eXB!Q$nf6x<^S`+&E`lz2!(*%&Bn*qdQ-XvQ6 z`{uk!5c3Nis<1-J>aZ1%3Kaka+1R+Z5j2 z*f%=AqOYR^dtO+mRQ`CC^AFHcoS5miL*D_heC1nEX8PN&9iZ4Tf+6!fw|#qeLx8Ze zXwC~DhujjNVhVfDVsNf=Zr4jm9P8~cXtZ0^z2*m9%V91QvcdM*byeoRfY%bR%Y2SCy1 zovY^_4;?c+#SbVyQL!&cy+ob_ZHe)@C^5Nje3zb%YR*CPP$*P9*p1vh|KmEq8lRcK zhP4v{05&R4poe-3+dAWwoa0s(#r{MIFr??sh(Tcbp9BdeLlznpLSkh>olOYa9_8ir zzIyqWDZGU@ZL%ivSlyOwQhFceI5#*vFOiKxu?&b27Z;g3;yS|se45P>DKx)X&^{-l zU}x|!V=S;b+J$7=!6jL)6roMRBFNN|AP_I8XmI{&QvZZc-fpSy`l6m#e2F9e_c-_D z4eO(%K;2jnyeuExP_TX!)}&%%Gm(9AC+(~b^H??KGR(&2aRJGhQsF9D8D9A}ZQW~`9T$!tg>>f1a^Gb&BvXm}8%f9R9?x?MIt}2JN)V~sP=rP$ zEH1j~nwz_>hH!uzd3pWHE`G991$+Qh1?hq%wu|W$yZ&2*#f!rv1&Rl72yMOVCr)1D zmhXi{5b4LR{upiB@IIUPr}p*G%96J~2$KgnBr;&R_HdlP5ruu2jBjI?^*UhH4;+x1 zU&E!gYx~ zF)>jwH_zxVyjskN0&cHrZq5dH%To0lISkmbve&?p3@Bj~LhWx9Bq=A;Q6Ov1Hko$& z*QAHt^)2i34;pu^)6&g~V|YhJi+I6rS`rtA!m`%s{~m!~MPHwZxWKveFSJ9=4@Mdx zjgirD68Nv<0en6hqw~nBXG(U5fxe#AD$4zJu$x z0x5`W+NGa&>KbyaC>Gua`yZrE0Jo_5`NBcAT%5di;jfC{7&Fw{A5y*^_I=#`kwOE_ zkOIpB*Y^fH%%5tk*3$JR={5FNRwQ6F;<`>ko>xFO{y%WZ?DK&<91C1vug>4IMpEZb zqTfCS#$G+O<03D=x*$f*beAGIZpc~-T~cAD3K-z2T0Q=MXM)qJKC?GC9@LHlh(!^B zR0EceV;dc216&#dO7S%6>In#muO%E|lAiA@t-XCAM>f3wU-~*gJM#b*ki4G`5U6)9 z7nsRxpGNyIbdGFvy_l{|w^APHtc_cbH~J4N8&jwoAog6QaeOrAr}hg(~*B{6A7?K zcUSQ%$qylUQj&6MEnrY>xc5|lp39W?uukWCAxN&gTInpFGr)*Nf2-c`VFnlIRY>XC z17v)nRfA5SrAU0w!ps36`wwWM+2$%FkhteD+!pmIAwkICl`$Lkg3J=E|Xuq67WXyMw0o5EX$`nf_s!71lI0TWAw7 z{^9uF!^62486r=zjf1}OPi6jGnv@8#cwo`r7hF#X6Z)4kRMLOx>szu&Nk8!bmrUB1 zb5P2~CFQC~z?7UaH0tFIBdI~4OmO=jnkIQ0W zVxd<;|C%Z})0M5k(_rui|2DWe^%42QDli)-vpCriZ2A}F=o!<^-r)2HnE@vAu(!vr zAdm@(2XNrX$yt`ie@7Fj*rvbSb@j?LRwNh;uNWhw<(|(3`bT3)walFtb8!Eg!i4zO zppaj+FzwjZ#IdP58p~FF4v5`YxF#weeX+nyazzj@GBenz$28F;!9m6+XDYS{zK=Gv zggDr2Oot{u7NS43Dw#BUZ~u$WgilxQ0>Wvg{EOV{ysoa&+kn7a(Y;Ifz^gw@S;4lC ztGaz*v?`PIPX9fV7W#L__>W5eCP`v{0W@YAz-4QwsL_}DUWR7(Pz?{%s_;dPFLf4b z<1G9aMqtq;9lh;ivfPwT{fD%m)%f8 zf+OYS#iCnp4vYQ)rvf1SyIUaSPEq{kSJ>q~29iWKfOeXm`N{`PB?iy6W!wjAQWl)4 z@ixw+gxOIqxX)!_L@G@c3lf)i64MzwD`R76&^7iQ<o-GK#oBYNu?H7_OhQ zA2Ai{C(*yh`vg4#X8gLTk^vnj#kLz9>^rnpBW>0)SP6LR(%9(_%Fs+q$*4BqF{=7a zoQJ&k2RvHWeq+v`Op1C@5^wsVBJ3ppGK9fE7S2_$#Pv@y)kkNG1>LNi3IQ*L2pY& zI(>zo5&#Cmpo08(4vSd)Ja*f<2lPb;NT)o@TnseQBLX^k<@f!-i&Y-|*g9f0};!gb#kUASzqaL?tgg%Nx`2gN52lN4+ z^LaC%g&`G0quu#d7{y2QV?|b!_^xZj>?q)Pdw&{~@QKQHF1n;qC0}#SWW}oF{HEr2 zxxj!D7qV^$A{bDvuzj`STJE&n1RnK>y|+DxPmPpv3x;T#E{I0E^RTjVukX z$X+Q?=ao|UeR|^YwwY>7U<3uMX%=8WMdW)VD`43NGR|czq?DJtdyG&T%k&FL=RAjr(*d>zLn)@&M zodi$%znv|um$&>tEjWkGZxXhaJKLm-d~}9G^UHp{br~9WWh*CEM%iuwr!I-J$ZZH? z&5F*xsPezEKXyo|I-INxd(*SvI#uA~&7#5Y!Gi}omTgy}VflcWjQQ9Y#uRh+`vd7J zz_4au_T;N~5IGddq!l3I4)Fccj~6qvb<6Q*s!)v=rhkFu3>!BjJSKJd(R{XU{G$(B z)$`d6h-XV&-?BJm|%;1MyLo`mU&&gD`0c68YF9kWFB8s`q$cox5_m?zwEk`COvi1@M_o;jEM%56ZzhiENlA8tg^3?0EFzar z*Kk>!)S!R6frN40Csntx9;{x6|4vO)?` zun5mSXv1A8<16KwC03&?nFv8V*eotnkTM$C{2=>u(HigTl0L2iK!;M0Gm%KMe|XqX zHe9-BbYywoM9cUDzhz)U`s49S51~Bv`#!(uj_ZraKkUker+M~2*S7`l$#)^g#3A?B zeF{CRnijUF=T7%Juh*`YH4QL2Rjq1MojmVr&G}Xt{#>Rr6C^p^?%OkBtM9Y z^phgBW~Cq+NS^G=qgrm=0_x?b{M2vEn4?$1vscEWg`{N*US$rVA>rw=Z~C2HW=!0ydJlSV`u{STk#yRAmuBJi!PyF6 zo$m$Z8Sjt7$ziR!=KMCM;rvYL11Y2Rw{!VUKR#KpX{>OZAH;0Hn_%eE7oBW^3P^@0 z2R+(Bufh3umjlSF?Vpbdv5$nDe=c^?F#&p=GcT;#DLKEKNUYntPD-cON@W1G>;mdm z*(^qYg}D34r69={d6(wYg8-z|%HH`HS1ZTGmgpwmn1Pz#^VZQ>$scL>A1q$&%2ar@ zm=9JKTgq)%yFMZ!Om|F=L!8K&f)4R~HEB8E&cSkk#2z6}>Hs0rpIln~ob%t9nD+o} z=YsWyX#j&*&>MzVdqkwGT+!=N3)R$)BC-H2hJq>nzX?Q|9L3$kKmvrr{{#PD$m*E% zfK9C@O9Rh51DN{p#YIHGq=yGULcOb&L2n|VYi$GxFHifAMn4v~09#606KMDc{g_Ly z)_1rJ@w9j>YW8_l;me90PC#OMQ~~#YvwF5bM&ZtB>%JJ77kZ3JU}fR%Tbrx5*Xc#5 z`NJN73i19Xoxs!Rhl$U(Q~@&;Ny%tl*%UuM7lCvF35X0}Bd-&NpZ*J($&qC)QXqOE z@ZK4fs|;)mPRon4)^jY;IkLZdE&Kl33lL5h+M4~J)2}X?K911e%He zTz*63b<_K9 z{_fvx0@yo$0uV8zMSxwp&0|58^8fc%VS28=@DCIE3;O-k=HMP+2>k#z%)kPcXO6<` zKA(2w%KN|2iWON$~H!pVqTcq<{znd6>OSo`|1q{z{n8@ z!k@ZO*6dnAUVi=y$bPM9rOVjvE!yyw2iHtlwwu>x@A6(_g4-!tT(&$AjPY;5RyruA zjeh`c`V$}_YVgt#o#1nFRa2z7Ql`{qEMBJhfEy^JNCpqE>~!yJ2%5jWzi~{Eo`>}4 z;{5rXGw6j$#)g)+EN|&akC`J1aW&0x9p`3?;=n$BN}9iApPilKJZsEW^Cd%#Ad?Ay z3K;Fu4iY&^wNWoWd1&7;Cm8qDtL?G7X%MS3FTrz*Q!;SiKwV` zdc6M|6J$=ohkfvT?XdG+T3cJ2ul&+cE7GpBn#rVpV6{-x9;)D=|HIo`hE>(I zZKI@8N~hA@APUkg-AIRYDguf$h~xt428kspB_ZA2AfOTo(%repJ*KzM^WN{bzx{Xr zn1{l_oO6tEb)47C&T=7@)e!%)I29FR$Lq9XPf-XNa`hWKP!ix4zHS~zT+uzSw?_Yn z=gGivdRKn*wn!p%iRP*rPS|&J57{L|8kU*-&eh!=(Ej*L;t#f?ZRBpld{i#2T$W$) z9o|nMM+5p;ZE<62LfCH+Cnz3g=@e~0xcLQ8C%>{mZ{deZL^7)j~}>>KuRwl+&O{;e(`T4I4TQ z+Ow(bH%!}r{&x($-FF>fLCwQIp|lC;n}YHh9mfG!$9yow%jHW%D#wYFlfC`I6%Ec= zkPl(`Y~|~w#mx5?PaOMtTXVE!^(3|T7<`g-)icKAN{;W~3?5_!xKgcg1{SvPcn)ho zBmkX5kMLlblu!);)z0obFx^)m%)1`n=CEXc{RufJdT3^TSb)K>#>U1Tyjpzp{iCS6 zBQ7a6+Z!#SW4^^U1^RK@htF49eAdTu$;^MZzjAYP(TP=1P}n>>BmuyBq*QFTji(kY}CwBdn9y-<+r|0pUmOT|T|?W?F!E}mbI?zJ*BAa8wh4Z2_t zd?jvOeg+dFhOc>i&D*)uTFANkt_V$dsBgB`$#s?0CUSnPou=iS-u*j<;fY4!sZb&4 zPK59+RI~`7+dwmaG{{l=3i}Q}JEZHU^Y$xyJ32Yp>*Y1+bx~*TRjUqy-pT~4h&kco zS#<1&n^QALi`mAwC$%;uI!aV($%Z4MhBo(ge&?M@N4hA^5_!V&{M_f)AdFYj#gmOD zqDRL+x>ZS!wfgliwws(&0AVpT0QVBKD+vMK>z4W zM^^yMu07A0;8)|Ey8ZTNbnLN6eIeHnvs+{Nm6hWQDfOM5OWcFuS%``x2u%DQBR%OM zzlco*m}VFN@doJ6gl0ZGNNSF_NzkW#)!7VM@W8G~vd^bj!LHXRT{>-UYei66c-=u% z$a&Ee;53OE@f7?NbVxeK2DA%FRd>&9x!L*(;Mk@WHC{5|%&v=>u$ zU;)2x`eDGhpESolm^gWPoMj$$^72>Km~f*%zO`*Q$5pAUwm4sQUFF9j+WVETB^rV1hQx<%f* zch94)-n_)3(f%9TE$=s>0P8D5aqk`_h32hsDo|?MnxXH{9PM-5?ayEujUkGnT zA?S9UdHrygPwBRNlZl$!&QApZPVo?re%A&8;K0v)b^6JOmjSbUuLo(-yAv$)ZmjfL zBV>`ImgUB6Hoc`X?kjP7O$irETo^8%*@UQw1ZN^cYs2D2Osz`><$-Dm7#sj;fqY{t zd>8e?(o!Z+m;)xPixt_5XM0{zvP)xo9xmDC_&TMqC2wR}W8xwD@w2l={YnB{eIfN! zTtTRR7NAQRS4T2?_E3?>02&7%u+;1w9QwfA0nWR$JRup$eVhSJ(pXCeb^}j7X%eC` zZ%nM%Y(j=g`_pBJR|xUTZgJsyKp>_2pocz@001nVB!h5^IaR!Vw<6#2_EWPN{a;86 zGmixbZn50&exTCf3H1E@=k1*U5aC74#d&;x0jBuGD_+pZ%F1cdcdSHZK7Rh(-hSha z52XllT%%PAgOfhf?py~6zfm(>B&ZU@N)-1KwnYOMVMT~hawGtG%lP#5Wb^# z4Ul_3q5}ctQii{$=*0Ea^RB4crTh7Fug}+@;Ny%}K zdmI2Ety<_mTI+ls89R>j$Uyv@E3dPsW`G2sW02cw!DA6mY#ErL+{}hIH%z~O)>g0ib z8P;1%_wo!O8$8N-QedM=|E+LGEXF`!q_>NZd1Nv%`7HldGwyFGMsRK()@6wzmAdR} zMQncQ!ey+$4hH3)E|!svj2@FnoO680pNp_ry5`dvv)Kr~&9ElgQKi)rxU5EzKa8Cl zLUg7w=5+xkxtJpyKWGK_wT@i5YxbH1{^>nVhcE##kkx)$#PX*WW!!<~<>y+SW$yN! zYd)E_)j0YFC9MSuR0p@Nb5aR4eyiepM5_&>!eo`;oTho zP$ARH53$pC98kmoM%@+8+Z-f`=bgj_MJK8OqoZ{5V z_I(}QHoq?xy;tmkyd|ncdNTn%cIR!F!H7Te$D#17U9?|hvhF7Z(9EAaT6Ftu9&DS< z`%OXwmm(5s6ix0u4&O!AEUDZ7pb$(z5v^829TX>5E?53PFdR?R%cBOFUUXmo^cJa7 z%=O`cTJ~dmRCtM{xnJ+R++Djg%P0>I24hY7md6u4JXLSPL*#zCKqD%;QSC3syW#Hn zJc{W-PK-4b?-b_^u??25F}ZoL@`L!S^5C5;%8`LDlE_DnNLm7yGcw8M_m>zBF6pSq?k}S_guKv(Y`TbvE{WzcuVb&GDhcbwvzW zyi%lYuViV!=e438wb0x@bPfLs=t_XgmKf~)C6V0&7fmXJQfKQC=xXnJ^?le;Zl zyh|N4)9)P)_yitgh-Ogm3Wm8v4Zb=|jTQarWuM_(Ao(-X3)}os>y^gQp1j`9ys$-3 z`QtIN@SONiZ@QD`Zt~e-?hE@RE9l>*rz&RQOY#xVe?xx+%@0{j5wzGpJc?ka0z5e} zL~q>(jibS6IZCl;KVU{z8w)`PLppOV z=OJ31B()bzBRYQuldQErjXUg#$}HAJ1d0fLXQvyPdt+XTE1E9x2SK`{e_;p{&ham+ z1gITiHIf&K2+bYpn<{4vb?y~P##MLxKR}Ces8F~dx$I(fzp~O8G!^0fr>RJ-zdgkJ zP!5eau9&m~AQ%dNr{4~EH%>(X;IXP6Lx=ey1oVY5<$m{9D6ezUek`TW&uSe8d&#Zh z9;TW+Ru=QghH#?4MDjWL=K;@IGgrpLHbi;CU40p& zGtD)k3)dopd~RQP5Jy4ujn)oSM`;2B^gK$GA_~-b0P`&dja!0do&a3M9vJ)}2dpFt zYNZGf4Cf8ZIWLkc;Ap7BUHoF}z;|aLE=FNoASF}Q#%*csu9i`G`bjowBgP*T(;0x% z)9q)qH>Z5eHly4{mf!%Yi1%GTaC1L>qR`ZnR5$BYLih|)qQmO#)Yx>LC&9iYu~(TE|=&gB1v12L#i^JV0cQ}&_Y@T2B11SZO2H68Jx@`H+&=*O8Zs0~ni7BnFCSCMSmh00)1mYl- zl*EZCXZUg8DxP6fWhbe$n_`tZo6R`9Pj@MT)&_Wi210Owy5x%?2j=^?6g;?5C`RKzpYcQ}b5+;976|gyEM03Ev2DA}RQ5Noa%uWPE!H52c_hA*PPd7DC~5D(l$Qm$pPpee|Gp2x1cw=VgKaj%u6t z@FSzvNXE+=_VKQ&e`x?s&47IlU;&AM9DsF{g7yxkeAN6n&8ZRibqjj*J31`s0N`~< zgV+6;G6R6q&@v#v`ZzidH1rN&sO6w(0)M9e@z1@72KO;r4_^!+!Lrm(P?Po#B}7v8 z&E59TLfZrP@q;k4;|^>B3tXwG`Sitojx3EMK%@{~J}($Yxh#7&#T5YT?=0Y#1#{52 zGu$!>#t%U)xfT}m0kEr1w%{XsKEYG<_>-EWDdBG)0?5dcm+>Lo4iLtuD1*&SFll2K zV%HB@_xS9+tJ|5E`pO}^_d^p&55=$We#+I{eo9@<(n4xmb;Q6$%AHs8m>Q&mInpp_ z9j#$PUjVEtM$CZ(V3R1s^~>Nlk)q9&i0V^J50|Zi{VA+D%z9ohxdI7boc*Z3eE$Jn zJOV}^kop7?(3Ydkj47eSV>d5av;tQG+Xa2PKo3)V&8Q8)md0TgNgyU7AA?f&lQeR1B{2A#AAqkl9&dbOod zj<3#I&Lz18(`G)DE1-O=k$)F98gv`d+J!DXf3zD&T19}!%@)FOXZxz1oajJMX>c6? zT;QUB!mbLb7zpv#fh>e7Cft0MikUCf-gu(E$mgpjn{y{$%4Js|BX|7zdu`|Pgz2g3 zAdzf-WMa;hV>t7;X)gx_ApNh2!n7>Eh;mpoYNIXgZ<}CX2T}xp9j7VqmqbhWCn5(! zEKE~KJA@bE7Je^8@K%0Wu!Rz0?S7n?HhZO^&Gwoe(nO0`300n!*f9R4fPm}Esgq1jq;O0mAx-{RD^dqo7SD}04S zkD{AD$#K8lVBwxdJxig53|bd3Xc%nIq9t=nOYcf{UBkn(r_@o;w~Q~#B5Ynhe6U!} z!w$o@>wt&~4UN=-{7G<;C4GGe2WJ@?6L1In+SWmy*Z8Ri?Ix?1IwpH479doJ%x1t7 z5Cymcy-41adzdql{+=@J(HPZS%Ipe}@fOH{WI=#-3NV%J&mhnk{&fAzHCnV4^l0Zq z)rb#52C;{S&qPwLrL1ERdmXyn|);yR{GPk9pVMOa!)H6krCOY5-_@nz}%71j@e_Ol$@ff*dZ|?RHHv$mH3jafIBuUxc2jX5fCV2X?FvM^vpU{+t!?nO%CHUV|9pZ1c6beEirUL_N$XwA$7w2kjz+!@4Fdr^3uiyPpC$q8o z)}w-(``RkxKZwb;>48jDH$X@L(D$lmRl-t%2n#)1H+9RCo!YdAAyULW&wl)!XLs{A z6Wjh+*8v-xd68#6-={In71e!re#G$`%@an0 zSr)69z|FhO34sF$Aq<)h*H~<9z$?Nz0V=b`m5^!!P0i$C_o13QA%dA-PU;wad?O!0 zwfrQ-BPOxJX=~ToGXP-J^m~R~IC+kRDE7RKI8Dk2l^NadcN+sNJrDtXC(WtmzAI|H zk~ez)NAh!x5+3aurO%r?@_Fh)KuS7}L4*|#5a|*OiXsUBL)ikbP(d|GoyZCfuz(S( z;-b|b_C;hhEa!x|T>}CT|999V{CH3deplC}nHu~hKq-!md!tCA6IASMY%0$I6MA?;r19ZUt?eeIf#=qgX*1FP}MCN{~ZKin3dTL`PI6(0n0w3!L_^v7@=^zoisBi0zUy=FS#u*Ec zJ*W+IG4!qdlnZ1dAK~s(m`C%UEO^-sBFl#4{mZ-Q#8hcOZ)<-yl}|!7 zDE7$D?_?eQ!gZKqGSlb9b#w9I;pvJ|a*e?1SmUp$W52@QW^*T; zf5|UK>aN#I9Tzj-tniOaiZrEr@;<`Vh-SInhKKGSC@Sh*rO-P{^411ry7B@AjsiDN zC~+Ve!@Bq;%g5_G=D(XhXVd(injlas)_h1@x7ZZ6A9u^U#!Y;?*>xg?U$(2KCxq!? z0k#HWBymy4RsA5(C-mk@OO%V93;s(rf$1^_c|^lz%y%v>p%d(g^y}>uE+g(_F3Tw~ zmhQc3`5dyr)gn}sj|n2B^2DrRKl-a21#JD=!li9X);drE*1H*~XIXLrv?GGO2Xpnu2D{uDh(H?RJT9wV|Et#%? z1CB$Pu^Hx$-Pz9*-i9Zg+C~LK{5K*EK07^b;e98}Y|Z7YY}Mv!+ShYUu8(!vYtGZ^ zD`O}9#0$D4@88{SZkZKAhogLc@1~*P_w`|<$Ji!|;x{!GZCp(1jFI{NN3_OaiZZco zFdhwor+8(lOrg3egiITGv9WYaRfJtGSJd#u*LK-~W<*c5ZSTd)7M@R$DcyNKIN(nu z)*9!3vNh{p;I3b9m~|*El*r1o`g-u&YO<4IgFC;9QK(+F3VHUhY&jzvbuQE5@c{OI z+gQR`W>oQNLAzNW^tgKi5KEW)N7q6h3gMAY)Av{clBXd*=yWHpF*NV+1@^A zqKO@qnc3D*n6SdYxXQth5)cd+RUyGUR|t?X4qih2)ni_FjjIvp>#Ya3R@RHP4pc8y z2xWNq72N8-rLiuf{cJ4%6j{{UpBZ_3>OHc?e#NUAiA;yyY~r(OAtjUl^a8vY-tC|d zi$pMxvI#7Dk*;+#UL)4%?qY7NHIWI#W_=jfq&OPqtKS+bkBFw8D4~imeUVX-nihc9 z{`#pQU=DtBPBx8CzJQDzcuR=eJ%s_m3Wo@dKKt?}Y`Z1UD;|sciaH!~TZT9L0}Ddp z)EWAndCIx;ryQA1HgMFh9knYAM3hrUa3i(QG0{A(!J}~CC{?Kc$I`_solr|!xw-Xa z(t58AFFqMe%na|$$v2D$D6O-*>BIV`uI_npuuTzaHC)b=X(!G6)Uvv$AMs*frD5~O z&JI;v8NgsQmF5kuiFx=YOD6VMRWEkIr__j$fMpze3#J(07ND1RtROc!*Z%{tPQxXGhnFX@8`@(qi7oYE?FVCdkk5k4ZJjL zCW#qD>(#-kw3Y_X!$rDXn7tm{kIejr;E-LWF8Lle-$2hsV1{uG@#v|sMfq0-DU2wE z$S$?Ar{Zz{EEo4dQ>Kp!A*YsKhQv%OphqkVTExT2d_ zyi>$jjq<{y0E3Rnl-MRe%%qlK39yT$OIvVAT}#=v@N`VxT3?pHrT(oT;pJx0sHn|h zQO8lvoF@RUuv->MgYW%HavkcqKV3h)!h0*|_%hB0pV(I(&)4O+;F~(vz^|PV>a*bu zCp}&@CoNmRm?Tp8#0Ta=3r_=?nj#k-%t{>;BSTyoW7%Y}E7hO09@j25${51*NiG<= ze6o~Q296Xi(cXyw!E=@UB%h&tuRCgpnnX+_lOEjkP6pN{pYf=bfi=!#cTMFj|H}WY zRzh1uqzOOa`?)kfrIA*eA_>fWK)zi3O9F&hfwM3%sqzwvN?>lU%x{T_oB z1mX8I5T|O#Iv|=a3Vv-sdV?@Ok=em@i0*&9%gos2Js-i__0Lr^#+)$(Z z=QmM<-WnlmDMfzE{)vXV+31kvt)8}OJ(i&`_PZOZz!*sWpEeZA8aMN9R;vw@jY4c_uB4vh^QDKZK+K=Cc=obEQJ| zaQ11N4SJ}aM=DPcPoSyIfT@UCm*@x1QnhEecSNNdt1^OJ zb2a7*+EJdrjhEM}qeFC&dw9gqVWA@~xYhQ4VKqP9_i?Di_4O1%z4_iaTcnG6@#ACM zSu8+3{*A&>N*taYwpyMuHm z2f9p`k~x@Oma{ntfV@wi292Z(jxU}9?p#AJ=D?xdx+_I*@`&;H4MuK@IU8KlMCv=b zw_5lcooYE|G1JycYn#J4H_#eM1}Sd zC>eJ`u^@q|xIEu1tIf;1rAuAf3fQIfj@8#%XTA_;!DmA6(*$d?IVduJiGEa)gq98v zQ)gg~+xGq@E*#|uMTaC9&rOb|_mDK~)$ReJ&Q~hi69f1;4>_@wRn67>mdVAn^romj zoOvUU+>!~(1yXKSYT7buMf-w1moH{WWT5Wny!VIY^W)8tqv>}ANKO6;L9~ItFzJZe z+MGXSsY~?{V;o#E#umScmJ#f&ddTy-tAy(G!RPh+c*K2~;c3x4=y!z^{KSnV$L^*mK}_DB$HY zwOiG@mm5(ky=Y`snSj;OJ!Q}>#=q5ngc5p@Uy0%Xxg!J)7|T5?xY$aXjobw5v$6!s zG+Jt?bx{iaN(DBV>=D&+w=%eYPyHiV+;;@g2ID&0iQxyu(#^d<-8N-4@hJO!t@@;~ zk_O16&M?+=?|wcO1=G-~641U1jXES;<6UyV=j=N}k1(x|Fn zEKES514h#Nf@~m{Du-^5;=-2)F7#p#me9!-sLH)>wutrUq)3^8HI@kzDtxHH4sGht z;xGjA)&YCS%W=Q+*y%njxPUKIob(3#kGKsSZkDwyfJwq{irxfmo6mD-fDzMeV8Z_H zmz>L!d!bmP?B_JDX35uBQgJ47AdCP{3k9?s0e%-h2nmvy>}4Sf1AKxo4CDLaZ8W5S zwDubEzTtP^3(e~<=2cHjLNScfSu~iJ!LiYn2df|2gge5@x@@ooo!&j&jzJ$G-E)-C z)>E8YAJ4z}VDR|r@^B=AbOi6mD&YFA)7`lqAiQo~`rr&BMdniYY+eF5WV7E*vozXr z&75zC_{}Asie^Pc#Wk>cYsfy)>!N;buV}TKgCVC9IAV}ygEZ$m4B0&%JF!f5O<6@i z@$#Q|R{EE1(cB|g2;XV;&%N_H(1}By1p^BsMBv|#pGHG@35d8%DN*j<7Ak`hwqx4< zo?ouhix)eh_WKjl)BWyWam;MCIJY+oS6yEj7#iBaAH3eE$`qDieEv4K0IjE>LsAz{Pg}H$#U>0=SzJQ0+Egd7$j8aK;H+Xd|(K5CDeg9maa& zLeNl1+JQ$5Vxa&G+0TFmQ9$41dEBEn`I7u>xc{X0_h0QiCLk=_kZ2(CrK*a@WK?pJ zygD3xMmD#uE`^DWoIH+thL9NAIw#wb{UZ=%VCj>IMHS4Me;gFR&t8Kb3Y`~Wj1M5W0sYtNLt}nQRHTVX5rjF;GFmLClnr zhJf|f8zQ|C1~x8-C>-&>irI%4t4qv&{}uRDO*gg>hH!^l`Ll)sFxYlXX1U`iBJCFm zvD8;;8FYy%r}Dl!3q+8|xq$=9*E zdal5rzdLDKs!MqfXpK0+!4Lo4tt(`Pd|X{zNHoLqMZHe`O$5-Y$;9^YNp9wPWC0W> zplcwr=n;jibEk4e6t?h@hRW~X0>LF5dNuI#LvJ%Fe!~UH4O(zCz^(u;RrZQ6u;_bQ z4N3Yt>^DEEBPc%QCYKjW7fORe2%2p)#3f0o)+1{W8*K)h@Wad zZnF@-qoN)4@Ohwh=t5tW86+3mp;&b}OrfyTueZO3cQp52K6hIg$aAXF^0P_h5LtLS z9z?j1(s#n^!)4iQSJoN}qysq3ICVsyfhu!Iy+cnSW z;;+qtAet7UPk=sGGVU#!E$o@DfFZc1si|6A{MOdi&A_~h-U9UlUuPtzOS+riRS2VN z>LW9SRD39!1o6XNpmkVv@J1p%3KE)7!hyEXJmq$FdAq!I(pwEU z>Mr1@Wskry)4{$z)?5+|FgiVg)D7_mR1Z@;_QZ8KA#1|3Nfz~u4N0{O-6-F3W#DmuR#-or~YY86r_(*P7O z2m?ETxUOwp+gt$ChctTkIf0ihdg5)Oo!Jv_>YkvD#6We8MFDdg2y5rNUfox!BLQqtf&dOS${TdBT!0Ffe#`QGW{kVjBFQG<;?5v?UM!rI z1}2{|`iI5^l_?1>`hKA_pc z1aEo|KiHd>PESg*v6v)%s;%r*fD@(}LE2du27)$9qGJV_m+5Tsz$XaLsY3;Pfb{uY z4YA&z!$R7IB~E#rFx1NomEGq z_=;D*s_drhCg6(M>c0(ZUhtaGNKfKJDqN<^_M!Q}jT&u0`OIJw>-M3RGT}NiTW{FW z(pcw_s5bCrPjGp{v5xYPko==h|F!xf$!)?7l_E?PPtiZ7L66gF~=km z8#XCdjzWJAxz=u)r&jT5K2D6K&mcH{hJ6Ea$PIhK?%~}m0<7qrA4?H z^dN8wS~ne=ZkfrKREe95sB8BDHz&pZpUCrz+U@hH^GlIOA}Rb4{Nh3UF;pk&$VmRd z|DtAB74D6qU(EgMM=ncMqmX`plNfk5QQe#^Mn#Yg;H^L+@UB3_^9Tdbh;L<6%#;`>LqeF{e)<;TMC9nW5L3wtFW3F#T> zzxFVYU3{GYL>tr={>jiQsNF2m4lFo;Z-kzKdA??iFN16Ofh6R~EQ250fN*??jdxz; zqd;-D8(5QWE^yWAF6sceytpP-NsUYyG@H-a)2q1WR3(wIL zYeY?3A4xR0pcZ>Q1vH|MG3Sw55C>!TufA1~v9qwW);P^sH~I(HGJ)6WhWD2k*1VLI ze4CJ?oW{THD1NXLF}^2 zV5YWSp|*iSKlRPyWjXPkNpr(PP373w2Lk??I=`8RC^*IubuKEQ9xq@rf&~@? zqR+o@lbxU=`7{3OWChyby0QogC>IR8TDo|ufZ%TBtF@02*oj>!QrWHGqaQ)<2_LXng4wipjWI*eG_UnDHxd%Zel^O#42aDrvW)96Nt)R5e4h4(9b@Dwj*cc5ethzN z4v6=a(|{BwH#crlV&cNhS=7z5#)9C-V&LFyVEw=h?SpT)mGHZ0vk*OIW?nVzkc$+b z;&q-)%&8ht!dLG9-D4|`2Mpf^+WFo*;=Y*p4_vT4w?UP1;V`N{C(rU@s}!pkQuYeA z@aH6YAeMQd{Y+94^D(Ne?u)>#&U5^-Sm{>C(>yZKB%n~Vgms0n1CQ!v=DWav{TGVe z*`|9pe?pTVcslu=TQ3AJ{T)0Ze)gFFt1>QI0T0b zyxIp5QoPWT5@cOKj+&C)C#H#&CyWXSDJ}ZP#wxxP_mQ@C?|Ul{EX;&F7>pmve59_9 zGK@5xa8L4L@8cchIF4ou8b-M8%ru>_@e#!1%b#}2>If``4x&bhz7JU3_ zU+Ugi15l0kUpL>zrTYC*E@}bHP@f`(d>}uyv#XvVO5x#r&gDHOj`ifnYDw`xUw+HU zkpZq1s1rgiy{&XJ2uIi1eI%V=_eOi3WW19wieJ^Zz`*cACjq7a%0+2vce+RuAs7`v zdFCoKv@tu6Tb`*&mE696h5Jc{G+27%?dgOxGYbs83$|y#WC!MuLjQpiS z7=L0y() z_(bvkeEx3B!Fo8z^a26ejc+kz{l=HhmHn0k=z`eTuLM8Z1cu8c01QrMs?h6czX!%f z()US$+TxS3+L*ArNk~ogjbWDO+>DNkq>8^QN$aJdEZy`tUBeafR{fI7&i(K2Sh2FM zc(8)v4-3~Kr?7N$mO$6>k$y^z)f$D5{(%k%qmtG&4`lh+=WYDPw!oPPx;&D^3{@nY%RO&=fRvNR+NL?{R{w@Kr($naze zE?ng{d{DbD?T~CH(p&ds#KaQ#0xkG(e zqOa^um>Au}%#Y6a_<3_7DdX`7s=I|XF23?0l*t0IM0$Fhl%gV3wuGY9(0HP0HdDZ8 zJa1=nIo2AUR6feqGi`pg9Z_MO7p{ttlfVAHKrc_mboL;@cOYqiXzsndw4E5GX;$=t zc_Il@Ao^eavT#%@l}m9NvkW@SCcC4Oz5zC}x6H(L8;xo6-1W)G-Z8accnA5h9ppQ| ze9Wq3i0Cdjv1>Csz7QD@^K=$s zLMb~{BgWO7Xi?UqzQYL*RvqbP3GH;{^3uyi0v~<$@gOF&v9!mU+6Am%lfIPp&OGjV z$f1ldkaG&JTm7MFkj}~RACg({ng(&P(*Y|SmG6n`%p8=JP2UZtl_}+Z<9-b!iL#0P znza%YrJLv$TEumD%T2NeWOdV{CxVb0tKg6oUP0k;Z(p_4kmn?#9qh@#clv9YSkbYCgnlh0n-W{b5_Ryn1th?Xu$}{A?>@;A|n=;q0LNUM*sa z(z|Gf)_3%X@WrEQNt%dmlegdO9XoJSecNj`s1{Z#-|Q`02k|_O}al! z{PY)LBDvuRsmd&kz))`5k!R0p;y}<}+8-cwFq3}$PcOh$FU*dNESBYV7;tAgVu^O? zcY0!MZCyO&cKb1&`|a4GvU226B~FOnM#~vIKBX2I5eJ<5rB;0jy&X15{1!xU$=KJW zSbmC=ufz;B%~Q!T>qisQ(a<7T?~ih^Ty=~cT(-N)LAexHZkRl=AXN1SeIV}p(-Kf( z2dc8#b~F{lq8w@(bseypUL>v!2$X$ux~OS_Z>0t7?(dX++DD-3?l*P!M383V$?L8w zYuESxDvEO6ep}>zs{8FMOERn1I-)CKozo3%O`v%( zi@!PzaAeeNXI5jCRgB>N4z{OX>|#PYD0Yr7P#;(S6)+$s{7g#+X7l1=bu9h8eT+PB zZQ9}X{o`fqzt%!hT|mTP?j;O-#(gFvy(R!;6v07eLb=?6-A<$ zwe9F+;dT=Tlihse#OZ28#@9?f&tRw4WbqVpwz~>3`#`lh`1|!cVxZ1Pq+0mdCKNKa zKXfenXal!^3znQ^N@KBl4cq)OH1AIqKZfoX=Q)m}Z9IxXZFh-N5DWwT3?Rc42xW25 zgVMv>K^kH4(De$>13yahc+=8RKpy7mQL5MS{7Z>>(W`3{Bd^c+HPt+(wo)`rVSF;V z-zI|%4X0mjzkSl^acz>;UZL6KAu6?AYGhCOk9IzgaNMely~d(jEvW)JA2@hWybs08 zTV|LbuXHzVGRx(Jk#(?xZXg7NHw|sI3lz1$0u>=RhzlTfRqmbm1 zbaY)1k%zt02N=hqP3uJJN{&)~5}n`U#COij_;B_E+@GW~v2A-yUJ{Bm6;U~{6n~CQ zcy!LW+Ksm1@kcs9-N}bndfV8!pjhG`Uz!T*sSZbQz?aifnZom!jy48RNJ0w`xq-5^ zP<^S@mN6YS4Kqn0S}|Bs*z%q-B}k;ROR_~b2D3imiO=5lHv@SaR$))w@f zn`MTQ)!TU#9~<$B(CMVe4?oIQQ~<9W1aGf1f6v3R5f=)aEO+^0+W0L|OvZR4dt8{l z`k+P4^JBPwPHh13r}w)FgOhF@6$NZJmo&|;%lL~yjA#qJj}5hZf4;EoTF#`d%CL1} zX-GQN&?xqZVv+hfcZ9I{GUMZ9^&c(%OkTg73@h;(wm!3{P^J}xt`Pr@j_x2B2K2as z-F%k=Dxw_G@uy+Wi13dVvY42MN)TBEYIFf`u`mo^kSsxC0p0>UR6E8fhV22cQq{~T z^I_9pHNf^?HF6CxpnocdeugqnpsW@~x>iGdG2=~Q7KZI@1 zPhuCAWNFL)oXg9CLmkJXQc!`#Rs#8MsIG0dBLQ^EW2k!q>fi#tfEp-v;)1R~Oi(MK z29@eY_vg<*wl0)9O*{fz9HHuHhD=}|?;Z=}h)ff{{fzsn4|E1+gyf1tfsYPc0rwSU718VRF?35YY0>>j%kEg7dqagu#R-Kvx~G;eBZPjs zlwz_nE=6-?tBaxm8Gt&e^i>C&n3wRlD{cq;~^3Qo3vihB(`a=ar#;2MLSG%uk zy%QfA?Gta?i={@J9lGBQv3Si9(ort7BYCLi-WRYu+0)arf+$o_pnA8dYVt*>r#@lx zIaSL>xOtJqJyn4z5q?(}RdAYi^6puR7#>;kF3&_X7khSu%;LJjceD( z+!pLLXkA|o2(s(zm;5s5S+<9|^mL)X{ZFxteL~@6goFLB$4dG&=yOhs)UK6AquGuf zZI^7uBP39yp#vPuFhm&&?f^x;KwuEy;4^9Toe0{luREZ9n@bGnq28#kzOCSr7(S)( zVw1Aa^6q4m8qy_2+Ctb9Q^f_L7vKRAnf4FUCHvP zcPP?LMiS-O#gi4bLKTQ&CG011JsAaOh;w~PuacAcA&ci>{f$%D4Mao(u0QJ!{N3() z!DC~@rZ)DQ*{2rNH6$_S?Z~J2qjITG;R}~NV&EAG#T+AZRh%Ff*bj{LbYgQC)g3E9S&_0+z!03RKLiKyK4Fe48n!uMs*Ks;Mntr&@mVGA= z&~^c5o zrG^HnkI>PsAdu!+Vl!I8}q>fmiqrF-@0yE)Tv4la|NGJh`w{C>}D?mJJFo z8Gh~l<~`!{LEq$}x(+t5Z|e5fe)u z7N-|iv6B0mZ{Yf!0nKxk;#Pr8s@h_CS8EPg#!y1b>o+Emv2Ko44BuCrwKd8a#Yp{X z9Z&ara&pEZhekBg{A^g+CVFyIV_RM$nujc+QDwRg$_?HdhxY?5usu8!gTZ*#zJDvt zuVa~SC^?j|c;{n!>9)2uRkcXlfViQRF5T&K@rg~h_Nsy^ikZD3CEUGpwKrdrI5WzR z#R&+miKKn|EU9jhSh&v|$>KfFOxWqg`p45nrwLq>O%t`VBPIRRs4fk|>3l?M<=k0x zB0~d=Ge0MBaU^5Xok-BJ7y|P5D^6brjjt&?^7USeP1krQradrF=))%2{-&uap>8DK zH^><@(Z04BKzTO3ZWw_b{L!41Bu+fl`dUC-##U@c)q-8+7DKyj8oa{8n0(uN{J{fZ zpUaL02iqQILV7#&mZlb!lF`(t8AIRE72X0;xYRcB!pp~1Ppb=hK=1XDorhacNJLar zRDwvlxQ?13aY>Q7td*3#XHw6bYFJ7`sA5t19aB+fLO9q2a%L3ZmuP}!!`OQ=vFvtVn7<9nZ$ zZ-wL1ukxMdjdG{k@>0%HAD%-nPqv$%F#VTAHKADUwq<* zQPQzB44jPnn&_SBC&4T7xBL@m0?~qgGtflcCdcec84A#6=!`rcn1GTt7sQ}QQvkU?U;0}}(%KbF@ zsj*r#iTY0nfFhKOyJ2J`L`WX$I?lzx$~{GlT>gXwIJKaFNd~15L4&9>sPJwFK!qY_ z$p?2hE#?qJPI}6FRp~ZkDaronJQ88qm+bbERpp=C7x`aRF|w^Miaoe#bQloR%2rHK ztMF7PH`XY>-~Xk-(J#sG-E!>z!`NGZRkd|(pqK~-C7{wJxG5C{X`~xbQc7B+1?f;k z=~PNeHmL|mHz?gHUDDm%cPx(QeCPZB|33Hb=TVWp_gZs~IpQ7f7;~=LiIPiF@_njB zVM?z@ikv9&Zwy#SzVzPBBk*Y95>{B!aFm|^vH}h4%B{K9D3oAX;8ro+#n4FC9tFp@ zTc4j*@*e@TQxt^9kX#79wC{qnhLjF zaw^pJ4c96XruVO#Th*Oapy|gFx|N{cLWlgN6vLjH)0PdKGx0|zMSl9LJx{#eJm7Q2 zlXT5w#6$KzOqi{zXNuNAZMp*33tM&H6yq7gvZmqn_8O@HeEdKHqVIyatEIW`*IXa@ z#IZRpwB}m$^x}uG5;3tI^VB|iPDh#iwFNYyQCr-D@oVt`2^0NDPi&dN69pU7K946) zY9d^R)m(7-P3(1+>{m87tMk)&EW%UN5oA@Ek{^WbfXUl%=hk}oPTtP9)V=vzDc*=c zW^OUGa`Mv)bCu0>#9?Oij3f?;lCk%IS8OIW2f~-fm|Oe9iG!H+cJR>*Sv!{TjN#*j zPITzlaf41UWOD^nkiPrvqM|RKdiz(%#PDc?4LYP0S(x+Z9H|A4FmH;yKEN?KOjF&i zaw}X9B5u%J{B)_nGncKQJef!^@#0qDWXrlnb!f_?i*vNp*I`MUc(3#E@$G0g5)>om zA;UdV6Wqaae7=!7iWFa8O2HeOpW~Rvw)^x`oQCj^3QqfTLLOd-Efdd7x0&(V{Qf8- zA9ux1=)&mf!zh+mWAR59fSVfSd@tDVY`SP;-uI--dh{k=(Kcpk3=ur!`ze#_@wY{H zQ}h~wC3##~Wt67$GoG*DG?UVSLf3J+b+0Qvhli~wU?`tj#KsuB`dBGORl!|w6Sq_N z)#J0$1e<;KeCs0kwC}3FwVU8QUFv&eo8TzbSfpm+)lh02VDxTzmL_*9R_tctZzbJG z4f$P%uJir7Q!P*1X2;dGgdfqb4B&?xa$kvbm6ectWnKY>6?T?D4wGWn& zS1mUnSlmjysNCyBI(M4`Mm-1%_2gy;F~$SSItPOG64Jzwl-^IV9g0gFyaR0_c_$Ll z;u+M-&6jku?ij;xDf_$cqr)+`p6Xe2`swq3g|<_#BL%%BM`FA&7Wu!Qsj9d?9q;@D znhHM#M+~sb`@}9vIh2<$s_9B0uHpfEk?yUr1Jc~xk$30TEmGUdQdC?Tm#1rg+LBQ8 zIUiI+SQ*tm@!;*lC!A|zfq_N={5n`aBJ&_)LFFA?c|qD9FX-%ymo;=z#F_t5aSCbw ztB*F3JfS}vCZ&rb;#gJ~(Zn`i@KdQE6)xOx97c zsfUJv;S}Z>WIe87dc#hdiI?RI+!Rhf3b35LFAp3`GP|&lo9UELS}84fd6zbm)z`0X z2~T!U?kfrTgXo{HXR;ny^v$WfbSq?jNJ+mnFfh2<9fB!~hxmeD4|Mz+vP2k-^5tSf z#fnG#_8gT02CPSux=621TBmjC4i|4-8!!n5Mqv<(qIUinOpZ=UMuy)j^gX&$ar?iMZg&ZYBenzNK}WH4q2Uv=^P-WhZ0 zu>zALC9cS&C@Lf&aw?Q+4ni*;^=RV~#eZO7?8*nsM$@6O)dSf|(v%hYyS ziM@mC>>@z-8021BU45I}9AIRx>2-TAVgO4`WOAvi)=uM;^G+BO&tYR69zn9_B*=>J zHbRP`WxnOlDaN%#x`WZ*JDCeKd8!Cyy38}ZXwSB)#;ftUV^|ZDqRu&>qe%K>I>7oS zDou1R_~+2&+&Z)S{ltWgyObBGzBRRjGg?PdXi*Xa(f24+ao$4qo--th^z|RUCHPj3 zwvp3BTr*nVOlk>;dw(^gOUgUVg+74JWDA4&*rbaspCHd0G5uwAO%Z|+xtdnag+@8X z3l|z?{1M+X1m4k_S2(c&!%yhO>ACUox7(o<2}8?O zaoN#kE6O{F-leBksN&7Yjruwtxqih1ce)Z?qtw61*_mWJ z|0+B*Xebf!=@!~QdHO9UcnDr}r>UDOa{asqq?!YCWJen!{Hnc z^LC_574n{pasJq3Y+8k5!osOW+a{ZI2hDTK7S-L(Q=AWn>K9XSEFrVmnO7uP z$=7oBkR#<9NX6Rt1F%><6OIb``4w5A?O+1?8FXMa zLX!H*WX?EO?Qs19g#T4gvezWA;pY1rKDxa%#a}wpGY0d1+myKGmj*;8>yp^+$?tYj z5O?kE2SjkZW_#hUWw-PkSZUFvRD_^KsQ)C1O&ZNa>94M3ysQYbcnz9^EOe0#DQ(HU z-(QjB_HxdS63kKsF1=rbF0#0z&M%D_yO_z%I2W!xc_4iI4h`&Pqqj}XUoVuF{R&3)?@ukN}i`%meEvHJpCZxs% z-Pl~f94+hZD35Hx%>2(oL%|@7=e5xu?CD@^57SB{;#PJk_-S|Q{G0DoCd46)wHckl z`k7+`^iY{P>CQX9Qq$xD^-ci*P3NG;*Tv-MHc}&u=2e6_8A9vnU&?XM8CS41?f%C@ zab}xzt(hZlDO^u_513)0@GACZjn-+qapIuL@`I{QzJ&B6Is|lhy~Krfjz*mwWi4n* zM4ZrVFzw&4(-A|ulLa+-{F#Uw=aZ{W3C3H2UR-Dhg=&lu`tdi`{aB3Qg4PFAMi)>B>AJq`rnXaD1COAM$Zs&V2-`d=OaPq3h;;>q@tIJp#xRn+D>IjGyuzt!z_ zGqUf15{2uzSdfw8t2MFAZ$=&#`}Oa(NN94n4h_tGF^^)8s9_%p(*5y|UL~E9eqm3X-ar)J(@_1ke zpzx>uHTUY~j+f1ztoNcS?-S#U9UA73dUr;y_&*u>o%NyGZrv-yt}>HuXZ{nlx;ig$ z2G?B@){IYY${!@$CqA!5+A%h%$hPv8Cfm|-Q~ltT4BoJX3D&Ta4jl`(s;_O--g)Jy zVfXFWq4<SMQdU?o~$a!k3td#!yx{T$v{7D<6&s+PJfs2j{4y(rvw|E zVfz7E5_KW9yfbS7X@JEF?BdSl(&Z9kE<^&0T7Z1y|c@ytA}%)b0%|3?0p_xsBQIW~02eONis%WKQ1FA)LTQGIU4 z^K#>k%xS*m>Hgl1K9X~SUsF7xRb>EucTDkrXqUH-Rrw@dS+528GPm0Mp02BX3v{YB zzh3u5tG~`ySMl4{9CCvZS58wzRmsTRXs#HYyT@DTZM+;?ai3#ryi{3gwcAH)^i?v^ zn7-I$b_cDsm7e<|b|VpYoxY6nl#Q%~ul5|Oj>B#T@Q@rn-aN(_U?O_|R+D;r5N>+0 z6CMRm{ux)bb}l~!r&Pdr9_ zad&Uw5#P~Pua48o!FIvY8r?zL44+W7fCA8R4(I-M=bf)p9^7j76IHi#AUGNetQoIL za2$JyYhJFWrXEV#JvL;>UR+J<3|l7{-e~??wXZY@jmzzr9AcG>?hq>xPlVJOjKfN< zOxA{7BgRsmUK^LPTkd*Xl4TIuHJ`P*^VlhUiPq)$f!FGGtJ4252uhAfv+=jnHoplK z*^G!ss}*0U+l(?gO)Xj0=)Ym+wAy|!?6AsMQXnbgn5Lva_`6}98_rNH_AQV7^*RM& z+y?CU`zywQRnCf--5;tXG%U;hgnPU>OYJnVM?UyY%$V)(UVx_G6SjHDyAm{fFP(9Y z_7sA;oI<@jM@9#j@QD8RTw4Q+n{if-vM23U5O%SxE-^-0RK{QQm3m#AcqGp} z?5t0qDX-DiPV!3J4<~BCfB8PY-bU$pd7yu12>)dd9hFh(-0#E4msEnc8$X>!24>>a z-(>f5`N*h7%+GNp>^@mV*0&eu`7~^!GPg;Dq1f|T`Cl$ilEdn-Uj~HU;eUEB+~45CGYSM!IXTFcP> zsSGamfdf91J?(R4J%iYAaL4KI6#rV3%n$QDW@BAq7cJ!zPqTh)sw~bhanshkl$4l; zwOp1}UsCZD{F&2XKQppL`1R1gChhH~UN-nTjJ$gsEqPA!rLa%y5cf~JxGQ%in}&3E zd{8p`!j5OkUQXUcw%GpA{I>;BE2pdfj3s;gNqOU|`Mb&YeWBn~B9zaoIf;9bWU#v- zn~4|B{u7`D6d3Mk7Hcm_>}DllXt}8O#Gihu`nk&fJF(Wpij_iBho7@1Om zd7rYRW7PZoP2WcMY^z<%XjP?9#p_Y*moEKpp*t?!8>(dt2)}8<%KgGFDdwvCqx|vt zd7pfW;i*8hhE_^)^uBfHts0RE!Y}5f&onFSHroZAl6L`j!Hih`HKRn{ME-Agn#b)ZD)ZN>DSL3jA5JJ}Jj!aE?QVTUos6G3YHTacNTV>9?o%PeAwoHrtT z41|NuL0&EGOWWk-?^jdiQl^APvpjN3*| z+D*4FTiWYiPcOAEQ_gVitIn`M)Qb08t!*U5 zU#t2b*ac(00!HbBBDAyLyyUc)oLY(smFEzKCpcW+-?Q2HX7PYDRX1;eF}H6Y&i)8i z!D7i^nh#3(#;oRZ_`rp|m6(jgkMV_#>fzO&q*pLqkGc$V#>1RTeQl&+ zC^tp%a|R9`xaffJr_}s^7zem6ccRplt0z` z53268%S{l$@OV<013q31rD_a>-=bd&ONtp9a1WwYjEz_jSQlYablcE<=q!HKhoFh(3D~H_03*% z(D1t)T6#TGA~R=Xe{Xke;v;P;d8mcLNfEgp&EmKxHf#BF7k;J63Pd58iO#w~YMtie2rhvuQR-IT*5yYpy^fQd&Ubt&5@tV;z?Y2qgRYL6? z+@2ARMWJlFW}VdOvX_Tpy*oMQdWLz-?=z)wTNy6sn~xKzD}4FExaH&>KsIw&1v@zY zI}&O#qPEq@$3;Ty5#eTRSm=t0d3nRS3>O%`-95_XXhD|n-n4bL!~*U|&2d9v zTv#J9DWqJ=Mw{Wc#viYY-S@FnT+~)~9OvQdvpXGY!tjlw;$pJi+O3FU^(NMBHt}o5n){`gUJXgSkG~Tk@P*x53_r{A0*Z!B<5rzQD7)2IXGr#2)TF1OA1FYx4 zEjlk7lfIw+iY3^@#5SoMN!(~1r~WcG75Xee1C8CM>&^{~jYUI@dU6Vq+h%!;sL%|* zcoBWap{K}-RdW-MlSSH$uh-5!M2=}N_?Mm&hHTb`AsTfwIz<)c?G4b9C7k1fjmgi0 z(-4i_8m~OPSW)*Y$nK~2IlE;qUKIwsEfobsAk_oJD`-U#l^EJ(>uXQJWhqtW;NE5S zQomh}5k}VQZS{*&mNM_l}n{IQfF-`gn`%a}9d*=||7#IC439=}}=6S`{K8*d=!5!%Qf# zEmGhTvkM)!4Du!)idAR+ zmd%L@ExosOxA-|bl6Y)x&zYE2J!j?qn#ZlKuoXQBhsU`&xk||QWg2mqeJurWj6Pie z=JpP>^CGxIv^qGAyDXV67Z;2}P{l+p)gD$I4i2Y&d8aj$?BvUt}-<4J4 zvd+aYT;86~wDIp=!mlhVE3^B#wwn`^I8=*z9{G@3?~_Sve9O;7qSAZdi4ix&dThYC zxRth_-xfps%0(vYT}o3pcV!&<LsnjL6ARE4lxe{^eBR$))y)9hXJ&9?yoK zvxpA}7~+{}(?ZeES6kvo#*O*e?;aIdbJUuV-y;;nOzsq(oEfG#IUof#O#aQ! ze8UeJ;ErGQyL@3}m~h@&TUEsfKcu5y(U-Uzw>BxqhLGi}!tZw>*7BS=^9(1)r9Axy z`MeK73)Vzb2x>AAD}TAp`i;uJc?^}?9H+IPM*+LJK&vkPqgCIZDnvmu@5RI~%98?5 zILE8+SKZ-kQ;>Bu|ANkfqTv4!jGvmPH=FxAhfH<##2QF24NB_XZ?M1*Nt;P3-VOdT z#V0Q0UlTE-L-}7NS`#c>nmCPk8czNqtChxG)52EXo=7|H#2Kc)_;P#X zJTK!CK^A=?C`sy5R8#(21P6XQ!BpYoUdVZ2W8U`#krT#%PKj-8-4t?1yVkzDc}NIE zTBa=@Y{W-S2ukN4N$7!%nbAfJ*XMk=^v+e7#Bh{S%(6Oor)&tTFo{&}{K}E1%>uo-~bYE?Lv-{|+VPlYKp~TMza) z8qMV*M~lIJenui{#RTYu41p7lkVC_uq+Yb-G~pq==TrA9yMwsGPV;stMFeZHU&=Sz zsPDb_gyWq=PDei_O3)5Tg&0UCH!i~I9y6rO9`Wv_$foqV1?vfpes!{*onKM}C^f^D zygkpjSI4AWi~ST@$eeiT*4~_lpc7bL%|&bT)>&?`=28amd+JY%8n;D|O>&U+en6rX1IOh^LDmo&dYN?xLi-@}?`ssy=ionm`xO3YM_`apY^iA&n zr6Q;Qco$0C7Y-;?t6lj(u}(1^xca2w5bs73BND81?p$2u>x`t2-GvsC8&n<=d_*#7 zOMBgX_UW^Tm8krUIdL!CZ$+Van--(nhloO2-1R{h%`?<6KP-`j7l{$CY!$sy%CNFM z2Z!ANIQ%<@6`=g}KvltP*>|bEjn%Q@ao6LAk&N;RMxSc4O3k91y@mo?tPoiu|LMv3 znur#21KkGg{yRN~p-0i86dqEoe?#l2J_8a6?Yu9yg<!&KdWsK&J?@GapQTxGLEn&;IpEfj z7EYv_b?le$pC+rhS04(jmU1P{nP9UCrt_T;vt6#!JciiY@Waz=992uw;-bAicF=|M&ThPr&TczIW(Ot`F!##nVD>JbbMg(y z*0Ica^3P_ZuzcUWVe(-A9ABHlQqC^#zYW#dc4dQDhC82eSO82E`kl34W5D~W8-n#R zgiy3%feHgsAVAvO;CMge@DS((^Fk_vogzq&eTLl!Se3+=l?8QY7v15{S!8msSeN0z z!YgnL!vwNz5v~ASu`F*gO{-V~&Mg*P13_mB=;4k?_LQ_eTTKW$RzOi~j|+NMF)z!q zbG%q;Z%v?Fv{WiVPfJC1!SBwqoAq^hdQ$rJiNPk9mFlm(rK+K&U+TQP*ctxd-F>OJ z4}qufNhz@;E<~Tb8+o0r_Y%9+C3gYw3LYEL3(=X&8Y6|NtwmZHj!T8pgU-hcVNCTz zy{|R04i35T6Q_pdOrbH+j%i~&IdXg4(1`b)uQn=KKUGw;$)|7grEiY7?1((s83{TX zvKtDrsdzQRS#-4XvS@#dsE?f(NH+xmLGa0+i8}*4T`g=qU0*M05e#hntWxLo9seq7 z?Bi}3QS!OvZt*q3`T7r`b5%t9qK}8#Lto+VmX_SfNT0Ap; z%^eU$-u;gs^lk2po|>`K=Cxn-=DO5LQ-4?5mCY?U1>JEyi1VO%_|0^!K8fmB&Ijyx;YO&SnagYN}#TZ z)y35AugMUUh=#Nq)`3)By-`kPS(!8Inm1x{N!!q5TN}C|JS-mju4b%fi%7nZ(5*vF z0VpeV#Jxt<>>AK#qcaQG3 z^lzlYX&ppZ<6I8Qo;>aF!%Mq-RJ+Ke5;nGQqB)E~{uCAvL9aDsrIiymTG&-1?gaKG z;oQ90imUTQnr>=Ndo1|2w)@ZDGCr3p6#~WzzP4M2y^v5NCElrJJ9o6&amkC8aPD*l z3*4BqL@lwWhvD9hRu#V*wk-oKIW=0}&Uxdjq{{a{X@3=awK0rmPp!e`g+dKcG2n)j z;dx>UpQAks5it1mHPtUOtVa(+FJHKL5$`bT%pt=RI*g1Upvo z_x3gwDPR2*v0Y7Y(bDiXba%NgbcFr{jZe6{cL}7e2J;&SiMU(g(3b>K6XxbmY=R}b zH;dfS&v|1;wQ}bOKDkmDz z6edqm~z z9=53u(qG)mR7mm;zB{k^<*k*mrd(0I!Uplk{C!uV1Vj8X14@*5792+KM@MKAwWQ68 zWuJ;9pC_3Ilh@%8aj;Mvlo78U;y89-JMTR*s0Ay3#R z_#ch zn(m*Ge;5obvM09q1po~gg5p*7Od5nfDI|g_OlIl|{pFg)B$sIE?p)Af^P)ywOAUna1zQA;__G& zJH37tr{yJ<{!x-Y?>z_Y(f%(vV`x*NKBsPA%gG*8`5xz3WX;pj+nD93d@OIQ9bb&h+oEV z|G8E5nUQvIt4f0~BoYt#*zZBgAvJkfIlj#YhdsQ-Gv%>-5g&K=f1cTxcoNWAN;tUz zpAe>dBP{Y{ru8{1iqO_pR3Y8ITiQFjp*`Jkdp zuKacaHW4#R2lFu+*5)FdL(03{p5DG_wAEQDU6FTSYy?vJC@QvP0-qn=#aLeoNYJOv z?tm!^gtK^gx0B!f;>-6IoM<=roNGxAS3!Gc%zIHw{@V_y_||7R9@0IorPt43-6El6 zp`*$Fz^?jmqB*0gH6HDnS(qXB`!EC>&Q~!q=q_(FFg&C`Sx{>Y1r@m%@V-+kW{MjeCfQ4b z`iT=U1?UQgH5J1jO@w>Q>Tx3#TxX_sGU$i&Pi2o@D;O2( zPZV%zM>|#V3eSNy&8o_OG5!|s$9#FCGsAx~C8bII9vx~G`{qRd9qvODnAbG+A<^5) zCx~zsL-)|1&o2Ja%D~}h7VNs_3m1hGr$t00?@E;bN%-1igh0f`C2Ye2Q+j8gAM$zCV%^KaiDA{xx1YWW6^LuWXVBYy%^a{jDWo7ke=;)x1sIYU zQAh^Ir0%TpX-`QFT{8HFh#}V1IaAHyXWN`$6&Fv84}8e7Ec7@ee0~;Aj&`%=b%+dW z%`J+Ce+)^fifK_0Z7)~ag)lMtK4)Z*TBZdu_5ze9aHrS>>lVj^&=cfHBmh_dA(0x1 z&Y_=&pW{TR@IC7sC;|Bs-3fgbG(0tl47GhJS8{+T5(0xka_oJOYRQ<~^o7tD_N|9e zmC7}oPWn%Wt;$>Qn7W8|C9g?+7l#2$3Wb5wjrZQ10F@PbPnRNwPXj9YS%=rY)WC5} zjS>k}bSUSSjylc1!aA}WwI1ooseSt;?Z}t$U_-5C&N0wxr{N_)f}|h($kcU~cONI1 zdL~Y4X0~3mloc7=WQ1WO0UahW7QsZf4JMC7J%Ml~d5C&^Oa{YZOK5 z3Qzg%+lt7qMuk4)jGrNR3FeZ3k=gRP59Ia0BT@*T27*@=XuBRkh%pufUv#jE*Wte6 zizF6_9-b9618VZ3U&*npsN)J%Hm^eZtGy&~t(y{!1qFiTB3u+A{chotXV@hkQ^yi^ z5I>5lY}Sv^kltAlx7xhD+Vm~hk^0KE$iu&sDlOR$B7od=qVss*CjdG$j1GM@F=i0s zO1QH|MVrPQ`kpp%kv_4C@jRzTMw1Cn$Ta5x&IJ4kUyOU%GRoMA;H;=&FX=a~B{|Xp z4$XBQan`zm4t$v+65t^W&mw_IBCm>oaQ3&Dgo3n_#itGZ;J998CNHrCXRZDrDOlSZ z3w$0MQGrM09iwG^-H&9iEGylS-eg#{(9#avSwLj(1HE2gHlO`mXqmt{65Q3L1h8=p zfCON|pDG%vfG~7z_Ecwu?ZI>a^0lm;Qk$r3_#2B$zP#Bd3}E~;fLU5Kv)mNn2FVFF zH^vP$@GmAddwopVKHA^Xnc!;Lx9vKs2$l>lU5(|{mY=0xXTz(5JQxEvi30EV^)l}9 z=b`eOd9!c53LUQ1D*7Rw^RE5FXaw1Hf1qbR^a{a@=hX`aZ)L!AC^@{bbpp2`CQ4>a zDls1xbrav{C{=^-p4G&16EqHBBPm?xCIaG|Bn!ez;x|=+FhCUV8Z`D}&NZGF*B}JE zlx_0NH&6UpEzA}X5~LsQm|w@=ZBA&1^VxT*dCZ!NK7w!>qSx4brPJyPhat!VNdT5Q zJ4zq!jetS&H7@96rNxmVJ`siun*1Jg-E`<<)~P#hH{gy)qzMw9RHn^J6$EX!qe5^# z;KHYn@=UClh4fF=H%**-C0eEypS7Q*`@!_`(F(!cr!i&J*F3IIOZ z#_5@0LW1~wxT1{~yY4IfNstpPQ%`&OyE+2XpxZ1k0DKJvv&7GJ%qYz1^RvU=FCEtE z>1WE9`sj+Kt(A{lRxo&XCkRgB_o&M?G_^;{^9JH1!!YLG-^ws$N~T>*gEEB8*Vn+= z=zRKp-vGjzUv-AxM_lCcTYSy8H4_o^+4tibq$T>Gzn+B-;k^<;v~)a?Zf4k?$l-RL z)Jdt29SkGCK^|+;iAex(9Y7^M_ykLd@Y@y7R+6gU1NygA@0ux)DoUo z+d?OuTol0y0G(SBny&TUyp?E+Z~a8T5eb1dRdtdguB$jDCITudp0_;%BewA%?Mqt^nYzgnT9?L{Xg-&qE%hhw$o_WQO$GG?i#5Sri+^iG#DV2b ztR>n@(iYo$5KUd*@U!q?QN~(+mXxrqfye&n6}ofi4l6YJ(Z^QAdpBD;&_mSZg8*v? z3lK6C77e-R4L?H@GqYxKuuqTWcCw*l(@spEAp-43k)Jov^G4>iK9Col94NIa(s+dbP=UHr9&dyg&l`X4;;Gp(^!Gn?$swj*Y_ZJV?;?xU!)tmadoHG>Ya-Z62uqEOpvUZ zBj11)RxFQs+?x>ZR)-b!T3@^DBg$K!X^<5+#%E*-&JiEOXC0-}T@@DLn$+Z?l;%oP z0cT=mm+uYu)+d=zZ0Q2Q=4+t|nH@F+r#sv=&dYe8vJsvl?IMb3AY|i!*8;V%=B}qQ zO2x1G(nUEIsGOBlyC^mkKR%S4I4w58cFyhe*vX8&UG5Yvav2ducpq7cRL`&plV7`D zyx;@2;iaZ5iSu*C|8^!0r*MorzpkgZ6M(ir4N?a#z>BycF;@x2zTfMavzb~gJCN>N zP|t1tP!B8tc-R!yU@ofwf9DtsT%-o~JCxu)p^B)+t$j?1D?m7XDCM_(&@iLO_R zzuj}smkPBu2|X8CK0aSAx6n4-wK6vjL+ON!70}ksXjE!u%&=5gD3U} z6~P1gL$BsXxcNc?CKa{03!Mu&;o%1ng(7OEmtGC%S7Ir)vs8O?Q|lWaSdaZ&4)c$t z8=^`MaFAd9D&D}=$XU7|EW$1yao`K^w;2GqeF-Z+;y3|vW|=&@g!pXmd8Y8xs`ff& znVstdath!jeuA?=42EQsy+ctRI5>!@k>H-%M|wqgFDz4zHJ=aSu=R|sZ7O!~ODS=j z|C2w`vmOf}`J>a?WV!ruhtNhl2ftei$WHcHuLhs#?gO54fE!{mi?R^D2%Mac$mb3H z$G>=QY#U7&5V+J-5fXNPfZ8&~AVbep!n1{wv)?=iPiqIMDNKZV#&~b6H_)Njh-JJa zOWd#?2F3*R%^CtI=2M=@M>tXdN-y%=Ns;gb{j0Nm*75>oqACfPbbxu?C_Q;bNzG)h zUd;#roJJiMC#GJ^Hr;hCcG`Q|~1F(}=0yuIzL z7z#ZQUJv{e1oS$9T9D{)^b6etP`km-+6vIamrf-i-`*ZZ`8>xlt;6Omq)>{PF+2#j z`TguBL}$QT3g756dQ9Y;E&}|fE>HP&zk4%0Wut!4>;4lh$yOHEhe5oinUC2PcT2|F z%sEq+9|DqZgmfev0{+{f=^wl|W|zcUuVr_(O4eQ%Z!FcBKmWfnBvpQSk6=Mq68FIG zDZodWI-RayXxt8F#QjiXMGaP#b{OUfKqjEG9oz+=OAhD zwL37s?(4VdBU99+{sOR43nLki*o@)oHyaiDjL&rXbK*Il5;o*D}!iub& zA{oEX2kPqc@uNTa>z5 z(wdJ9BQ$&=l1rEl_hOQU6407C?0004;9Ui}n6A$)QA->xR+OhhCt5TN>c|QU0clbi z6bt}jK8GvaY)N*vJt&ILy@}8#7pP3p9bAvJ*ahak;LPZ8KKnvy_4K%4%lH`0I#2eMrf$>`VIxWF>y`<(kS>mfG@tn%6@MA92`ybv{D z!jTM)E+&Pc3y10Pi==MJp{voI{C$rW7}lt5D)&toRO=vIdlB={J(H~)W>7|0|Jog&4h>Yj{ zq}GIlGP@NU3t8u)y0$-iZ4YKJuZ7H*YQ})mlqM*txg_-{1F9oI)o73l{kAjKED1QD z$1s%d5KkcGzY7@;l#Zkq62yb?V`Lt309K{CqY=`(6QWfra zPz1Xyac~<_bdX>)RCvhTf*D|s+#0_gfQlA+ts@eMx59@Rxq!wL6cTnn8nq@H$~m`G zVJ6Q;9n}L_zylVBFcY5)roTe?gw#q%c|o!Vm^x0NyAdLyIqmwb?NTg>qwfIQAq_>U zTY%oZ>e+;&BqHv7mw7C?OgjUoMyCt;egmUX%hW$pQ6cF55QzN%#EG7Z8==G^|F5!y zFhR)D)vJXf`t3zDC*ihem*;Sz(e;DI%_2sKg>PAgoMIfp$kA5H3-43vXi?ZAy9Wmq zKYk&J9*|cEp93CQ8B_!*qqN)AOigwB(n4N|#AS&6C4u;5VjwNOaxJQQ5(v~f+rrm= z-DJ=|Q*|RA&Hp9=1B2@!KRz>K{s#+N{V9#Hd~$f}OoZeTp7}lTF~jcCvA5b&&cKb2 zEJb%Nx>4t+Eb!QK4Jytu+B*yJ0<0EEe8oq{TWs=1@Q!|2D6BRaMfeY>JtCR5#_uj8 zK!lK!e1lRD&DPgA$c;!cpGDnyRKg)1w9ogB);6Ii&g8X^S z7CsFV;o$(1A^xEBqR3C&pI0w%d!N>wgwZ8(+QI}`_i5gyd@bo$B1h7S9u z$>!Kkep9ppMGN#tGOz-#s?#?H2R{W>J&x$l8@!qKOgM1KSAkWy8+Y;Sx!}_nGx(Vm z)Dk|lEaJ-&cGoF{MarxdsWA1yF! zl@g((ef$m%nl8Mc+;eBoVOUgd_|%(0D{UdyiAxeo=ku7)Vv@{7VX z9L6$Y0dPEk3}dN#WOj8qpuOMV4a@M7M+!sZ{{haCni`UFM%c#|OfWiqCO0^Z0WV7K zQMQ+R=#-MeW!0wxOl|RDICUr#Kf21U;69H7}4>q-VIu;hl2bMRf=a$)z>qy2ylL?G)v3sui3wDX-^iJoU4wloc@yI)JN+W=QpUnb*Rsc-16eU z+eO;Mvh+7gLkP;4YiJZA%_m|q$1>H?a#-G3S)JW(;A{7Dj<<7p>h1_H>ffvS3N$y5F8%C~Yg^bOrJ7_Ys`V)4vY zg0XU1aZ-ZGz;`wyj~S{H6&8omDcMAYp;r54MvnF{lGxA47|ROZoAG9U(mWuM%?M*8 zKDK$o9`z-@m80jx90wnSrDyn*52k`@n5p-PvyvT)Zp~x;QIA)YD9g3>CApdRMaDp0-dKsz&C(M4 zGcDMKj8Cu3dL(Uoe<2xpcWAS57ENY`nA$sT4eQ#%%?gWmBy2T3ohnE*hSAhI3!iJoHL`*l`q!|ck(Yr>=mHJ}$Y_m2AWx#>;@ z2bR0H*te`-ZuZCIIWUD233m6I6pM7nmn`m-XitjnP1O+3u5+u8y6uHHTRk6-2-*0B z6Wge{vYi*TH)_>ym*iiTWiGq3y?bzlSfk7cOQZbdlVm#m+M1z_$ZS_IJ_K08Ymf&w zM0ToSmtuCQ1}7eS9GN%2xu&>r7%*%8#6IAp?tZ6jCcQ9D_R=d5SFRmEi{YX6R%R|X1)C~ z+KBToj8Mc!%9aU*EnF5kOxNrsN?>F!>(GDrCByhL z<|qEp!=ga8|6G%#?n?k`TTL!E7t^Qt?lLj0fi;t2md315dZWYD{F$E_w(a0>LNu;9 zpr&25-bam9QrOvc7Vt$yTtVi@LK;10P z&rqp@kb?OFMccKAu)O9t^B&g44HiwZ#KhU#rA|wjeHm|sk+LAX>g!D10|!Q;vbT-{ zn5@&I79))h&1nt!3$NxEQ|nKD!kNixzKDLm93RTNxYds1&l1PYYKE>o-t$_>YVRUF zo~O^*736h3u9Mn$H&k}TwPrd}9Wg2E$^Ige&=24WK=BB8T^BC)@ox*)hh+0Z7>PZT zdW>}5!m~v51g?g(^)m)lw-Co_29;vW%M^nCI#?fD7U?(V858w^2kQ(m9*J+{D3jJQ zvh!v!;T6uJv9w?R?Nl`PLD{HsmaAy{F%7T7Bqi^5r%|8Y#GqXAYH(!=3{AH(@a#e{m6%0qg-r9>HGMXg&SZf zUicwl8q3E4K=SO7{w1o?5G8q4Tc)gDPOBNIzKh?7!_8Y=3DB%-M;1Tb5vJSKrcSRb1ycXEAOKj^i>Vn*jzR(dnC85-Y>ZAni)-g;6TiK zu*m1I(LlAg+eNc6ffL)F%^!_@v9>Q(w*3RYlPG_=gFAWYOHcA68w%c?O_ri#u>00T zW<>1B<%V9)ITx*cP@av%oNgz!-Ot!BBU)bROuW24Gq9P_u4t?@{PCl(LFd;r#CfSq z5z-stIN{Nud?-|pDszQJ0F5G#q=BGGf-6E-VCJZoxxTx!o<;nulyfBJFoxWi6OXT} zj8U~bD<=J^n%Ky|_o(#Vox2)&_NSZ+SFwuh90R-AHPncT?#rCkDB%^?GK-Q5Gu0Fb zGk1z8%h%@A$jw_yGLRY5cRXZq_%zAy@`XiWbn*T63@1Co6k8{o7}n3l0_qH({isPkJEZJk5VINh$OelWb_IL{3Jzeez60w45>Um zS!BRkrr}~*#^G|vz_mtl0?P_lMZnaIPi}EPwP)}p-Yi3;vUxnS3`OU9EIPiYkJRkJ zbvjgljF!_^8u>erT>j;89SHLA1kGG|zfbMf=_KnPCW9pFLm8)C6JjZNWFpy@W`oe> z9EnEb4D!3<6SZ5y-jYRoNAyU&H#h|{<21-9 z!y?t#@c+8D@JTO#xm}NfFcCM2=HM;IQ{BWbAqH(AAy$`}ex}S6)2On45eQY=MD0bJ zBJ+WVuyAKfcXoR=0Eg)xAJPkTGn@J2hZt3-pO;X$I`^d&m5Xz5t9CXTA>}wdBIi{U zP)lZnAzILuhA0=l`@jXNe17@^d+CKZ%aN$P)CCb0ttMi6mdmyMc=`2Q# zw(Tu zJtgZ_2>DfsMl-}_bc8tOh?L>i;AQ2XwAuYpWJwhTO`LDuH&RB{`uX}U-kI3jo%==T zu+*zHGmRqPXH4Tb;aq%+|+SLOI~pc?;Gr@Rm$pj?7B0 zeZ5|eb7=y8;W(e8Jm2D_dW7i;(r($Fch>698FUIH zkS$^lcJ-;JWzv8%01a<5DrYkqZBUiov;nVqA8uWZo_B4#swz8x5l6c*K)E`>=Kka{ zQ7ilVx1S8eFlahyuihhtRZ-WQo*V=%-3(04_V`K5-YK)ghFFR!hvVgu@{m4&TruP{d86}H&H?P)yWVrUtz74O3WC#Hm)#bV{re>%rbN7TnHD>8GW4u>vJ(B5 zyh8pWzJ=dXSlt?e+AjiQLdlw6@BYK&P{S?ztBim2hiAB{^F&$dA1k+B^6T2WVS}hQ zVq{>W|5{{3+X!j&d%_6b=WpHHmd#2I^MtZ-#~R1=@;Yx#C=SK>rF3s?)Js`bD z20U|%10^^35W#US;r#dh7d(Sl6bJ335Ch}nbZ?^9L4L~f>5^H}Tyxon*n(vfx3e?P zt2Y1Wip+Z~?IXxboQ7u;49c=j(e$7R`S=Z|oa71&zH%wNr@z|xa+e6PsTwpLg*BPr zv5y`>Mncn6WX)`9dAa7e%9Mf{>@O|z z)9f5OL<>`Bwe+9=2FmCvL;~2 zi!SUg#Tm!>wsuA9clRlYZ=gFp%_`>G8euef$nH#yCNfmaMczH^I`NH7v^$ncQK(OJ;9{?q(R*2)I8~s ztV;dwWoay&1S6!-#ylA7-;h|+VULEUj%g=rV6n10%^RR>424}(A{O_XE&9C)eqjgQ zyJ_Iz@mx_m8LIIvauekmA72{SJ2NI1v@kC6SUNzECR|RL6Q~&N!cSm6knQ{Vrfo^3 zI3lTYwdjlAt#^pPU-Uv99AL(>{?@hGug|@eMt2;vTQdiTdOiw zugQwzmkzP}Me&Sa)Tr*wzpl<(t;Xg>{Gw&vra}2?J9tC?m6Y`tz4GLA{CD{@36z&$+???(%P z=xJ=zbdsEMlysdI)`V~ucD#sYwGfNDIox7aOz1l8H#XWjzrEw)iD2~`LlN>E1&Fxm zlazVUVDA&by2y3kH@_x!9LTDcV$hR@u7}^JRFh85&2alh6V{8qAp6ignQ18vYdAP) zri8XEQ^OetxaSvS?E^^IBR~yK_4S#+Snhyomz7jP+(DDBe`{RI4waUj(sbV#>^`|C z$fM)qOyxJ35|g-?>HdX`PtUhyybc)g^_ri5;@Z0TFzSp8Y!YjB)x;|jiBXdo?!QI_ z*Mk6~;ol(LwD~1ifWqWVxbNyY(>5YD?{WVTe;If1h7RIqvz&?fx&ncnKmM@i)WW#k zkGHLUkEO!}X&U9)Z%MZ>OO^bU*lsnIYhlo<(J z0LrGKVRviQVUSK-WeT;bhMNGUNk=&#^zzc8ldxyV8wH_(09YB|HUx9Hik@?W)RNkW zgvLlexs>XMA7=mQ<;xa|o<_&|OAagu(hO(n2BK%iem31oOI?^;YJ-bJeGPm4Z9h`FWX77v=c;<1RP`PZGG3K}!bguYBm zI!|kV9=PpiJt>dY@WEvY-9Gu#o|ZhpKLyoUw_nUkp3F~6h_JS;-+wo2IQbR-d+n=k z)#lnKIExFc6qkO(lmR`{({F-_vA(+I785l_>T=5rXPx;6YSg#_(S;CStYz`e`8LAqkq*O_iPoNn(aThNAHS?=yvkq6YmMY7$FKit8z zdVc_>^3BQJE_y5HJ^DkgWu1X|L_1{odBuBjHzPH+*T3vqvi)gk_O+Zu$TW%M(If0D zjn+B6Ay-uP9g92|uZR!@c zP3LvWedaErMaEL!Zsmxb=_#|lE|3qJ$b^o`R!FhCSJ9xhL)UJ_GK@2dNy9_$FobGS-KhLRpSf=1& zf}|PQ3niW)t!qdU$j7u;X{&Z92)7J#RB|Jq+kBwIt$H&K+Gi$d@t@O7-n8UX7KTYK zPJKf+HX(mijpQ$8SS$^(pL9gIA5yiob{RE2A@R0D>ZuTM7)SJ$CiwCHRr(4|SgCGV+c8b>(6-XPdT&-ojAEO40Z4ez8?6vovM^ z|6RsLHNMm2F>!-}1pP(3FQMc zA;sHo&`pwd0I+XD_=;1?-YW*-9DH7&N>PfEAdXU^2IY03sDYDH^)EO*HFAzTryJHNHF2=GaYbY~h z$BQFJ(jjO8P%;RPzZvNK7Ko9Zzgc7z^dt;a$BUJ}uy;h(YG(4DR3rAelW6PJ7A2H; zh8`NFlQr&KE^V=Xl345wL#$5F)=avOON-A2U1S>s#l>hU2ul2Vb zZk%N#MG>SujI273Rq>P*pY3$7cdDwJM1p>>lq0e%m?`Mirpp)k4?FEF+5r_nAxA*# zd89&sIE|qlq8+k)l*lZqz5N+CAGIsp1Qbft212*HfrK`cA8nqlfU1KKOEmA;=YZCV zygZIaQXKVEpE#V_KJqJ5Q$c>8keTgq%8^s2_UJ1`HPIJ9*`uH z#3_Y0n+vO$i}7N`-uG=zTvx>@M@7bFdzgJ>ebBQ>@C{`BPO@B`Bvvf8GYD-skSY~X z{m*_ICx&$<5N?kMM@IAJHr}~~`gUl~(uOcf3cv!6iaD2+ekWN|&)oV)n~zy7aS3LO zfz_$3B^6tp(c;V~p7LKXBDx2Id5)KplQ0^^$xFha+>Drl78MHuvB8osDxQNLj;b=A zaX)3rgPHq&$ZBj4$eT+XL_CTW1=4#PJ(s~rWcXpOM?0hX55e6svp@}TtHX4Mc6oAbpQ z2jl0rx9pt~X-N`|t7Drx8{HBz(VUt>?KDD_m~m9JT*)bxcDsb!m_eem{YX4z{lf>( zMO}0o*~q88T)Lb~I#g*%69?D>W&eTRkAu|=N3?ELu#C`Nfjt zs~orL^q`HXTA6ogea0d7WN)Jn8X&{rElbXa> zwp`$jd)CM%FFdQc(})Xpd+2rF-Q#80A9RVXx_e0t3UIZB;Z)de$C-GobG^;KoD^x6 zNw%G`JavMcOw!FWIvZH`97Ii85(;@?lu7eOfUX{ZUPpk_N28oZd0OgrOoC8=RhYqh z>HqM~#;^nYHGUUaU50J&z>@3~8KDW=M8hF>SHq!-EKIU=5Z-7dC{h6I9@?A&w8;V8 z(u{|fy4}GSlqVV;+6!!CI+H9V%KGf-Fv_lVikaCX+_gEAbybJC{|4q z&$)qyD!6o87~Ad$ZH+2vR;$XhzaWhr(=b`{G)H&2j@T(ys$u8D68|I!MuRx~FW3c@ zcOT{oFp+Ao(x48b1v|7;06jL4i6nnRGWJF)@UwHQmny^drwF{dkU_ zn2IuLTAjL+L!O-X$FtOLW7J)gC#VP}7 zG_-@h{b%gsXt4mfb*sK4>sqNMD`@svIJ%RJer4fKBD8E^E;m4+ZqVp@fcDZs_Gc>g z0955Mu=XCsqeBOEkq#?8vp1qD@0Dx2@u7V%sPB@$m3oK*F{P{fPFZnBkitR#bcLZG? zP>(4hD1uxM!*YqDLuCXSSgsCPMKpIZ3WMY2%;g)-O9nheq8Gb;-YW`=Z7mnPMTLAE zEo92yz7WZgQ!4B#iZdcM+`0As+5k1x^uXRVZ5{S=DLQhzppMui>qAKH^rY5dNzRyC zRLo`rF+6Wp)v1O`QjA30KC+zVH-)ZAnuaq<6KsNN9jo@y#>+w>O_YGsELRZ&F&%36 zh2g((2vXUKuQxSkdiA16>mVhA& zJGg;Q;42q5tOL%$4*TnCC`45HlBnn25;y3M_bx% zz7EdL7y+oNxG)4Xbygc=>)`6;=4megQv#pWUA^p_!8<4he{~JO9~1Brg$Rp^|EfXo z0nJN7gdyiZInvJ72@Fb=Rs;MK1x2b31UsA?2^7*Q{ThIs7Y6@purP0!sFIzT3lwGG z?B?QOW2%n^Gonq($By9X;O6?P8W=wG&xgIp?smUQv35QVwsya&yMsx2{!y7$0z&}d zVI44d0>%Zj=J1Db(E5Z+PFO1sM{QNChL(+)p_du_R|BpXmp_`-15I55L#H(&X-x96 z755ZF=!t<6Rd>8L%FV&V5#oZ8*2TC;ni#ogOXHl}h-6J)l#!#FxT&E4R0$?2ZD62c zXK#SeL2CG#x}sfC25u^*;?mB}UdHZ{N-ol9ZwV=V69-TiG-@O!h0qp-`C=XQ#LO@@ z1a}u3Ur;P&jCUnEpwv-lEwY}i0m2rqDh-Oo!Al?RCaQ^rNvL_b7%4ewXo4vyVSLCC zO%r}iYaBIRl(?Pl!b1i?Ak7^xdbDMRcWq)atc2qq*4U1M*kp{NlG zf`+*vOc2J}>Ut&wh&Rp&=P8PHGV#%rva=sM{*}BAiK%(l94|LkT@~ADpA1il`ml#tbb6cOj~xZ44ZpJdL0j2pr`I=IQAT z#h~=j((0-zCaOdih_{ce1RetOB&gvubs$=XCQ>S(k`@U}lfk#Oax#bk)R+ zoF&~6o<7FT8WK{rI^YX4cXcp!Lp|_ERLN6STg<>0M}~N7pmE}&5Dir+2^)2ksD?Vh z)z{vP02EBs-rZT;(UfeYBVnlPLc)rfVLhEC?Y&8OsJ*wgp0=2phYM5+@2o^rR+92T zBix9__DWJ1GmM9ss)n|THjeD7=LQ4qpwUQOvW5#z(^LZK>S$;rZlo`+rKjg?L{L#h znwg496LrBZRuvW3vBfx&)V0*y43)&4u`U>-v4;uL9d4q8uyeBq=1tYWK;Ku~$P?yd zA`Zh4l$CJ^bw^t#N2rn`-bTk%%ot%RCIzPFYmDSDA#r;SG`Iid_T zaC%O967~r2n356+VJj`6u7R~hdw?2_B!nZJY^?0ypouUN1^Vxe)3;SKmeg<*H}SGl zL!b#rRZU$tCjv^^MazSr@1$!4oMYTnB-SJ|eRW}c36>UwZt(uM!5{9=U!)WT@A?jfZ!J429+}*?^Tx4R|*1IwZ6tO^=aGBdH!cSZZ?nV?BhTAG@wSRz`@*whWKj@DMfdy0Cy zs^A@TOuSv)@hYae21Hi}T?bJuoV1CWkG{663Q0}U!`@xX9qM9eN0YL%p{g0iQ_Drn z(?r?V*w9B)Lkz14%BA%kT#O)2c8+M;1e{&m#c@y#I~#i?Jc(e7H#WsXbyPGpU3E!f zhIojCnUN#X5%1vXfHL-=)mL&iHq%E+L-9Cuq?ip{!-l4R#!gVYfvXWpN>^9eRFrIn z_Og+%(?BVsL~UH9bbxSFCDldU>=5>*u3ktVqLYq0n5`#x*<&CsX1aC=eOID~F;UV^ zm5A~504*w!4eiu4^~BJ4DP4@Wv#PQaR??B6t0ZA6p{FF~Yv=@VwU=_zmJ-uZhxtIj zS)u8oF5!jMgLxBmoY62BI02>Pqk`5@LTl@riMiQ7{NC6bwZ(>+Yf^MrzI`x?l&Wic7md z?Uc1N9i=c*UI=ZhsSS}f5u};1wX>Tv(Xb9Ia(nK0V2_$iGccL+oBuy|t zIygeHM68Vg6z}2#QzscIOS{^b>Y|+Sj(VnWA0L>hk{bbOY@(`fgD}GxlWf)CM&9;l zf`R>xVIk`gl#h~7q^38MNwYHUT9lsBvQvn z*Tz&6;%cj}FRrHM3XXCmF=Km2jJ~U}2@2=p3r-VFNoi@gqa)5&jqIhN>WmZDl9Ysl zUR0%&0H6Wye`1;)fCE1N0T5w`ZU3XXbaec5>MC$U(vyjJzbg-owrUeArl-5}`dpRL z4I#Urdv^~)_@nM0MyTxZVwv90JX2?fi!gI@jxH&2cxFEG`q<;x$3C*1Z-~K=q(%c;mgk?0`n6y6csL$w%6{!Kp%DuWc#@cS6{4uPZLOoz^5Dsn zCjx?z=R_0~sNtBrK>Jnx#rt-yDXrf}&s)UEQop`js_+>7f`8yg$3;KdzTzYyB* zqM4~B_)#*t9+Vh;e6HIK)9MvDLz=jY$T%>=AVqFK!*@&5TqFSvf~L~#A;GmGMUSsCdfieD&7f;*I$e{r2-ui#WvJ$5ba z>C>-!;&r+cF&*1Qk+rU-scCe&Zz6>b%rKUdUk;6o;5d#R^%s5_ZP)lX>5}>Dr?i*B zbDbpG3w(>s@43w1)7jrY@%4yjsdEn$#njr`x^2_-vcHGfVBY}q#ufD^kLYW zyy91{X4tb6SLa7lSa>9)e&$tFSS-DMW;8iD8Rpyn3bQ*_@oFWd`rW&C=7Nzw&Q%0{ z+U&S^SL)1GFbj!3LC!Qt@@uoLfQX2jKL&%jkWH7_dH4K<3n*=EmdlLE#-NoKX=#i` z#>S^IJAaOS&BwP^$_4nghy`P=Rzn0&_z9uinK+K7w38$y5sp21c4HmSH1mq0I=p{$ z2v9@g>aj9e?btogJ0Ao4>c77zn}EUO4&JFW-JK95q|5%5g*2~LUtcepchN7xkEyGz zjo(i>=<<{v=zUt}GZxAl1O4#u?ey1`_WK4YTHc`Qop<4BP$j_?403!23;h z@Y&=sf=nh$@&!t{G1%5E+I?>btc6GuiG=9+#xNnC_t3p^^&$eZpDcD!hoaCkIA}~4 z0e{cQ(GikPAC;21WFcS!w0m(#$ka=EU*5s$9jMuLCa+AHxO-5mFfvN-w2+`3_~pw{i<|om+;Ibl}kcA(eS+M%rh!!_QUCA^yT-@A!fTgn0&Zf zxvQ0ABZ@LhN_s@&X!}L@wC$38R+rr6HJxak$fo8V^ViNX6W#~EA836atwq*`m-e4T zp>FQt)~-}!i@TZ=|Mo}KR5>bAv5rm`8zQkYmmYlvH{R76N%yIPP27~rK42EIgpB?B z_gBVCi4!xxz!XzND#JzdHdpfNkM{37u)i@cBvjtxLlE7&Yrdz?9xdlim1S{ z7bDl|>h6R(b{9EsbdY2oLf(ycg!5Z18y(l)_h>)AvO$l-jd}L(hOYmh4eS2>o4aN4 zkqF{D(#`CR_IRDhJJS!ZXRK2Dpn0#DKf4 zE*G1s4BEWMl}4m^2`GP>?Q&m)IK4Vu_x{ml%hnl6)YKqqDM-jyWw-9Yv9F+s5iVukYHA=kb9)1EtqEO%3etdi`Kj)WYRTbvw zLQc$bU&zsi+T%W17hywhCOT{-h? z1x*;9FS#!CwN6_37V8PszSl0wLOM?_TV37sPOJ=Pq%CfopVxTTrG*}RV$whvF~z0t zby7`DjRKxsua<|J^{&s8DB=7$h%@-x4+^V7#D+5tmo^AfP*C72sVt)Uh7etVK6#8! zm-(+kHPqD)ckZi(*iZGBjFnASMpiTr9hH3EfVBzF9L)NbbVuN%qjx>UEQ=iU`Aj5- z31`jh!rtaY#V-Pj^qX}hJ`Nsy6iR@95RC{r1+TT4txl+-q$8 z5wFcP@8S92W$D*uS(YwgE)qWu9${w>cBPh|TYK3dtLEgC`LX4ik)B?V>p&^6DD~+} zOFp$z>^;(})3<73PAjo#DkUfH-)%2izcM=%%))a;U#KlSm1f>hi9a=Y^Dl-#Yw)~t ztXz#5Ud}dp1qdFw0+X!v=-q{8^Fb~C3biR3;qIg76M|B7;uWv*r5k1l*ABjV^~$o= z{Q;S#soCALgSBPYPz&3?8FS;%9@@HSGrKq3fJ|j_Iev_#HaMZVy8a8%8~yO%m{ z2tK|s2DTL>cXhb|a$pcjn*dj+DoTE3eLci74u%iw*x6{pT}594L%{Jus%mU)WxaXh z#_gXYp^{WRVK=G5cahW}jeejgm9WM3&E)bb7i?NDxRZXVlQ{({Uj&QhXjCUBAE zW4$+jX6CegtG?A9#w)Am`S7`P=REo)38aWIh`MocNdFOGCKuWp)#)(uX+okR_pj+b=0nKX9*q+ zP0euvds>lqKbLjd)u~MGil>Q!E;>Q4`OgV{IL`Z6PeRHDdp~1-M2o;4OiBnkT?Qmf zuBx?$JOz5OFhKm01p2=7GfgtzmtmfEnp|n>13R2tHF_V(=P^{GxV5Aq@zZ} zufDO7hc*3}{5U^z_0zn(JlUW8EG#S!h*z{~2cq+1x_KmB!Fh0ME#}iFLqk*39y=xn zS681{*cF;O_7}IO_#TS=`sf0Bjyl?cTUl9AQ@B1S;bCwI3dPB1>z?l#^g3XJ9_?)m zpQYy*LuOARCF0hZ34VUEmSb}{Z{92@uI))}8CSG_w{TLmvF+=MG%EH+G923cuJ|6X z4B5+Ht|l($yH#yUOmFUE6P%$c5~rR#__eAd+V<~DB>AiYR|Tdk3JS!Yg<37?>*>u% zH|7CAkk)E_4)N~bDgUN77UI1|Og+UOwo&lI<0YdBgPZ-$agqWjo=LM6QI1O!ROSu~ zqtnyV$9{HZ4s=!b7CMm?g(jN9<>vNIZ-#J)G*ErcfEH^KLY~C;g-r*OGd&0KYdx}M z_TJf7-VP&~t2a2Oe=u*>6>iqy@^kKeDRhcb&TfFe?r{#QDBevEOLR%RbkrQf@t#6lGdHz0V6I-MeECGc0*bm0RzTw$?c^-ERs>LBN<)JN zko~!aafPc}a?`TSxcmoT=|p3+q_eLSW*^-rTL)}*GsSh{bMAl-O(R1tFD@@59UL4U zgj_5puD`qX>N)QaP))U*?hq=6k9#mi4?wA+#aI5T1Zt~4I2p>9(b-q({kA{n+U6DU zo_m++X<%@*%2=_9`$Xymt|Qt-%x9 zdi!ZdRrBt08t|y8Q3d;hy`9J>?`x<&7*+ALXOK#DIB)dsU>?4eZ=pj|WY)5;#1kLI zi+y{2PkC!=v}pCL{UDh6F?oghmKLAO!V{OTZp|mQgT-xsrWvJ&K-~OzFwfzmAa$Hg zP|JNtD_-7@9U$e|?835#u_d0PbCtSM$`@Zog8foY^+|n^l(Z}NVhDktWyQiA1JR5# zJi>8YQRw{px?okrF-C~Y>nfFuo2TfR7gDB%4^oc5Xff#!J&^=#c*4*gbHJogEm*x7K<`{+Rc^`TpFl9IKpy*$@nY zK%jS@^02z5ADb#u2MkuD-^IH{0UllO8{v8)R+UZbya4Hz%W7@pU)^5+LRNgPpRUp! z6&riL2Yw@XY35sXdEx~Tk6woW?46^!4_EWpbRmFu{84_Oq?S(zeMg zcA^xU5V4<_F6+RIJ%@LV2qi%3sGq}AhxjN@R#VSkwDqvIUOTb*u9h9~KI-5{p=0Bn zvt2X7d6vt5Q-63`S|G%rC$y_a+WTdD%UV($oyeBf(;{Jj*xVm&^X( zmxyCALZjy`x5urv!AQXg_F?@=yZyrr`FGbQ0T8F1gyr3-G8-92nP+5g&#VKhPdaja z^OJ$Dw<+tpC$C!-x1TIm&SaNnWN0kCKCQlTqXOl?dbLgDRKV*NlUPpK{*3B$c7`m* z9s(PfL z`WD|e+gvswou`ZA-Sd`yvmZDRibX(q>koDeT;o$RGBoruWo>+XX)vI&?D!C?;)Wh= zWpCW0b0sq9t=fl*IL?vh2~e8xPaR^X9GAhQZ;B3oMg+#*D~XjF z-@6HMk_#_di3z+er$C$;(FX=U_eS`~*)NS*JbQMW1vxQSOu#}rviu=S@DINhSJ&QZAP1Fyp}7+???6#ERC%Z#LH|{Fdja zA{QZS=F$FfF;xRFR_mwkK|e0 z*gN!xY159ANCNvbnf9vMoODZG))r)&US5>Qx+O&=0C(2(F#<@@Q~j2HZ0S#+N>WkC z&v570)KUI;Ir4*LPhi86$2HZ|A}09XxpV`qBqeyw0# z=O^ZX!}lccU6j5TOZxBb7+Js75`CLv!r{nW#=}Rja;^~-=_l0dvlykt24~;A$sPVy z{}|O@z*nmjRh;+5<27e$FTWoT@||c5W+J56moUcgVF$`d&ImrCW2+%zgcpz}0#cc& zCXX$e^4tlQZnAZSPTeZu95;AXq9I=f&UbOaaQt!2!Ii5cJgCYx05Zl`sFQs~A%P;U z$Zw3_1taU*_W{s%M^HQVy2gscJBqB>-VInZ}^w9D4(4r769V$ z>48sqop_&?CRp3LfB6KCb3H!wQ($oa!44`_xc9(EfT^^fNTKG;W^(Uas2A(C0{HXn z`SXxUR`xOV5Zef+-Mee$MtCA56s_GcKM$qY*{d~8sMYTCn;V&{x1kAs->?ql9{M^f zpR3N~Hw}06mwV(-uSf@w@2a^jtc>C2;HOTZ()} z2sqPaCZkPYxTfMkPih5X<>l%*gS4yHJTnb!>IoA&3qP$j~iL?{D0{}sQJ5_EBgBBhVlxihx>sHiI=_| zzTeWKnUj?@nFqvak@#V1dU|@HjOJ^lX~zk!#XNj?)X>17+7E0`m-58Tp-9~}a0Uxp z#cOEnm$|yN!P|)hHudcEzD8$>GO=xKhwt3j{@HcJMxc*jau27=*^qsqFeEc@G3$@p z_uE7yosf>2@dIbF{60vFv}a?o-VVqR@OSL%+wZ)9E2SC zaL;Omp<-lYM4_7DXG}|?Vz|##{~b&JS;WHD`gBUG!um^O2-M8XtZiwsul?~QbCvZ& z#OFYa$>8Wr5_jr)3Lvo8kB(;wn=AlFz{`l!s!m-m;mB}4#25eh(1}HjQh-uvzVpvd z6PZi$d>_yI8>Mi(C_tQ^wUG~uMYIeYabTYkb``Cgia4@%KeN*)=*O1oJr3(D-Y``; z-8YAgk_W!;W`5yU>3hU_C2msI^?i8@Kjp>ueI$cdw+C4+(_fQ&u0!eXS!HrOk$`Ax z&pKp%_OaBPRK>R}kkW6@a;+oaN9P&J&zZcRW#qqhIf3m9e_wdVaZYi09^w`}7s2f@ zx;(b?l(8;7bGWE-b`JlW;x+X>Gb*`zhkolt=ob87{Hss*ZygtanI?f z&r{882Q2yD=)I)w;Vg1Jw^t5zk)BLXJ$mswX?MhB{?>iVlI!a5rTmeA>RR)H)0a*! zR==Eux3gz8A&ZoPl1$$3;rx2X@o*j6Oq%uKTAm;Vda{l`#o6iw$I<(y>eBD z=8i|q_sL0o44iinSAupWykNfuq`9BnqkI>T-01rgbSwB~rw0wp<-3*NBOR{883%S* zF);jR%N>26>quSY#h3p}&pONGhvP5F9@prUb8jDjykYA+6(RIh`|YVPNxFRub5EXd zWAEgT$g<=i!=D8IXC2*ZpSF6;Ut}d6xA1QbYRQ_1<1cXKd@Au)82j$W_^C@TtXI)b zQ{qsP{E$PqtY{sdw3Fo(AiRs;=>gkR_@~612lxkyuY|W$rQ9Fb3p}N1`?k==MZVpK zdX=e5V|U<7W2W=p>30R&9OkeN63RgE>%NJ*jyUc%#gLwr)Q^!9c0CC1b~+F-+0;1Z zS8iB3WZ9>rj`97KJv+P(UpQ7a!?BBDf@ps=^i%T z`SV;}9Ix38yl`t(iifmU8{betEiVsNIa`Di8*yQkp2D0Q{l|)+DL~pdWY`6{n&}eP zqAn?db)RA%^JG%n62B|mA&$L}qal|ogM{w>C7^7^eWRt8IhP^^hT77`3-^pU3s5Rw z#2uJ~2lcn}y=S5OCDl5P-_*akli#90A*jU|N|Rp?%R!gCcrsUf7PhT^k^57BKj`nx z|I^V&>=7 zl4h2ycZ+BvqnqG?HgFU_l|TCFzl@QE8ESfjJ5Tx6LAc-97xK4x=-%%vW`xMsJ2rpq zElzCS88NJpkCl!5?33V?Z_kzU-l1l#Tjp%`a?p64as;GahVgByzBa zD3lE@j$$Rv2wZuxF5xG#k0V1)*pU~Py+7`v;i6jOe-m&j&rp$-WiV&(;U9}ey0;bl z7C-UTx0-21VkIjJII*9)ZWqKJvfMOs@lROZdS2)j^3#OGHh0zP(>!%~b{4+v;4;=e zT}80rn8t6vcE;rVBz!R!nX9~YW2(VQ9yHu_`*koOhsi@J`9i^0myfje`|c zKR!H5mI<_RJvj}l&FGq^CO%B)82mE5pR&>UeEq&948BYhIES_iByKa0EGM4uE_c&vyZiHVZS5^I zI^umXl`p-?iPMfG zGHaquHQS9-=!L&Vln?<_$ zx67XG5)c|I6Vtzfu{|qR_nFgkJL?Q(V=Y67boGm;CLiKOvx0fcW*k_mpUv zHf%?IIY?6$nvm5read~Zr!3Sd8`X~nley{?n=dH6K5(o3sz|_XW081bE#vC_cU)u5 z98slmOyrLZ$K(n>9UptN*1^|rN_b*>MO3JDw%2M~E)gFcy3b6Vt*Kf3F|-aSP}_yp zhbhb5St{EO$42>cEM;^beth!e>1Y5r$)f&oafX0V_%>Y2N1irX0D}oDH&j$qT*wJQ zo?#*M)en?+vn_B_-$BnAbUuPp*M7o;*JW=8JC~fi_I$uJrb6GUHzoOE`111BW}U=C z$B8D6>}~5yrFyNp&sTJvc8f5{!8+^Kb&}u-1`zE9znhGvXR`zu1~y*4?i**K^k@Eq z4y#g{29%sX4BC}*4XXb*H@4myuE9OD#@aP!IXVKdt=RA)hqKW1I>(-Jr6}vc^0Mod zO(811G%$MQacseuwxmfXlHz;^Ve8;gC5pfOdaSHOUD~-H@}+^0x{CAvdG^?g)v?HS zNpH!Yyc84VS^l%WM2BUr3!V6_Lbx!1Q_?Qz$2)#cB2F&@&Yln)^+Db9>&O*Pj-SI? z8XV0sM7xNa=8KE(X@()pD2s_Qk?_xx(oUKICUWB5J~`#Sk6c3~yde$6M@CM{!s?{H zbXhR23P^-MuX&#M^U+!_pXv2}tC#O4?u51O)umR`H2jN@SY7{S>hyd5@j}el(y*;{ zgoPCi#r^$MYqyU;#my_jw}9m^plpPP$7v++{Fw`|DqVN-{4uwrZdPua)VvO zF!cldc;TavGvlkB$Bu=yPLMXG>sue@e&6^`<-}^6>~)kaF)w(NSN3rgc(?zwuR8f# zqok%d-PT8%__y7Mi_eCWdT)p=cr3<^DQ)`HS1Xqf@E0Bba?&ACccN$PQ=84Cuk9mE zW~k21p1ctU>Ntozjg6=62J86d#UD;z@w2?}kZ)hPn-oB~I z%1k(IX5I3wvtBPtb%AS2;S;m1;%oDtWotGOC}4}-M`ciP@xbaa=Z)|G&jY7}WllC{ zhWO~K>w%H!R>^!9A>!#zg46)G<71EcFI)HYnU#6XAA(J<>a;y4HJz#eR22EdkUkr( zqtoUW3RjqE(JYs_-*Cy&*!5cgt~1h8psMi{q)uPx+dj_^R)M8YmbdQRaVTE~=J3o_ zG%n*X^IZHt0L@e$EOGqSg<5icbhu)}oR-n}EH7UL-w~EP$pilcV5ry#H!Fo;CBySW z)6=&$HaGQdYuwCE=p7_&&$E8vcxF;O2sxz74Smo@VY~)k>iWxT@7`e(LnaXGg2`>J6y&Fl=fO(+_&eL=0WN%|{Qcr76X@Jb;^5n#d+@ zwV89Olk_$i6!dNtxyod3(Cuc7p$+wZIqxN_ARoM4a?KohEYm`^eOZY6SUL~WQVJ-r zCmIXNm{ne0EhoppRb4{IHrgkX2adi1@MmFY<5c|zTQP1ZcP|AC1fv!7Ha{bj!#OX` zgg9PT5?5EA$JlIP@%*>0zL|D{h8taN2C{N|CRNJvp)>(Cj5hPIK)WUzYuh>v{Myn3 z;`AD4c?b9`v{Fu{q|2)v<;1o!a5_f}OXMUR+44Jf!uk!r!6ka~sRxq(Dr}9DR zvI2+_3}qMPW!t}|cnxlR*6lB#+5_8FW6nY_GCG!#lY<150}2P~YXKMVjvBSRlm9~C z20ehzG$>A64G7gaBKOK+EN~Q9#(i~=C@ z3;%z&2P1%IPV?-&3VrWRsNP}+4)S&2AC3(co9aY>dOh-Ew2Iao>I9~Rm2VavLx?g!}i)e$D@LnCfk>n=qMAGV50W_ zQjA}1aHvTwF9OMCPsK5YFAI2;6*5{3NH(K%Z+*d`fJT{_SJGXQJo9uon4uue`CxAu zQKaD3y~&e?R=*rDta#nd7D4}Oi;$;+=5{l4Vl#lNtTGqOG8nph-A*!RstaA_)4NS| zhs;h z8-`DT|E}KnM|Ze6T98(gz30mAct^P&{~S*H47P2%u;=}`S`m;7>mWxdSGaU6KJW*^o|UUtSpx*Y8@TA+BOcRv0{dBeF|Wve@UeUY zE5te?9eAUM>&=}8;5)_LPC4hVZV)8ZE?v6x0yw6;{btvIS_SK5cR$tbUa51vG1T$g z$)CmnczMl>)ejs_)tWQ^;Q)+Pnx+=CpfQgR$LGWy)?raN# zt{r*+`w*~GC*`gkt!6t`UN0Z^W(9Gd78-t$p02)B1F*cLq-08txtX3eCZSOqggxeY zt-^sT2h!|@cNi5_!4jJyZu-?z*M%Z%NXgeFOKOjBR-J{_NtjP-#auu{!bf67^|kVV z@kB@4o?@c&IC<*Lr|#}X>PUjIl@Ib{w8$wE{kdSF^CZfw=E-t0cdAo&cKzc>Eebq4 zGc(fv-j`?UTT`;(1-B%*vG~cPXuvYu-U}g#rc3^4cyvKyC}eNK<-6bR?5A39=I3~aUH~!^a29p!S2XGa!068%8ApKhE`VLpuBW|_ic>2!fX_N9s9tqmyc>4F`X)Z zRFv-afy28PVL$3;;liEW-DPKr4>fRz0?E8(6EALKM z;8`1KIQBnl+Jl;{|EoDiTBYd!(Wf)0v>f+8D!I}s#s2@5%HQ($IZwgrts~Zj{|lR@ z%})*i`r2>%;4S~M#eX6B;x5k#Qk?U>T^>}eFZIc8Df~Y&JE}IBSaFkf^n6%m9_yDm zkf`L~Llx-m32py+SCb%+XQ(3PhX3+8=%73lDR%DJQC?r{L7DX$#%ZpqewGfv z0*`ITa`%KDN|)IVV+17ebeB$Jyn6Np9pz@X8+BX6$n(Ua9m1rJ-TI-kO{!LC)QmtT zW>?zLT7U@8^>Au(+2wFn_3zU+S1ze0UFko5J-eHocJj7^Bk)EqcZEKnDtDM;Uzc5_ z#U&qKSjx`cGnbu~L}x#ZY&~0;R)jFSS z?Y|V*?ArVKgxBWF6W9FfakPoi;D|Pl)sEr*X2WhVP`M}IYBK76Y2GS};r8uglPwJ@ z;CuAe%CK-m@yLqf;?W!lj73a-W~YJ7_>cD=UB}-5p79qO1dfs!t;7eR%-wCI_IrsF zx@UGocgBshWKus8?et(kJVXod;dDQ0hJA0<7kkf{_Jp$jC_9dSKK{O)xYaVi*%)#Ji;emIb^13 zc$Fw7eCyzE$Na-sE+xQCYl#~CiMtx&GU1Nf&iSWnkMRvI)A--x(yt~HU4obN?EMbW z_AMutH8uuNwn-o^DX@dK(tbmK1aUQ~?nPghYZHREBmlm-4DiMRV&IBBEB#Q|pXh>5 z3`V1KY3GxSms-02HF#R|xgAjICSC`|eTAjt!{k!@5of&RV}OqYY0FGwp|9k?o13#u zLzySx#(xfmt4Yhq=#eJQ<`>4)+QH@IGPv=?3IJL&No+Gx+b!MzYtX3g7;y#AYlA=< zb1&uPpQC}saNTTK`PUdAb^Ce25RO=GiWHS385M6qyuhhKS<=({v(K2FKIiCZ$Rp_CKd>Vpch&9d070&&sd?Zph8aL}N8H|F z8dUx@jlU4AI#*uhnJN&9_@AvdM09p9Z;46Tv*!ImP5*DBAbM!edh?fZ0`&c<2^5B9idI?#q4Y}v_#cO05XcQodb8%HeT6&t%eg-znl#k z!;YtKTB0=XKr%}|t#-kZkp;BuL*Ndu5 zg;xi>=$>)kx|Lqa3$bN94p(%94%5k3|ZPb*OHeCa4Zy$Q%V z`K`?gaqJ|1=}uFs9-_yTMpPMcVa@sm8}3-BJ)4`(lYggVIF`t9kOKDs5!}l(f0h5- zJ%F+z%bFdqp=wk$NF6Oifwy5+E-YUjJ%$gtr(;H-pN5IZNk=Vj^;XGE`6$00yHDSK zG5yoTFyQ)LKk)2OhijV_glr)bPpjAz#RK>iDj&8{#F}tlr`z9 zh1~a*jYZh{&O%n1bNkxWxmqdIz^p*0V_jdy4>>xi02*<&GDk`%o*6)gAfBia_ z?RECe`F9u^x%y&cAgZj~7*bcRq!Q%d==dPu&2j<2eh)dU_kPl|T0-TqC>Ir_dPisuMjUByBeK z>mC2h%tx%!n0cO$p(o1%afxT27ef3_t$l0oSV-e%MMtZOT15oAgOE}NGt^}=_uEZ@ z^6nBGFKBnq$Z_hr`~cT(<~;vhKm`G#Al>$<+&x`>#-T2Cs40Rb^oLKoZ)fjoW!N6R zgCtE`IFkTV{nAn&O?iTL&<78 zu@Q%NGQ=Bi$jQe%=Z@!WF1F5-8JnY@J`%I&?>9cDEOAbLPsN;;wC~jcjfF&GkCt&@ zC{&~i>i79lmxT+WNz(`Lx2?=eZZ8Kmmx=83&58$+qUbT=_i+ZwMbrNzKktL!bHf}- zbz~a(dUlEP4`%(U|8`;*<5BhwY1wGNqXj%~JKlUQx@*!oUC&JWwtY=5#4p)?Byzpd zo!z_7sZ}2z;o!)6Za@lsp+O$U$;g{&#>&`3rX?HvJp!!y0G{1vTn~ftpttW7@i%tm zUf8ug(B%O(# zvn4`vaU#MMDKd2b6#Z0ZG$pNZ6PI+^*uSqFCt^%mQoY(Q;?cnqjk+1rpCd$%GH3P z_Z>Wo;baWeRv()cK4l$o@=e>nQS0|{5jWHKndD@@aQDa6j<>1@rrkR>n!i*N;qXpL z4}?mRkqiAFzutJ>Tro3rg-YzGWCdD&vQrSGeBwEpOHxuQ-|{O5C7J#`E_J&37d!qM z$nNtb^I*25amo9LZvn)jmt?brXg-me_33Au}>ZxCRYUU)1sQd^HTMf1m< zoPWG@*(J36!!^(gA#-0q0X`+QQugXz<~*-|SmIdtW2g+MxxJp8u*~&s7aMc6T5Lik zXC659I3_C^2fX1snP{wlt!9s3==!+F_Y61ItT#o-f$K`A>wSEm{69N9s>yJ6?$#u; z&@mWW#{+Vry7<9I8`+hdP!KX!=r7U1X6pbuMGI`!)|Lr0pGNmY)+E!;NJom6vfo~O zR2ds5_W3{L58Po~eIqL}+jP%Rcs!4AU>cUmApORK{o2XMhCN%LEpj4x^#_5v4iik9 zrRr+xxR(<_Bpl0`@kk8gxj${dW9z~l#+cxxLwx-mAeHOF8r#O?WiczgnQM1%zL-o=o!$$9 z@~~TLc*q@Y?8}8kwKr&o+E4wI0sM4|Nml=-+X~WGJ7sG`{>l}Yi22%KM%43({N_Nu zlXQPESaV7uEq(`D7un*48(pu+Um`|hL5|b!BrUR$<0**5y~y?FTXXEzS=hXziuhS` zZIf0&BeKFCi|FmZvfT!EX*}3T6^_3w*{6{(3&A(`Kq4AZv%?$)O-WC7|}Z(5M{qjj*ilN zuV1y=;N8_6kY2Wn(RAu{TFfuU;a_a_zx54)?PsC!WdDNSfI9x2p?1X4a5$h;N(sYv zp*w8iFZ?@iOnJHT8^uhC@hFOUtN$Vi5Z-?{On}}#CjT2cguoJ@u*Jp2h68`^^0!$q z<7y0WX@7Cq{|5Ab0nN8fq=u%2n-{MC-8cIkME^6$j?qdJ0olVKTa&Qw>}c28)BH!w zr2}TC%Sw{8|48KtD6Ofrqvb(;~?OGaH|nuP24ExhMP@-*0x7<=lL zY@>fvF;ZTF5CvnJz4gC*shs})ehx5a+Ksb)XMv}>GaA5&%XL-l2An>PDyNNCrnvCk z9|Htc;b4HGws{9Nxv*JpAAkg$z!Z39%tD|sN(9!~7dFss?w*}#?L-o8EyM&_0P zc@P}PTu*yD7_g5f5A?FO=ckX~icHmEuK{;x&)U$JQc+QnKhG5IC(Q4>k+{2aaQotrH78L=h=yv?GNLjmkQDtx8$E{_ZxYP|g4E(&{6c8A8AKN&qps$O}tIA2~~E(0>bwY5VvlS+KQJKX9iE|zAnP&B#i zoLK(7b7TS}{pAJB?Jo^{)pmcd_YqfS2BGfi*RCfbA5gz`KgdV zyG?&^u{G&pBki6w!5w4+>w#zq&}Vwy0XY^LDUz5y3o*0*w~1}gR;|V z>9l+~%G}Z`vn<*^huAb}X1oGP>dHbFSq$+P_wD5@v%fraG}0&44qNc{$syW>qd#sa zdV~9MZ0IG~VA0jH1AwaZQT7B;jf{jCwP&h-<^o7-`=op`X0sK3AO;0CHgA?e3&ao4 zIDrVP>IDg2WO${!p3nSG9a?xKRT{+NK5X2>a6{Q1)tE7zZu z<;3ySI~;n@#kZG#UEKRQxZq;bOIcW(f4{hxChzWx@bQ__ek5ZnT{ZvQi1}XZ;}nq0 z`|<=Xl=bdwd*yr1_J7>{$@$1Xy3s*JKXQsFc@QU1lKO^DnL+o>!(aS;-xJj?wc$xT zbL4zu3{3kb*kgMRuooYHOylLL?nXvKJaXqiW_4)v5`V>(oJT-~LpZ3w50qAaZsmL> z{Pnc8=;cINX=oB@g_ck6zVYO&wo*HPVd)TDP-5Fs(4*IlD{TdB9Z{jYpBbVu!=Sap z|Nrt#4XJ^Std$?CfH}>YGI@#5@6j_bh@1#TmcC`n0C%%pc#BI)Og$Dyr^d&%{oL{4 zyF0mb=jXzeVSKF2TH7hxZP&$swYni&MY~$FbEP-&2kk051-^7}CswhW`S70}>X0ju z+w}CKqC_b8DFaU2Ckq%3t6N)Am)$?dk|~+i^yt*USC3&G&;CBzT>Htixq^F)$SDtJ zn-PiHiQLoM#h%yrsBe{c-B?D6TTjZkKqE^E4I2-0L6!g`&nbGXfPjh#IkH*6Ko+B$ zKy*~pvOIiO@Lu5l?L=(e#1csBcQ!*lZbxB0e*tk7K8$I1!pquDkQ9n-R2LlhYGl;o zd1c_f@wdA>WAF$FaA5T(P>1S2nos7}9$BsJb;uX<=sHZg#^=U#;B;S3{=6)6UKyo1 zoGW-W9sGbmUEW0*@1GAZ?qU=XTokk@Aw1Ld_5quvC5u5o$MOBHVB*7)Tg-X=e+nf? ze0XNbHtjTP3`R@AAT1gS4h{hU0mWiZ_I$8e#Kpyv*4BIrfLRO|c0MGZ1iN?i0Lo>Iv>V#{E@px?I+FnZtoBUGp zAr47l`91KmEM2@H_QpLN+HF6#DewMJT9h`fuXnhqLE?5`Ts8M`sErFW9h#q;lPrUE zDhcmIU_PK}5g02r?Nb$7Y9$J;jvuD;asH>o{f-}k_@JywH7QaRN{NOqS;2*b7q?6) z-lZK(G8|*GuSxUK@RsK0fNOT91(<5ap1xzBU@o6u(JtF{z|xLK?T46A1k{Bo%yR8Q zO0?|v^ZMj<(bH=A*5Ky^dU|?7!3Oyd!f~PIOr*_I`(r;Ak?$61cLqV(y#9q7`L$sK z7n0L-J)Ng()N{U$p78U!#>dARC=;;eRZ-?sxT^$7Dth?p^Fm=V9jyP!-QCPBo}(x1 zrY!&7;bB^b;71uajzGo5WM$Wr%XB_RDxcvz8;kJWyzd^c^1hr4mY1{2^gM^4;P+(f z)(p1s#qE_24DF`CNzwe%;`b|k8zg@DU{SblBuA0t!AfNcIH7%d`E|&TVKBbngwrv$$UlMmG z@kdkyBv2m?zovaoFot0_B3~^>u_T8zZt(i`>(iaeAG>FpJjVWywyzGWs`0r}!O3gvdj1H(vH*mwmP8?N6c)?14H7ZGTh8*UB4t# z1_skdzKsj%TiD#^c_@x;11+O-TGZCrMD4NZ$nX6ROM~ZzLO5!?qT-4CcO{8;g?&4>KexeawEbb@YHFp`OQ>MzYEC*<2-E7@75O zWNCPdE+!i8Z7p;DW$839>rOJ8F3`wb=naXs!Os5s{5!)%PogrydR9j-v13znBb7Wj z#Kewxr>B|-(X}geGAYFefS0+Ro`PdC)HRwtsYz#yTzRtaNIp_jw2xQcrC#3n5>gWR zja-rNK%w;%I(q^I{D?LxGIDL)u~@)LrlnFrr)LH5*(gY0>if48kuuDp3D=_QY(&sqk* zFL`ua${T?LmcEKooxD%wjhl__5bxfQT`i1Bm{#k;l5a zh|z5Peu@PTV^gf%LXks_s1%{6pIC3PeYY6IEoo~9q_ZlA80e#Tuk;b?_tCi9F;>2? z-tXwhoUE-Cr|esM6~L7~lB1FBBC<8L8BZY6%1}1qv_hir{zH%VJX)U*v#cDeY6E4K z`gLo9i8|#g0b}?XL|?IEi5s!QGdi)!;#Wxdi#wi@X(XkkT?jbKR{_}VQ^y$6(YI@}hJq~6h%9!qiKa1AzDW)9+IsX;wG_+)pfu>TV}I9bNno*P!; zSsqZ6b)3`QixteS6cNltO#+U;H3~h4gga;?yQo~~mEyp0Ej6aB9i}IRBSl{V=thKi zi)~DZA7H6!kE)}6bx&X8U0s7SF6<9k(71YubiA|%E+g76` ztjQR?LNU{%z?HDN@_=i@m?NfAll6}j&-y2uN`2y~d>7;18UbayM1>f1uSX39}Y4PicaLHmwb>tJX zKOZt-NhMQN#><{tYzH0BKio6jNELTIU6GBYt4P<6?%{hlk&pwgXG*^7c2RYD3e+^F zMPDinU;5%38xSNvQ!u7V-f?;QR7dXV?5Le#@9_ZAX{|%5mT#|ZbR&Tv6b!Wz#6Jxs zqjLlIuD+iaZG1>Zs&ch8NttH!@rQNscz5@5wPSjy{wxq&L-|c)h@z%`d9u z=6tkBFaCpqOi~Owi?-NT7rZy+>TX#RD>NGdWA$O4ji6_Z5eo5GoHyiwM3~H8XQNJ# zZrEsKmy<{6c6Wt_5-@*p%6h1 zvE3q#*xHrwP*-|)J9oWzG`TRGfbU!40`2RT(?GIb3gxa6@`Vx{<#h3ddC(9kp(LfN z*C|2ykDlm(^g>tzvQlE_emHA;^Rpi#qSqfh7J6~qDd%!ZdwpR{4Um?%FTzMlR_?Fa zPR6-f@UZ;S0Aq<4dab=DIGstlVXI`<(m_wWAjK9z&fPP}C2Uk~9sGTrGFAO2sqi6g z9g^7|IC~q94ulyvU@-d_&`HqIixBC9W2rHAroGOFZI{qP{Lg$-$MO4k+>$! z{H+rQrd6>JMcoJd2Nuz_+F*`vM|6E)9F z3LT9{Honnjb~8S6(^(OJqikc-fktjO^G3JF&zKRDQWWNo_4f%iZp?PFyMBlfVB8zd zQiu%ap!x)k^^y;@yfB#!4!XGW`pLa6xpoV?Y&DfDW9f!bIiz>wQO?Or;fzi{8hLuH z)LV1k=~kh=Tp|3d>OSx`Qhte?af=Xm)&#=d%Q8{1hHf+UcAp-6>swk|rJ;-b+_ebS zJLZK9YeCuzAArgNbV?HxN+L-YbAZx|Iy{nG-p+b|t=^vKawgi}voVemEX6L_My)vY5m{RcRL!K-IZG(2IGl>CN6tEd-Ed^FT#8r(Qi zRl(F&Ljgelhab>RdU=wZ&8HPavuikU{EVK>;1Q21U$(K%D#U-C_!=nWrmewgJ?Q~2 zQV?7`eF?%lBwqzdKrH(zVD>b`C0+Otnw+7LD|)=rcyN}}CO!}Oj~7zT-umoVuN@Q{ zepv`9iRwpqz|9cEqZsOo{BP1b6yI_bx}G!%DNM5tAPTrf+OIgpULE43R&`8={9wBd z4t|@gI9XH`yxhND?=$FHH@7x?`SJ$}NFOu#;+b%GbOPt&LJR$i8zBKE{99p8k*J7IuHMHaf&~M%O0gMbGy%u~Uj6#Bu+mq4eGbP*E7u$P@OQLcXYFM`ez5 z_3A|D(Ah-_>l!xko^~Z%RASioWTn~Q_{<%IzwqGE`1_b9X42#39^3G$^O`J{P-e|1 zr-V^;f)`jlk8GM&H6@I0xa>YrBeLEvTBfcQ;pwEtItsUtl5am8S$jZ>0ocn8z*LnQ zn&nMqKdpN;vIaP<%4?sYX7v@NEPc0!8gzn|*?R;c$kQOa6qEcm@nz90O>s%ZE0TWc z3xxwrtGiSfawP%uvHnY4<{xCBNtKVA@-(+-2?}g=zzco}6QD+>BYw@^Ipf=x@A#O; zE8g-jKQ*QTQX~&R4vY&hMhmwD4Pu&KC6BxywwYuf-pjLvlbGVHB>S{c?pdQ%`)3=g zXjg7yVLonGriF}0-*!+sp*X5t4~(}H9v zoK?F+ZzKJP+mCIZtdsJ5w2)pM%)x!R8~fqrvOlQ)DM30XFH%MPDD{qOvSueGv=7I% zh*T9c+UvTOa$X#i#_I1fxQnY%0hJT5I(^H0x>c8yDe2-Sp@tnUq~ZopE6OzXRS?M+ z@}Zs=OfYkYD;e@17k+`HXMvf?)FvVz+#5TI*zD^C0izm0R&@>m3OXFg#0N|x(weu` z(w2Ll*)yc_I3#99wjGE9TJKpe*z-P0&6r?qjJWZjkcVjtaTNp-RaWqf@FFIy=&C$^Sf#TbjI9 zvWQF~v8!_!&`P}{f{qr|PmagWGy@tuBqW?Q z2whg`MQw3acR%w)ainK?LSg@cb=!EYrb*?KD5WznBNPatzvPFJKo+i*wD3DaH6_Bq zc+gLa$1iE^BWk|eP^b`#7T8Zg`u6ty0YFI2V;{^Hvf_XP=a?Z$)FGzfcYr!sQ|wp} zlo;=yPM%S-63droT=bD{Y0111Y0K({-E#BGOwKn&3sae2n*HVUdMHiw%M-==E~uO`w2r6XeD3L54>4=7k_s94Oi9G*wwelzn3XIf3R# zkn%Pt`F*l-F|#Or#~}~uE$xBA!d@ywf~~-xp9II;)CAE1szUFC^H>iHuWa^-8L|Oi zCXfMzGj-Ky8}G;My3KcUP)OhcaZP@IaZc7A%T1*hWYw)5dy^$Tz~ zcq9nX92n&8d0btHPXlNRB&YH84i!{@22IGMF}aBYWv(xJd@Jwlh1)81N~D#v3D1ym zvArFjT9{|(aFCRF=3wDX^#XEpNQ}=9L^fz80MZpyznGjl=k#E{LN@*7!)z`lhrbpx zAY1m44d3L7aKzqBG!lwJo)n$mB?IzqiQVs4(QZmrt-9L8gh{cvvU%jceaoIsVqtRR z54zK+WZN4GsMcFhb*6n*ihbaqw*wh=e_LL}ydec{VAsb5L@C9>?b^e6fee)@D6D(~ zdG})w=E6hSup0thff6GNe-Q%%xbDV2k}$vKRG7mZG5J4D&t9NZzj z{sj`8fW{A!KLN>d6`$Ol@c4J9nLxH9E|QXjlF=F%aU~Q1QNFpp8$o1a4|_g;#QBN2}R;8sd+4pdtrz}J=-=CE1U)!Sg zmCt7P^qCkLYR+x!JFOBsPCL$iAES?&$VgkbTI9T^`f@(^7F3I!)F!>)c29Q&wJlD_ z`x6MABaxfjzB3Tr901q+Zt$xcCTb3^h7V$~)Q>+zW`h-v+-G45y!t_3_EAf+pM9(p z(iNWJWf_0pDqba)?}88D>az==i?b1i4CMcsQCsXSw_Zav&Tq-C%ZbjaFC6=^H4^7`~d3N&#%6bgnSnjS5o5fd+)Z2_I zXSX422bBY!Ks`eyuaC5#ZCt?j`st`A3`gjI!t2|Lrl+)SS{gk94{7rbXNVzPH7Ha< z34D{Y-ejijWbgJ$qQXxD-S_Lz#$*=`l=O#Bz2mA7jOPn zrIx$!7gdYQAPm@hvLnl3kOJ3^ZX%%dUf$FBGZ)Z5qKe4s#iHZKQt!t@)f9Sq|&ar#D>)i8n2Z{nb>?P}}cfVB7B$tbauA=ReY1x2}BX zeLU$Sp6;t7VZ-d4#iqj0em1uIwkf-Ed(*+DnN1T699BV81U64D@t$^Zzks%zWi#q1f0Cl+J=b-7Vo6f2#5g z2+kGBluL=mJxl5?VQxb)6@Ny;6Qn==HbyUaF~rDj)5}A5^PSk78K4ARPaXGoNte(*MC1*58E5rO;5i-S3XaLE2}5b0s28mz$ZpPxd;Wnb5D^S zY3r-BbVPzI%@414g_QjBE;`#LeXHr)w>&H_Kl*)zz!h2wfs_aTCA%2FT!JgoM*ICe z!)#}B0N!-1F9=Y&{-u4zV zF-&)_@Q^6xt7cM?lT;}Db`G~w_$>9%M@w{RrZ%I|&Xe~H!pmMM&CClR?bE)KrI4W>rF7rZFa2jANO1$)LZN*I)`;z! zUjjPj+b70}O~l0|KlG7sUg0g%z($2-p=Z|(HBe{9%2VQ}e^Ww)gpxrE~yC{663z`m!Rx2S0M5r2F|3lF* z=Cc!3vcQv&=emb?=C9eCx~=OUI=B=+j{`VNfSNRV?!KW)RVYJpn_U#b1izWbGDeeO%-_k{;7A zz(+MkMu@*WtmfL>T!EAf+oS3e>gx)AQJpUL&rj=sk^Ly)5cwygo|x%Wev{pfYHj-c zmbgT@fY2DR`0dG-O)S;&@X%nc3-7~e@!TBqtuuqs@pPOg6KpziSY+2R3l%Vo?qi^Y zmJg=rsN$v%zk7{$i=58QB9B<+%TT12Tt;kn`@i=Bl$N);qA?S#xVGW@11;;a?3K|o zxAe2>%@w!wqxvJy;j?>Bj7*J;xMd`scwXRXl)u>Jp3Omr;m%u(jgkWci4i`?4VP^mX&Pz}r29$>gO=_MD3844-p* z_`n3H(cYIyr!R1IkIN&Zkt z7I7Rt9ngPsW_m7_K67R}6>h@x<5K5y+h-8iJ1FJ2VjuA+;ZMtwPalbTYpb8#j8V9zF`WtR=I z{*B@sx8;=_HiQ`KNuKs*+M_lO_ z-K*cXoR}~sYr{)=D-hP@#oc4#-Ft81=8C%AIw7%fcc^;Vp>tS59OrIa0&YdWFjIH& zOO4WUY}@f3xV&X(ENB>gmTP5{Gvln=4WwBeX!rfn0}_frX->HmXTC{FJWc9y~z}xx6Y@ujjppZ>sIWpG7o)(@=w-|_@Ct6 z0|Jql56U(Qa*}uJvtD*x#eXiYi+ydJk$5!Sb7Yd2QIz$wA^ghtyz2#X_D`Od8mUTJ z6hlSXpZcGZNaJagRXnLFY&@QMgKokxm^X;+cvv7w^<30I7K~<6E`O+6%Di~i^G(6# zvGDcc3TP}eG>4r#1hO7YTqft+Dt+VP>$`AqR@j@=4{TnK?b8eT<{(Ma7{=*)eS`w< zh4kA>8TYgA2DA-)n(kp`pFD*KSDND&((in7;T~8G=@;Z&`I;1vWtgDzM#SW~UUdka-}={VmPxq3?9lYHyc{Nk z++baMb%}S{)_oSDw#TBe@loZ7euky-nMcc`H^xRZ7~l|EhwEIgKX z4;_xDdr;OHQd&tXi;R0S=!Q)mk}n!cC}n8{?FI-%cbxdhSM@f((RIMlv~va8-1BA1 z2Yb#L=tfT6)Ky$%)blTTTl6N5Gxg3=6hxL#LtM*k!#|62DQL#u?k|FWTp1_cNgB=s z?l8*Op|gO82bYn}XLdIGP7#D>7SdlF+-SWfu&?H`*9@b-iuD@ zIoD?l8WSQ%P~3T5Jf6zlU$MkorzIHg-&tB8527Tm2u_4}`|BjZ;bb=G=>wygm^vjL zXNh(d7twP}gy)5sS+iz`xisp@LF`1&KMjUTX(C1 zPqB}lk9AqpuxT#bC(IgOR4=>bTRZ!6)M;R)QO-sJleBW!$LIR>)|=xKQHQ73E_d)2 zZTsoq9=?pT;>p`J1hmQu5DM>Z zcR%<(&Kn1OCik-*liC;|c^NAw>kQF{M?d8qYiH@tieHubV}M~tLj9j%M;W_diWNAK z;|ju~?{t>bXMmQT`l>k;_jKIy_30q{Yl;yB z7qS3D>+`vRFtWenhm|Qe+E9juLWIX+_3al(ZL2a1kuPAe{MaE-R}PB?p^NS31DwiU z*hu{OEiFw>#r`sO(9bO1_FN@Ob7f`lv7;yKiFr74P&aot9RX{2LFLTT4s2|UVxR5G z^o|+=YOd<&rsJ;EUQxm`x8(XSi`-S|18q7QodRK1|8^quAHuSs?7svJ?k zTW#tnjXaPHYyznMuvl|-XY_NDfc|QQVtJv`_XhPdmS|O4E770UL*`49I>E3du?#Uq zX};;6A5g|la#iTzi;Mx`nu$3f(xI2CW-)Q{>5X)Go5|8>8)~B{Jc`wI!`=5^^!0%8 z>?>1B22`g)J$r+Eq15g**U&^HL=qh zAJ#PQR`kY-Y`91%sCJaTX}&sr7B%4)g0jFKPW}rqX+7@SHU6M}pNnhwuwbIOuJhd3 zI7IWcWfZY_#14q*Y``1tQl!LrzS_AcXQ-;GG>M!;yCp83&&*phveRU~$>~{?QaZRs zK>%hY@NsabWAAbnWuRm2=s-}7p(>1n4g=yB*L>&EHa70LdrDt5q=y(yy%05tWj|W| zIVO~ZE7-@)yiDHSVyfNpERRQ0u$hBSh^zvAS5MWWB;gT}M1V&(4+ySEYl^fZvG#!l z&nm!a6f;(l2JYb(yYIvFRClyVcwB|v8(&%M`eaWirLP>I3=mLk+vU-L`A&1~-}Bw) z@|EX@o#dFD@h^a|!Iv$DlRA_G0iv|%7>fh@#gw^2a2E4Y$KSq}M?`&mC6}k8h_7uHJ7hLo+6wz4;$}KTY6X^ek~Dc=tY= z?P%4hrWa=YJ>hkVl1IE;&nIPDI9l5o#m+d_;NJ3eAu^+Rgqz!SI{t5pIzJE27zw(+ zEb3zd27%Di>6n-@7-*_17cb!AL|P5{Q6|4>ZN;@Yg@e3vl8d58ONaBHS6&aAn27(1 z4;&_YBGy*S_@ZFZ$!t`FE!3O)F?`S+D~cI+Gc=x zmh!)iN>@SErZ1}PB7rNEOO&JgVGAXGW5Pa(CP`MWx>r$STyec1=z)=9(`@5PSsMww zvK~qAzn(Wc$I`KgBr$So93A?tjrF$sgXx>47|!vbU-^Efnya=;j7Zi=AWyiXGQv8p z75oR&VB0=01}mHFt*_F+QXU?DCitrMKY-%Xfb55T9Lw$h%?Gt6 zyIu=WymG*pJ%%JN0=m9dhrMxs!1;I(QNS^(VORFvV5an~inpaq+NF-%*SZ|=dGK2gYj|*<$N%9p zcrMb>NZztE=@hG}5duFEn^@_kKZM{cQC| z+<$oQ{5W|lV+TY$KbzrW8wV?O3AJ(qdIw@ru%ZwT6ndcxa1aJ4*a4m!y)gsAhT5}q zTP2Wr1S*VpI(zebNMea`l6$dZ!>eq zaJp3M^P}I2uzF!7G z5M}*?dEC*G6g@C^+rS#F&t_-iIT2vxQbAXP$CwfJEa3tO7l0j?VJnHm#B*=fh{>H+b|DiUp497J>u z8VdO=@qeco4IaEO>^p$p z15blqa$kj$dX0T?@2dhdpn*Jh0V^LcPpOzXl|MxGBX}dBmRY30ICl61jP+;Z^Az4- zfy>4+W!k?D?a>gw=u7=te6#8>3+t8I@FJ;N^3wB60UQ~?e>1<|JTQOsgZ^Q)(P+?w z1D}LRG;WMO|NRb8INsvh>wg)^{I?zuZ{P4R5v14^kx*tdE8+pbT zz4QQiZn<;5f}vCYZO;qk#3<}pI#rl|`ngO0K5P2*R>@p{s{7ads2u(tYz+L7@s&Q5 z8z>^HNPo>-JT{QVXTzG0Ngw^goxl~1kT0;dZ~PbEa4~pckEGxF-`wZ=!yY~lk$edR zzt#U4vSm?G9*jy~UH>CKWBflnT!H{zOdMYUQzc&{T6~|4hrugN3VYauc1rrcvju=s z5>QduL9R@Vq6^2G&;9Y=Kv1_wfxCOnL|Xx~dz7e;JfXI5k>+(EpYlUl6;zHp5i;uaR5@;grSWcr{$)}2O|Mbe}C~`PPW5)A z2-)O+f;uxc-9AbW?9Ta{E`YRgAn#c@by&daLl^1o7i%ZTFdsU34gcFaRBK=f9J%9< z?ehPPx2W#>32aAkdIp8ri!c9$ufO5##lo-ZTB>38|HWE-0#WV4ew%*fhlgZ`?;M)B zGm}Yt0p$lt&TEbMY;3zewwiL(Mcvo`U>qZicg(4tdi&V28fYM(1?yQ+(}DSKBm zi%*Cd5Z8OhLYW3iSP;p@hePElRRyN(buic8gM1g8p_JD=wr`V(rRD&ga4+2WH7Mr% z3w#+t{tg=|#mHueOROcY)BA%sj&Qk(hmo-9Zr z2{Kwm-%mPTzeTPKk~~<4^o-B%9vH;fA*U*G=H3MZK^+q?AM!Rl1%@Qhoo9eHL5Ufi z9riP@*Osw^c59Hg8A;~EBJD@xe^XUsZ?9hSphJAD!iNuY@G8Fk!NCP+6$^yUfX;xl`z{2?hA|-Bhc&7G z4bJ60@Vsi^)$G9a9mBaa(2Ckf@!)SOev84^+_s;DDKU(l*J{84w@_+VWa-`I9iRvT z#1;dT;ACZmffWJSU2nFmP6B^VFQ{XBi~&x~i@`g4@2xkI0n)!S$!ijLGFAz2{%=(P zOm{0+Qu(?x&_s+qkO;b1X_O53d#66~xoo~tDo2BFnaoMNvhY9xpnmN){CJC1&LHKh zSJDZW2VUGK@FAdIR#svm-M+5oE$)9Hx!l-X8XEX)PN!QPwc*~dhkk#U>>k-JYZqKA z34@QpL1UZ$Y$_mhD)49uwQ5ls%mpUNZh8`sP{Y|9Q1Y|~17LfCm%h@NB&&x1!``L2 z(%&n=fq)GiU79Zjv4o~tb_yt|@@ltFkHIr603+-F&5%B`u%3Hp(z25Og=0VZm?s9x zpqxP$6cW9I3i$D@-(d0DH9x|UVV^Y28-Nt>{rOdb6>6(SG&0ZMO}loj5o^Ow*4^`( z(C;H~4TY-|>jtP!PX0cLuHn^w<75G=88*njxz;u5ZP}W)-$42G=5Kxf|NepoTpNCV z{{j~SRn;aLI%%KZn+x`5;(ImSQk8{Kfn=QGcqe^LWA>97?Aq@JuOrmbto7RdxtrJe zrCaE69_Ngn=EkUF4h4ikBwV&3N~%OS+2IoE%T$ONN$o`|{alYF}ucEX#fA z5`1%RJWq<>XWPJss`_|v?W5a`(7VrEiGfp+|7MMEit!2gQ91(d#@c1Z-D}d_L=2_s z`gKd|p}XVWkBAFGog=aO9<2-3ZZWYqGNQwa2R;F}Y5|o6KjI9!ayPnFylSQ?AibpVH-!F$)KJ8-lJ_9FpdZ7}!H<;?r~rTm9@fO#0fFDF8HykFKF#lr)3>{uS1eFcUK*`!?D7qVfJ4OopfUx7N^) z3LO&V-M*7qn0bFF(jO4*#7M^u$#ybtg}+E-0LWJ>x;lDxz`fTwMe*(%&a-x;sN^+7 zjnmXHR8@N*uF91!KMR93f-u0!K=H5$Ov9meUM%l^`d8AJl>ci}!wz+5#6$>0$rg_% z=ek|u9aReKE-Vxq5G3ke)jQYv%e`h_&g9K(VY?_tjf;{l-m|2BeI8-CaN$>bvM^x3 zaRc~TG^l7PKxr+N=|}8&Gm^i6T$ZF*SFSKuWcqkRFdXlvCe9Ewmdn)C(w=Hv8*T6B z2I!i}Kc<^KaCoCjsxvh{^61%3=!MlJi-uMgzCv8nQIPf6Ehw%3XW6;RBu3vnCM?~I z);wMMGUhMhjHLi<2xjVy8zNqVOnNerAtAOyXKn^E4Tv?iMwX7K@zmE!Dl=zY>hAZG zjJ2Q&2#7R`bxSD|>R8_Q|293<;=5XFsnG-PS7z)&sxpzm!3_^w*ST*zKJVU`(WjMiKPH$ivn?1Ixf(xL z_4qmKT5I(j=-MBkPFES>*4gIo8JBxoUNYBtpW#E-W!Xtaew3l8?M>@*CrZgj&e$_G zmtXp~HgaIG$-^TSZ4>4Qo9?Wd+QB=!ZXwv`25rDO4Jjz31bhBJ;oJ| z;_8p})l{KH@+jHU2u`~v+;iTKbgGCKdP zXE_ri!yaF8f`Wq1^orlbvF|Jqj(qj1OO2_iGx3n2GdO6a80lxm#KmpT(DK?mO@F;6 zpHa^&E>5*#)?Qt=X@scXAq`77%VlO_Lfe{h`o?uoxYt{(X;3C7-!@y_d*-JTJ*Ifu zMENqeYbBp#8WB_V4?{{Mu%zq#1&os6q%+^FIw6b9A(31;tUu9Ue4~1Kd)tPK^?By5 zRecoPv)APO(*{rae&3=>y*%sVYOlkVruSCf%eX_7A|rR^x%=04j?hwmjX&3BO$g+$ zhPp!M{Vdu!f|j`gWkHA-nP?={-#7+ca~Hr{fs%^3@vc)JtOzL-ZPv&0BIUSZ%DNiu zr!UK4?tvI~7t0rtQVH4A2!Rx~Nc@`H`rC||b8~Y~SRbE8&tOBXGT`6RICnGf9XYB^ zf9%uCC9T-1mS+rFXiXuwnOhH51`Fg z9eOgfR^=2h;RdfLYA>Uy;wtI*Ge(l~MvpCgZ0j2t7)YJod}h1WQpSnu+P1G=ySqm* zLIFQ4M)Ai3E*O9~)8(9Mec&26v68^0Ka>+qqMYdOU9-mpy;ZXADm>v+@s*)|o(xHQ z@Kl|1yW2`wilNn2q^b6lsoP3#&Z1ub;`g%=JJ5}3B_ zqn|LpGS76GKij2JWOA3_wG#^*H%C-VU%Sp|@w<&1cLxsC`pViJsMhgbfWNF@UAZ)% zUod9YKjZyrGjU=>iA<_UgU+(bqkqF)Vkza~?Nd{~^Mx|Q$vV^AL&u5?*Oe|+gOYJQ zL`b-Ub@s;}6_a&aAGC@6@IW+-+NrJ*v+Z5Ce8QAc$-tX)tXxc^4XqOyzML!nB4_<~ zn#aZPiE~DshJ$(e;8FA~dip~7ybb;e&Qd3~jw5R+yR$KLCn~42^VHocN%N;%WlszA zO|Z=1cQZ$hltDX3{5;_O3S4WrxVX+?D=Srld**vmQ<5a)I$=_UP*936}PU&*aK z2e}z?Y(9dWF|7tKzw@9eleZZCLtyqT9E!{{^*flW?Lz@4^w)|^brgp4avjgcGH#z2 zPh>P3BsY+#i`+UotqTd*+SiHp%#+&W#MQZ6wU>CTI^}RS+S24)Ti0*@Q6dpFNj&#) zmf@xG-9ZFxcyyhVuV+^}sHakZ1+#YmeC-nD(Ed!Y8#Ms_*Z}iYnjDtDofwb6mFau< z`K#nsKRMhRbGK3d-eldq55x0zd+REE-J(IUFn7+e?a9kcwTXqdGs_Wa5Il6w$T{pE z%iSTw+EDmhY*GA04l260BfER1|HB(gn(DM4^SM(_@37U}UGPJNilk7)Fex)5vTPD#aCTZOQ&6N6;1S@I^&21z2`Ji(LvU~VA z5i`h4q=3(!JdzO8-XX$UzqcV0Z!oL>RYonSOP8=6>v{v3){!af0{2R}rqs(mq+w7M#^f*Zkh z%+2<0{{Uhvse*hWApO$+GinvgG^Q)iT)7!l2q<%K*!Aw)<(ZzsJ+tfE?d8QrQBQ_DYk-po z){cU0>ufKZ{1Ov|Bq?G}r`^}7Z-cExQ?@8&2nPu~_%%Afa7_QbY?Seh97ZHFU29d7 zlKLjaMh&&Bz7bF9f%LqrwiidYLy2&^*DmM=#);$9}^9&EgIg=&MJT6x?3$kvm zXY2qRJRf^sAetBQb+DQCK-}U&DF66v*NW2aSaZVUPZ;whs^}%&BH5a*=J)Rbw73eV z2qo=!&6Ll!nXRTev0w=(=BU|C!oURS&$uMZ1TDIyzI6sEi2U3AbbV9R}NbRAt?6vxI$gOq<+cvxlSLzm zqvs^8uiQmwK+^sjJIsi-yhT&hZuL;e9&C#FAyNAs1Bsuucedi9|uu<%i zyI{ct+QGDv-m$#mUX+6RXWqpw9n{&4Ts6K>oS4<(&uy? zY6BHg5%INUENm~&{oaLGPb{94dw`87j9>>7sAzl8stzIl8OiAQuF>r?VDR2u-N}H7 z(GC(aD%&uCoj_V(V+;Xce^fMsf8PTZR53%2??fAC62{9bPG2&xUFLPIuI}s+>{%=% zQU$ebG~fm7yfL65yZ%>m!J-xc0eDwz8b_*MY9?E^3fPPC2CRQU%On96Fz1TczXyUG z4}SlC%axzrv8`E z)t3lQvC?aG*7d?UzDgM0Pk=Fl1;%wa*J58AeQr>RM8krSor?SLSFrJtCd;&PcYZ%8+PZj0NIWY_zp+ zZM0{5zie6Y7V+h}a3T@9@FJf%U7)Z!DwF zH*~R~ z*fO6EuNSK);<7BQx~vbwP2T>f0kRPU>2`CuH+rq1pGveq^*hiVvEPb{bjW^_%X%8M zN9QKzt#e~m1Z9Wo&phU*kz_DzRZS9AC&he5()HJitXy%XoxQNEEUmCyq`q@>fVx4O zeza$#A_!aPzlSaVQrJwDC%U!xK}3r1ZS&FvYGreqA0NVe6Nay4t$mlP{c095k-PgI z?bfu*OVm1hF=07!6mkr_-`hp_+V9+WPoj3|k$}PA!=Kegni)DikPkZ32%+meG*jqa zn5ZnW6f$u9+%%+;!5=rDmuCRn z$D>o8NTmHwyNhbvcShPp%d2%=Gq2CoIf~6*VEJJ$&}RKsp!8R|kFh(~?XF+a1`QE# zQQL_HJeTdr>{3aq#)AsMN~hD~%tn8(wT5=v3r!|)Crb9WJU~|do8S4{gAAAMwU;Bx zX1z%P$3L6Dc^>!8RGn_q8+cZC4b|zYJYGQ0yTW#<2P+3eNdLP?OUUa-yjasKweWJ7 zj`pD9B{FB6tnyUvcw`WDl-7KPju)oO6}Tzx+8;L&ErRb_4a;aTzb;0TXVf|u-h1zS zmQk@?(;B;-xmKrP!PeIOIbr_RV_bWJuBwb4$BUlk$CVcxI(FxqNVVV|u9w{NOF=91 z8M-tm=b~Vr2+OtqtvfTrQJ6b$H_s!#L~+`5?o=5@9zEI3m#PfJmzC&xUqE`G%``NQ zJ<>lmP$)8MhbTxrTdGuiZ60~MsebxIVs-=>gBS1VAC`LPcdm=5-#%iNbe0HrUcUTp zfN5ZjLQd!3&!2%NbJ&q|c26I7>%wLp>%8zyq~)x1wC}Y)twpdH7$%lh!$!r?n(g&+ zy=>C;1{lAHV~cKbkMbpKNtbs>-d{J_EBd*#i$+)at9B+FJ=UyNaFLyuJ9@n*+u94L zn;&l(uBlB;T}T>M9hwtbog^H+W2vB-Y7wHHo@M9+k8R1Ia@eG>TYaYAP=JL8hxz>O zGvQBBPTH5EXytjVQRL^4?3MBskl3>OPSWKhm)Km?1=mK zyuC#xGwo&q=IqlJc9ovyJkQC6N_H8m%LTz{M+ascknwqXt;A60_tzU7yCrnC|U6wu|QHgpSkxmJ}+Af z{T$l1as9ViuxbZ}Z1k4k)uJfI;^alTRNP1!?8Bk9Ryk4Zr!w;6zIt7oiHy8s4}?fX zzw+&*P)?TZ=;tt|SfpQrnYJO-+lv ziYK=2V9Z$tl^<3~sLn{u6Zs)$Rc@F(1=;aGI1b)owLN937;|dQe5q5p5bj#9?rsd? z%xm;Q!ekK}{O{Dm?(FF?J%F?JrYlST#_r<0^VDJrN6x6nQ@(TqQEFALw8;7=XL1GJ z8QHKc5zVNHON&-!>vWs5W9II(8oSBMi}(}<@4aTB)NzNU&KtF#8iVVn&KNH@RAy;L zk^j*+QeuxHNd?Guax!pEG^4SYh-NWknSNbikWA1dA|akvJ`+66tzfS2*8p`Xrcu5H^ge1E&s2%#|I^x+$5YvMZL26n zkp^StA#;>ujv}*+k*Pv5XUc46D4AzMh6o{Ko@v51ZewPOjm-174d1$S-|zQ5@B9Aw z{hr@`Zd-d_*Lj`iI@fuuwT@%i8<;hDH@NrY^q!w%s8PF@73*6nUhBiAl-x5JWbfoXyw|v(+;^PG_p|7oMd6eWf_0(X zXb88g56L&G+<~KpIX1Rx@>x_e5O%_%DP(fa!x8jejAu9{p&K^dZ^S}4M zB)#}_5)jT{&RC;nSX1FSo}J&{$lF=24b^>v;Itls8A9*jWB=<-msEsFP73u@MmkB> zt}R^_ecXl5d)kDE#chDVZ4A4JU(v5IFYLM7lN(vSfYBOuo(UULUJ-Br~i>1l4AV>`TYhVU(#hE zNYrh40YPaNhJBUdTHvkw==8pqY?N4&R5`Mt=LLRTU{(6UW=+#Lt&90B6m73Z7wYdn zv$2#W8HU*(kLpf$TX1%>x`L!+4<=t6(%w1iO^i?YpOEadtE7P|`pCda?#1r&p!Hc~ zXnc4_?2`-xtDd|lGo(9D*_lr7mee)PX*?YS518k@H(54Jurr%R)IL$SrMt2@C|q4r zdM;VQjsSn)^#3}Y(hE4cv7NH5f~K0b;ITX>_CzF$*eqM%k$1&#Z^e5KXMO0|GWRP4 z{Z>{_cgts}5q;WeRJ=(&TF?IU#N1ITkn7Ek=gD&?mAPx>-gq zc!bPy1YBnA_R!2r1^ORO z?GYC?MDcf;o2cml4@&(9J%P>G`4ZNzZEG2`L9qQb-Sq%}=L<1n_SCvgr}Xd{YPXCmfv@s)j8Sj+ zD}I;L;`?0T{z&LHedfj3OM16|^B6PXw?W@TQ}G2Vv{%c&=s65|Lb1sf57D0Z%2UkV zlUy{v92a=CXiIYRxmkrp!|J@p?$*i6<{@Juzv4T`2C~~>vKILnc1Rjk9!B5q$kDR#|PjR z%|j5^dMyC1j1+@`7Q+f$84&1P1#!)5gfn`>By5e5)0s+$Wwwd5`L=IBQl-962ArXr zv&$zC@(aR3f$A|rbcRdN!8of1Rm^BdNe~Ibapg{R!b^wg5-MGD4wqf@$LqN@I!|k&QS3ANZZ2)67GY~>Km7?Ebk)LfF(ufAJIt4CfA~) zYMy#3bgtv@S=Q;g;X>g(d3(og%@+@~{0zRajuwwL8ePY?Y}vA445 z90+Tna>(1zT~E`+NG^anOR z+44$PT$f~BZ7{GqmO78(W95}XM+CGF#<;hA!YB-{E)=lrM_jr$RdIT?L7fwj!@&^_ zPQH!~8=Y)42q5e^b^4j>)n@(E{7nP6d(YcuICxcGr<2#HbT7r${3xGt&@3a7j>zXW z;2-6-H2Z`3Ld@Txg6tQv*Q8XxgnWo+^q-O6ADr{q=JT8#a!X)zV@4;zecfXYRXNs@kkPG`LXsDC$DtUy4BC6T< z4Y|QF9~1`-RK3a`X#5)E>03)WW!mSgU_)X^Tu@)mW)W7^vVa=-5}aGL5)ipojX&Fx zX1`pTLFK-A(D|~(95uN;a>7|x{p`W&ADaA4ZCg89Ys-<5>5ZX0o#~RC;Vy>>wcB0p zRp2B3kRag+gWKde2kS`_Pg)Zt&+|}JUQ=oUf87uaO` z?BZbICyijug@_J~gm;02Y9)x!n* z7ja@al`CC#h)+naK}E^ivO4yTe5$ABp~IuH-tGX(rw=|O{W7uW!|@dLP<-;;sYya1#ywix!OE@wW11X2!r z9pr|Ikf_vp)>0TX@Io`YAv^|%AV0F3O==CE`O3ps2MjCg@GCNl8pX~V)e^)d4joo{ zVMZmqoyN9lZT`sc4=CIK2m52pQ2X_q_0RFjgpzHrBA&+^iYn55*Bh5;pYQL%iTE8n z@2(!>Z-v*ew?dFF!}IxWc!w60x z%#Gh|n3``J4gJ> z1$b$QSOgofP$=o5wjW3nn1C~=Mq#K5v_~j#B&ICfig|lQ<#7ZYMt4~5zEJ_be?GSfQ9!jbJ7vAd_C#bahU6$o$2!kRfBf zos5jrucS5g5BoGh8p#ng7I?cKuGM}6MGc}#clpB!xnfqJ>9YW}CyW^mI4SNASE_$* z=fQr1(uj~rcQ)&so0zGozk7s)0ji7%nLuPSM_9UXjd>3b%=1B8Km9Ps?E_Qa`+#8g z7$X{lARx^KMLFUC%!2QLxZy33k-x%SA+awHRNw_pgrNJ;cP|-{noWx<6c$cm8?w<% zh<+3(F$yqSFiDEjLAgxaP?V2GXM%21r9;^1O&CZ%_69xCwHF%pB*t9SqmEPzcLF7;27O${T}{pI{?OHyT}>dq;xxniKG$S zQuw!s2*k^P9u-8@EN)L@fX7A#d}IVWAwx<yhEceKkCehrqYP-{}cQs7OA=(BjmC zB*T0-ky&6kv(DSW``I*HjM-(_URf0sCWtlzNaP6hXx#ZU= zKZK%;sAkoy>|h9tP`+r2Hvq_x49x5_(Ay%ie3(QE;GQw&+RDoIvh@^~iGcj{mwMRJ zQo@fw3Xn=X_;=H!j4g*5-}rndDllm#%w%otN~fRSiwt5Z^!B&tXr^_`V>Knhdx%t1 z4oFOh8ta=u>95^8no5DU<3eIV6*|A?@?InP{)D9yGU{6VoyTipqdzF{h02e66MNI7 z!cn|`Cft}VRc}4YJ|9Tgw}%aH+{mTTX}#Pr)cgDF{C-+fQYXSW|A&Xmf)oJ4Fzi0> zcyFCac03rq|54>5yK6?;LK|jlTFw%4gP6)B|t zoHjfze{kFXWrW0iL_Y?m_MGkyQId>4$+}do%Exkpcgr<_^(qOBEBToZ@N?;HL_2*s z`;r@f=Bp6#N6%pw~n5QBT=2w4SU7aP(5({>7Nn^$zW zQ~?^id~yWaF7WR&0;#LoG<=_>+5HT^jw|UimxcVWLta9JUyZPP|L-SR(TQ!0_Loe+ zJm7y6hgU)Nf_L@xramJa@*me*crMR72m}s4P(8X!6g}Hy6!~X@fr@)ohXe!mdndfk zZ~zhY<&$0Jl+PrEzrN$d(MdB+Ty{1{Mjl_N!}}dD5lJPrzc9SL;1?Y&X(!;pXzSHk6 zq1jT40#Bz9X7Mx6PN6$kp~rNRO~4smx8{}(3k0S|zO@OVc>CY3jc(J)$Fy&EY@1{m zO$lCaz5akwp36c$BRaQtzhrvo z;1?19no~3l8K5Z;U`hhgl?VR{cl!VeVUg_)(9j)i^k=v21zdBjxP^x*FDKgQ`|u2B zNs{ZDf8#Ctbgr&T?}}%MiKDYRR60vyW|u9vbG|1)6en1M+LKpM+MX1#=wNwZ-XUF= zzHeIkNm`}kB8P68YIup}yqHbX zDN(0u%S&8+-oDFT=TY7L-1LCQqh(KjcWBLN&PEHA#JnH74R!x2oDSqClOkH8-5g@c z=Q|A_e!hpw0xeXkDSG-cyw4t1x2s4sK&8WLGR^9BG)xo&Xmc(?91!tWoHNUfRzK-Q zuYMMpE#d+BXDRlaH`dmUnmok&Me1`QqNRVNOw35J8(B(JG2p=%zygSN7?G>MSsh@3 z#vqM(l5kv2&6nR(=P-u8xfl}#u!~#Tq9qK12xycOguDfOApq?nB4GfC@*hD%0EG7R zkxifvoQYYCK9{Euf0as6W6~2i|K54i^i*lfvhvLJH;w0JN73MZK+6+nms)NKK+d@k zqL4$^kJ<(nL{K%||AY1rfrW6C--*#A722c;^&l#=$Rs8;M)Lc30k#VzGh|qV^$aQ} z5Sk)s28t4P<$qc`Ai)I^E`jKm%tbfU@-POD6$FzY8mR+$B*cN(Nf_dd_RFbroJ^q^ zzybiS-Baj#gdlhToh1+|sX-W$3EfxjMhq}6E$#Z!*xgy=mGta)Cjo+LyIBLU#PNlm zd++@;ozESOG9NPRE{-{rZ7m9p1S$tKGm!`8?Tec{jyG>2`b67dwW2HD|_@2)p!2}DkGFlRLv#?ko606W7@zUaRYb7aE#0nVN= zfirv`62Tzcffv>{&v%ON=nyET&&>X`WNaK;6P@B%{q8U$TjP=v08IPo!Nx3Cf1 zu>Zaj1yI_N2m-9h7(?3#EFh%G1Ry?SW&xi@;S5@={g}P}c^c&^dUCPJJ)?x9PDxLB zi-2c;Yx5Tu(AWLx!A-F1M)+T~Kf9v?Ms)Cl|5XCLq3$IQy(FV14OoT<=n2Do#!4q< znv1`}%Q-;w!(1tw;3UBOq;}dZ4M2~QVJCRd(5}b?8F-Ys!58B}LnAwyc;IjOa2;C{ z;3;TS+1#FHN6*{7oqzPZ^i<&xdn$oST3%?;$>Sif#k;SJ%Tkh7ap5UK#LG?MU2!$J zq;Air6+}fYVCNtdWdi9{*nsyt0QLp3I$;TfXj@EoviI=Fysb@rxdeVi;U-O~rw3KC z^-liF7S1styM2j*jk)e}_&uQ=FQmihFGwmQZB#^b=WVl!3jDd7!JZ~~aW6^1aG}%G zt!9U6?8-}8LjFYLK$~>tDtgq$m`VK_L4eGtPxBkknXwCq{s_AiHa1yu(xn$|ghXga z`xxf9Pj{5z=tO$9AiYr_qQ|n{HJ?mVW{?4@rU^`UEkQ-jz(j3;EsAjUwb6s4!I^tR zrm*awHx7Taln(y@G&FIxG0&TROCt<=GDQiMe)H9vn`<(+&$D;8`>LIgnn z;Yn?}#fxm(29iR^aV2y&Ectex4mQ>FT?ELLXc6fWsh=^Zh3>>3V>ku3$wp zE1(34hR#Ao$!t2qr7k%kqBJ=ivFX7j8Hleycv|2F+tfnmvYhXevNtSl@0;wA!Z0L=9@0CrIZaHElJdHBUEa8#aexk#H11!N^l> zi1K#R7ays(RWX4*Cc}_J_qiMP+0oEz%1P1_7x{6!jH7AS2*`h&uwDTdV=h8AgcOpq zIGR}3>o>#sZb3w0Ut_!SXgb~qq9;3Ii%_J;1s);Ue`xOuMIrx761jAdLjX$g|4#G=7``AQgI9!aD{n+;6Uui;jqMt^}=-6iRe4Oe%58 z{OR|0uM)2Is*$)ZWY=pUalg#b#rXBBVMY4HG7&QzyU*?#vluD1BPa}CNakVrHQ~w> zMpcU)5;)@wkq%P-P}mUW)taq;kWvk0azK)8NQ-epKa0Gv4gs2P1or=J(yuYbrB{oL zD<~X;IoAOW{M)jbH|RhJOT%IEEAF zIb3I7oQVq0}o&5u{eK9@+!X6pC;X$&Wok=f_=VAu}J48 zI2Y|c5+E&nPOSms(H|_avY+H)m&Z{N!qY|8CYFk}i|99@N*IxZax*)xEc6Qdr&}Ku zI-+SqL_5+1y9PHDxlI-8^$XfR;Jp<8{g7njs8b}r(8+{%H>FRKf-5?bqS2H!`(Nuh z_{=&sR6UYoVzoA93c<*^5JJr`6KEC2SjHs=S%q7M)zplM6J+P9%WO|KWOST3y6T13 zm84kM>_NPeuaEg&&--q4-0uFHWQYxgZZY1KWYw*9Y=2#K>fD{4Wgb>i#y@5Cvxk5Po zcQT^7ZW{$I+7d^xz*cD_j$i`5*5@C`mnF-*=PP_kVQ6?#V}11B@3`vyiS=9WpRvAN z!J7)8i{^YP^4r^j<+M^uieVydgjbLc`?@k56^lv(2`txS_HWwEuRC0>5d8a%Z2Gn8 zHfrYWHDs|uw-l2>f(1n*P1wtLm8lAntS8<;R*4({JGhTeb17k*#R-d5Aa5_;Fi4QuN=@NQnn&!sK`XTN!~=cr=hc8E`BWVkE#c@GK!PP$|xnTq=C^ zr_u2bs<_A3&|2q;XFA|;4iP}|`N(f-i%#lSs{8PiLLEl8FLlidsk4;);ral7=CHja zS8|#M@Of-q_;~UQ6iG53?x6Q+J4um3{lFBYQw z?0>54Sp9c&4!wR7kPe=I0eDW8HnjAqJ!();8)2`BlW*5t>`s2=Yw&L0W;>i4y*6h$ z&$bj&s1g8$T)daT!0?xSlj(OfqYV2R)yCDj{atuozVaLpPJsiE1#R6wK~jX1Y!#_- z;oVbsFG(4IH2uZ?Bqif72c|l)HYl~^Q7fkqkb*IIt>0LiBR-4p@gaeZ#7YR%g}e^1 z!ky6<@TA^AO$+a>}e ziuD4>%G|z+REeHj3%O2e2~Z6cGtLID8M0kW+LPj6*^Y>k^NTT77BAKoSop$dwQ`Vc zxsSYRUkO$22Nq@09R=BPQ`j^GwK1_ujh!tcg>-2)@jx7P(@Ofv((USrpE3$+tXBCS zzYKA5EZfA+xnO*XEUMGX{H`}!@-5~Ip!-4>8Zss9ex-Gf*53sFgpeBEJV%sVF{L5nf!y04#$*C!&tqE!Gt&`^H`d~Klj22l7hmB%e?bpHTmQMiV|2Ps z$#a8aCAh|=ip~7#wLQLJd$J&6O3KmvIoCtO>szhW+uH}8);KqmbC(#W#P-C6H>k$_ zeV*ZpJ@%oH>ddw!9(`k;mc#w=wiY^d^W4sEva{cc@A{|y8CVsSD^Cj}u zp(1}kF9|fpM=>&{0qTDCf%WOHGd)*)DYiS%xlwuH`gc;=M0Vv#a)Ty^ z7}=$<>*A7Amg|>Jgf`u&1LK=qmU2g)iM^B;_Hc>Lgc4H>SMU`?&r+~eIxMVbjDxXs^tX@|`FJgyo zx%o!Ua9^_152vvrcU^*5Bk4jRAgMB`pGjj}Ph}KVf0wN_#9F0Az~lDCqEP~$@};V8 zEIkHy)@b}z2KfmM4*NH1X7Oa{PNECcacUIxb=nu90eSjQsw>Dl)SF>x%AK9%4JjD9 zPa-XeOpxE5upNz!tr9+|#oYn(Ayv@7F{+yXd^3~* zjTfXgY<{`IN#wxXU=t7iA!%myc-%wc%f5Tp(z|X*!)zRM+=&{!8VLJj71GqBA@W_qJ*jKB^u9a>}A{(64}@>|2XhnFM0?MRi_yyS zy{5kt=D~oDKod}>HxtaU1iUNpul%;dH?wIeX@Lpqi&G~nY+VU3wkIBw`NHN?P(bkB zyo6$Y4kKchz`}=O?B+TB#Fg(0LxTdHol=Nc_NBc5GsI_99|d)zu;-E}YhxHJ zjM2QX+*7BDyFy#yv0T)Rm4)-iuuluOB^!R&ayfQx>w=px*xrrt=fkJ;7M=H|5jwn- z_2l%8E(UY>ILJSQF3!jI8M=bt__V%icXU&%LPOVyY8|6{(_VsWoLLT7CVL zDf)*0dcDdBDRxLBjCZd`iA9MqSHgIiS5mpmv@wI?dd3{L-1*a-PcCI;5S9$yj05{} zSXT%(s-Z2Vn%Epz+i=3O!`1I^ox{WP*772+Sb2yaC@Pbts|jtBNrd@j5Nwjt3oDHu*p$rxfO@AmaGcgkhpS} zlok&Uuk1AVp3sfA;aq4NRB;G$&5eCq!c{8{G+UiTS_EtO&A!}K1xYQ_ezFRBA}LRb z?n#2myweS|y728sANa-zT$WbKvOdUth)H^Q^u0F2pZDb=Wo>?b+TfH7=^}4`L5{sz zgOn=Z8#3Zd24SHuZmPj2E@h7S(vM|$>v`^;z+*hT;msgt$;CxGQM}FF z&Qx7iG@Mpz71g7&dui!$$WhkGqH=a;t~{$=876cg3*qC!*S=U`-jHMI>h}B%ge>2~ z(1m$WH$FV+GqB8rW6ojm)$|M{EFJy<=(BtXE!8{K|IzxD2rDA$G~1;*39oupSSudWA@GaiKy67ZR%~cf~I|5xk{`++9hH zNZF+qQRlL-41&pp7f`8i`O$10r$m|uOWEm0e&Q=T6%Zc51>mpToX6(#xX;Zq3VNze z+6&Pvn3dm`gNcj2NoBUQnC(fh)4F1voJ{@i4XKZAcqJJ%?@T}c{FtL2)!?bku&
rad&Jz#3hllKg%|%`aP<(<7xN8iKipz~xy*NFA~;YHOdCd_H6f-&Wny3M-tgdfB+8UF0r{T|Z3GFOsgHkNnd#L@GbTToFBL75$y(?36K2zI)546+Aw zRdh+BgkP-O^e?WmCrUh=!HAQ~qvZN}i;CyE+uHk_=Df~oc=$Y&h8fEq8M+;dF5QY;W_PlPrDkBI=T+s?*Q|aqWU97WV!ztNqVWn`fVk7@*AZ z_TPB!t>%kMoDz&&EY1J;>0^NWTA5)*I6Wu8_am$PB0+KK$SP|O$2m%Vmbkppm3CpN zQNcc77er4=qBR9!(1j@9XZe5e5rWh zi_Bohqm_%_LX*>W*Cu-$H$Gv~H?%V6W_7aXcrv-^a#x+t-n^mxY{=`u$JFX^a{m?C zMVhMViJC+WEe+|I#y*e1WcnP37>A?X`O9#)iGsSQSF6CYn9$A!BOvTHF1j%0@tH0w z=lvGv*n94hoU9L7Uz9EqJ+l}xA4E-EF~?1PHte~^v)3bSWhkh+5m#7y=GFA!Qp{lQ z`a!MJ$q~NIT0*Q|Be^R}6dOpYe+k+D6Sk!5Sb&(*!cRF1*Aoi~_Q#^tNM14|!g zZqL1pcu?CSe9nN{^kg!0)VI4G$I=6KBMv7shU?PrUNb*x4Cc zS8jB5-G3I|=i6W6KOQ_73ujV-hJk@+x0+r}i|sLV+{B1DO2Usnwkp3;tGbE2n#Eu5 zvud{9XM}}G^WJ4xm(|*D^%glyHJi=uz4uWz5SMAJvq}%G zvxc*oN_buRQm^CE`+ZZV4mUQ#_k!Y{2d0N2Kk4{8z!WsJYF=$>%pYh<%^PI?c{cN5 ziDxesGuHUNJ?k}3-wD^|SNXyOI{c_$J#>k*>BFaIW?ohLa@o5Z%Rkm=CB9P*aQN?W z!;~OGVRZOadvQdUdorgvAp7G=Vt7UVi>s?jeIyR^_Z#u{v0iF-E=^tn>${?%wJ%(= z@mqLMugv_V+}g6SM;Zb+Qp?2k?wR!FcjIs`?o02CCd^(SiQ2Cs=_M)hu7AA-88xqldO%>o*%?H4>OAH3KIx@y@AwL@bH54-g#GB&)k%T z&x3gOmp$K#W9`hYmvX76jAOK)XY}30k)lxXkHd0H8g-I39_2{}BHGB&(wVWVxo*BM z?+W%yO+8=y{;MLmy@x3DuY!aMa7cjWQo3ZAte#!#Za67Yqh^!&VD3RzMkOOWQ2=s{ zLzfia7jVrg4yUeBG2ClKZ=8buB!lq4Zk%BtAEtf$1hUWhve}$`@3o_(v$e}5vs>B2 zWvB8^`|mIC_Y$`R{flkcD5D_-k&UDVc--F>n2`KEMZ@Ki>5Bk!*4XI&9vy|9By5jO z#Y|#fH0`L1ITY7=bZiizTD`;^;pfEmHiylY_qKsC zXCAs`u(oAKL&SgHRrNe%J&?eN2XrU-6Gcc$4)7Zp;6;)x{UGj|EyuM*Z^Sj!l(tWM z+f_}j?r`?F%yS872;%g#{W#C|wjpoz#ZmRs`G?Oo5MP~X zq#i`sy%EC<7@O?_MFx9x20Mg)j7S*DH=%vx^RDs#_7-heF|ww?Z8kS^rfoc~`8qXc zKh`F(_OpG!}+^cHJp=V_M@RN{qrO6FxBea_M6Z8wl{N7&9W zn2vpaYs&>D;hB^BL?Zh#`kB95$@nXY>K6rdGX;0~oq}%&XTIs`m1((y+%cR7hxr0l^D zZUd~|(v(*I@;u$dF)AV@-Sf<4#QUy|)H=POS>fCw2bM)B3TmuqVObwN4_m|coQyURgV z84>4!1T*DI<8z_Wr@V(w@?A@t&gcRNqcMxNqut&=lPL4AXEHF45SOEbNr|(QuO2y6Trm2|GDz@2#>ufjwO9MgwWQOX-5mJuvkdeB$F3 zHOGTIQyXWH2avFxW8|$V%U2(GP@+x$ z`0EUjj;!lw3(CDFa!b(ZGF?i0gLSe3srPy;{4m|q?|~_Bo8ER6N?KEUc@4uOXTT@I z(+ho%5+Y2@yB#14l`AVC{R1pRw5oCMx2R=4+yrm$d_KF~*aafz}Dps`;uMweh#L4eyiqo1RhHoF;e_%UX62J zHMO~yVJjXjSHZQIS7s;-i_m@X?p;42&MljQdAk69%t2pBM?rHpvS^NF9?~OLhQE(V zU^DE&XH5%@Pn#cF+(YArtqRmPpvv5N1y0k}&>((MnkZs1I^92RG9Klb9j zyVfddkn_uqt6PULZCxe~GTy0wQ?4CnvoF1+MWQ}WWeE@CqY6Z|LIio!6_Uktu?zGV z4#Fr*F+O_g;z>Ev{4*rU``#MDW$Rg7cS^*zRM;#-^B1nRp>U07TgDbgipOi}b{=Mg zg~zoKh#VS#LCYgXfJ6)CF76|l6*>)-&v(4bzC{xmI+8!hytt*wUQXe6Z|c*8`ox#z zfqKbr6EbG&0iO`08f`7FjOK5wYO&SR^J2bPN*`qwi!W5F^=4u>ODo0g$iK=CdU|8$12Kis_%@g zmXM1k`YGaT;+dsU@Z5*l?yHO();oCdi4T3D)y6Olby2vohQQ|C?%~q_GpBB`vtg@>Katj zpZnY1c2Jgl^O@@&U;~S;PlL|-kG;7@Q`Ml1wC5!XIdB5Izd3{FOH78mO_kNRVbhQS zI#Pv{yydDoJJ$^hxtF3Q(kY5IXJ9&xf1Cb99MSG^QeI3;O)u`kY6@y&F;t)hRCVT}BAHTIUN{ex4lF+ArYT3varxh4r{S6IH6HxUFzVb9;JDq6g>j2U5QDSAL4_f(AhYLln6^xH#XO4tLbk?s1`>(MtSym2@|SCwljQ77 zPpP!%&@i`LJ1Nu3FJhu|GqXcTKc?MaGRXL+y{K2j{Pg(`b+<77xPfq(*W2;>7fHoU zO|?9)4Ubj)kGHKFW~4cJaE=n_uyr@qtT)jJJbj4r$}r zYeGoI7NRn~sR-jTW&W|q>TOAbiu*bBkC4&RzfISRLauOAz)(y_d+(VnHgfGwlnNJN z@LOEt{T=UghC-nouhqDpw_Rem?usB%jPv&tGEZdiPs^wm6>_tY+%<`mBGeB1LNN$xqO@h3u4` zK_kEo&pIfUxu{A{aB)MEhfMJTQu~8gOEt$T5==h0i`oQSECYBO81UOSzLyFLodY?b z3Wn_H#WdMHbO%(yC$g>sBUwWh%U&igj*TQ0AqLQSLr{v9yWS@I`dB1!Lf&Ct#tXpU>M*JlfqsT-Sc8DYq{$@nQ9Z?TGix{%0;;I7$#UA0zhXd z5M59s?uH^mI#f3`VDCV!qZLZ9q)?pH$}4HCxOdq`mEn!kFh&+0z2D6Ll*)3bB_}NWQ^FE^QN{jcRik566 z$#@^%3~`htPQJl}Zu3oM@XXkb>i1Ba@8nywc|y#0l#C3!9(J{=sj%#tz1h?_^x&lS z3>n5;M$R7+<_4sk1gU+sYN)#u#$KyZC~_-Yk_6loD-Npm9~;`++!hyzkys z?!!&Hy(UXGj0!AA-rKN=a%M2^^}sH yJc7bVVFC}2DT4a+IqyBaqc{KG{*RdM@YK0us-8NsSC){A%1JBV&69fM{eJ*hui0<_ diff --git a/docs/images/centralized_Balsam_ThS.xml b/docs/images/centralized_Balsam_ThS.xml new file mode 100644 index 000000000..c5256c2f8 --- /dev/null +++ b/docs/images/centralized_Balsam_ThS.xml @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/centralized_Bb.png b/docs/images/centralized_Bb.png index 0a9ffa175d08843193db2d716533d7b7b46a132f..fcb0d1ff9a28bc208612d44cc653ab9fcd152dba 100644 GIT binary patch literal 39907 zcmaHT2OyR2+rNsET_KUAj2yCOiq1LqJUI5s-p(_et?1KEZ7&muzAsKc-K^zXpYw2o+C1|fiiv^Q%JG>b>BP0uiXK80< z<7(yvMqzg-98Dz~4U`>1RY;Ei>Q$7AiUQUNchG>dnbTpjDA3eZFm$L9F&%d`OHoCf z9Ex8+#e*LK$Jncw*-JR7qSc&zJRI%R9Xw?1lpN)SF^*mWC}#l&5iMDJFQksUjfZ7^1>GQI)cIX(G4>8F`Z#@C9dj+D zvw;rQQw?b*s_tQfLF>w>ifba}J#>Yj=bVuI-e#h3D=i5-(2a(YhMRz@ouHDghb~4J z<0fO}E+?;~=I*Vb>46YLC?jG`yh6$?3z< zD*7s3>M}CQ7%y2d0|R>>Cw>)oH!W``TSZ+}2Td(KS509HePNult&*y!f|{Pbx~LLL zP{dl5-^EH_*;(Eiv?8o1Bckl>U@s(wgp2E-&~nN)ZYTj=8&OS5GhLjpj=ZI}tQFEx zRlyRcV{f3Yp={xSg}W)5sfh_WXo$)v2)r~cNKmseOE;-Hzj^Z zzKVi!q8<)b^70N^SaAzQYkiCm5}X!yLSuo=BD_@f)WD&eH%3=N*##|b1@uQw#a2f` zPDWHo-$O=F&rwdoQQb|E-_zDW#oR(x!`#(TM$ySkUReX9Dk`oBOdjDV=b&qAZO5-? z>m-N9X=JIV_V0exWUJ?fKKKwW_q`jQAFz7}^+guzii_x{z zQ@}dn^aNdSigvb+q5>)?lrY*>UddCz*}+`VRn-IO?CB)osW0z^vv#+2Rpa-O!0M`^ z%r!OK-MsX@fXO3d+^keR zXcYk!B^6;SCszdxRZo{pq`GP2*O!gPQ^vn!yKXJ1_o(nE+?<& zqKmclMyfk1+QS7@5MBldZy;KPgn^ifH&V_S<)#ge1nh)Ra##sDS4%HF8%r@kWeWje zd!(X^r<0z8fR?MAj+z@x}sci2e=;;b{LQciaN>2_vD(oYo<07Pl^-}c^mDRAb za#3~h(3f{t&@so#xQJ;xz%49PTvV|-np!AV9UyQyQ5|(3aS>gV3(ihl18M;)im|j8 z)50J<1uSGmu-<>z6`26qwlGO#OSM`BxEh*h2?OX@>n%3H!HLiQbSDu zrGm3F!>C!peO$F&kun$wTNNu0O;>(9Gi98jxCS5PXuJB~a@uW``z1Slle2rmc9-(>O{1QR;TNklPhNt*q_DVaJkE>3qpS}0=Y{z&TLefNwPmW(wGeYCtcHS z;BHvc9QNof>X?|A2zEBMtJaU-1-#`K*lXc;k)GwK#3IS}yQ5({(jONU#ou;|UpTD4 znE3UniIvTCAlc>4B%0W}qlGelImb#3mT9ez^JEI z=cfB7Irf%6yuCS@`viwNGV!prRZUY@SH9e-S1$gl;dhH$ifU>S{8k_P1U_~5nI*=Of|zsHa7&-IZ>$;qddd9e&ot>TK{6O6k+{GhU z4&T)oR&e4dcVJH1IT8Cboh(Hc8+`k`je7`uZ0OeH+T-!Ui_dTFYgdoz!ootmFarf; zpPIgo4!wli*n^2wIijYY%ah;OxVe>W&keWS9SV7O+fV31d7tsQvwJOtRowBJw@A4U z)brREYRcn{iq1BUJtbC;d8m!jz;eIod;_FnXWeaN`ta>U-q-9(Ay&8k>JU8ZVZW4m z#{-Ab`_#IDkUp$TcgfDP9lc0WXj;!J{#A;V)oSgW3Y z$-m1=I1sQ0Couii>a4i5G&Nq6+ABVsG(-|8Q&aV^w)D0{qjV!NYjd3pj6!_&Hfr{MA4+;u8*XOd@+1dF7w@VathEpx&=V#r-bLmOYV;3bUc&=QTgmYLH zCVNl*TK4@!9o%%A?!Yav_UCj>^XcIVzZV*F`{Ys+Ni0SNYzLCT!ooK5sZZpDc#&9N zAES$(6fatGUN!j>ypU>;ebmQfx7+;0h?c|J;lkk}lpO$aa8_3r*=zYs@bpi^KQa-0bM@JVn z@FA8A2#<}AudP9uudZXZJXzGK#Bpt=wC&pH{MPz#q#_)CH`r~wRbq5-XLGvfL-b|& z_rPR-7Jo|WLV*xga6o9wNyyKC@SKFek{=8^D11&;e7kbZMqiNt;+m; z-He*HhfLB%_ez71R&{r+w~!e_i22%FKfd?v+qVYg8hL27jjaK{HRCA^H!rW}L_Kcw zBi-p!r{45(QZz=%-lvZSuIp=L;v-vU=dl8!v8NGU)<8&87S*3_&3{#YxO^^Q$VtEC zu?YzUPWxc8#_Zsq9_~pzpDC};xSA3v37xdGbo1_dp2~I*fhl zUcIkAQ|8nw`c=|X6LE#WPl41aGWiU3VKv#F(@~$JnWvzq$AB4p6c-of-c{1 z{VOfS&H9BrI;}>%;X%fNK5HYw^=Pffah8@QpFZsa_h~*S$UrgsJ(Zv?wIQ<{k@{$Y zmq0+nTvw*F?R#Gl$3e~`V@ZT2iDYc2Tnw)rb zJdRtpFsh_PaB;3?e$_$k1F-q9nwpx?K^Gj3)6{>1T_90q9PobsWY_BBaKMc{hrbcBBp0w`ARePo(C%kr)!Uyx(8 zOYg{mE$DtX27z&@KvJDa)RCr6i&LZ$-rUv}xfw96UG75$Gc0$Uba6h>(!y7-;#ZiJ z6-aiZ&WstYmLd*E!?YuN<>lm7?YO2MGKsB<*E+nxlXPr7eE9GQizHXEbN@A;G^G*D z;M($oR+j6(!YHO|V9R8;eCWojV5A2^s0x~EpI&JJ-RI=wJi$$59~UD)L3e6fUS8g? ze3qABROK3cmZ7h|UlEO_)yVIx{f!-JAZf7?bL{>y9@U^n5(jzUjEsu|TYVYL{1l4H z%F`aw&-IG&n-sWvx6I5~SNuIZ$|l~aDb4hhhAutYtVE`zY3&MkUxu}P+sh*zAiD)Z z2*caoH$yx8*@BNHGps7PbLb2N0-5FWwGR-yVt-n>An8yg!PTl2L`YLQ#4 zz`4E~e9g@Gy?3pOk;QNMl+9>SPdZ`eDKA*djV#~SpU_TvV?i2bRarf`olBgZ7VBsC zL0(~24HUGeTebZB>L!@4<+0MD0X*a4=FTz+Sn1Ry3H%7GCvVpMTI$bwA-_#``x+3{ zT_?V%{BF$q|8TDg#sZBa}V0=J>i;L^j zPPbwhfPg4CyOh6k><_-NbLwq`qhg;sWRs#&u(LrAiXoqH`>mDN2@>`jSNONd^8*e* z+39M-qAt^tx$9-+<>3RFv5fsBbg>^~tUo=5lN_1034Z|}T^w!c7`JIn;aZ;uuuY>k zRmv}^7KEA(hoZVw#(B5q=4L!Rr{aW=E7z4P6E4!V>%Z=Itk!+j*!bN@Ei3J*`6MSN zm<+Zw#{SZKQ*BE~a@#@Sg};3UuWyR(t+y79_-}rbL#BE=ziesAYq)XYNGFF%<^W>D2bUJ##rUwaX-qOZ`wE?idu((4hQ+S9@GVcoYw;{+wBDR; zdhy#+k8=+q8aFvShkBVF{c3wr*`DO8+(iPGvwW?5sw+jp4Oq}CvbI$9#R1=W{Z}{; zq-z5bC}Djk;8QC;$_$ee6BmVSTC)e{o#G0a#GFy}S_L|>MfJN$k5xEafhrHwv#+|u zUaHObCA8!1nf5X2&S0)B!nL6wfs2Yv$tUWhGUEg9L-&(JgWkM}0TUIB78~%&dY557 z?S`J3n&L?xm8uV_a|r1)oFK9TmE;^!?mz2w+>k z&EEcT9?hS5;552U;gwjrORUH{Pw{bcpXr+e!Ry9x$O@i4d*+3c&;#zaW@iqPY16xR zOHyY}oq~UCY02yIoanmnFdz^M zyFo&yzL!V76A;V%Mru8}e(~&uZBobrqozsWvTTfujPD2Pd_Mq-l7nLKVvkMw%C*FP zOTo|};~LNCU%$);+*EwBii@Kk#mC1`GKoSF7ls<=dYB%FkZ^U`^Ab^MhJQePQ*c1}0@#A&16d^9}>1qP%rBJpSd0w7HIidBC%K11-@ zu#_9>&FSX-JE}rx&_vivb;s$9=PdttML%T8_D&;ZGPAlLp4r-%>Gc*d*! z@wfY-!o1&JBM_^7y6{aI^#;ju(B;ehE_bM4W{QEQ4q+@iD}m^zjN?gIWJF?7W?=Zx z**~#3{J#v47=D1(2p9zF$m_5_@O?!U8s`5P1wP{-mFf`N@8Zz0;5Rvl>4;KD@nriE z0yP+varDj^IO+hVO(p-&PQ&lF_Y5e#dS)!ooA5U3zG3Icx0MIVZZ7#rM!XMMO*;!O zr_L5o_0In#$ng7L!6OFwiywkjVP6~XqN`pg%7R1Y(|Ugo;_Q>sx;SCSu6q}VM-Yvk z5BnR*{uxs;MT9l?sg1?(D_mxLH;Eq+Samfdtae{`U-n z-KvC><{T3~pT0r-KG?ykl;DSZdgYP3yF0~%*F{3ND()m!EQ$^Bj_e7EbjykLD%_j4 zMfjq>97;pHFH|DQzAO9!(Fh()i5V^WOi^FI=S~9|Ua7v*_|4W`I%(pVrcDM9-HXPm zze{i5P2a$$kKP~Id64l0TN&G6?Crs4Sk-UT+8Y2;hY>&=?ND)(!bGPF0nBZlTG<>0 zaeMy*NGuk&waYYT>r4VT*x5&I0b>>2)YN1%{wYx?(Cmr7<0_1NDN9oCtb(%!y85Th zIU?yb{okb&Q8nqO(6m$)Syw&BO`SXJ!!7Vi0ikzX zdYSNK1>iwqFgoLzGvY7!0GYN~>#e9R@W7;*e{!vFVIj4_wkpZSn1;?pmeF5_<9F%$ zJEF>nf7?-1JPGP|GN|HMK&$2EJbmiVyLysIJRT5NPkTzW#`sl!)lmp-jT_5G*Trz^ z(O7Y=_lVp{zZ((GUf(a{w>nw8B*We-~U@0|nD-A$t-znbjHC|Md$kz8Zj;tBt`)7v}&BXc&y zbisiczbZ3aXSxNFBTwL!65Ox#*FnpJ;QaXfIgh51k&1zVfeZ$j5dd*&z!}HH#j)#A z#KvkVi6m9ZC;lO=xb8h~W1sg}*Y7!yW$eGhS5yr9{O%aM`8inGJ99dr`t|-dKo~t! zH%qkj<@c+=@b8SFcR4@9&zV8?8-b)prpC!1taw zXRvL8BJh*$eEYn>S!7GMt)l1LPn#}c|Km!CkNXNw3X%}b*e_T-plXI;spQb0pmY~S zL3tt+;(Gwh=Fy#h+q_reK)CBJiThnTeN_4wJFE6TcY@6DqZxHJkly_OL&ta z!IdnMxx~UOVE4)ZqQ^kstp-mEgQtbS0=Gvc9Y}}_AC7*Ml5$;jLMgS|iTGaNm$;h2 z8;LPOJmURb(tW=bP^k96ep;lb%1gpPX!?Q2W{N zvaD?3t<3bwhXRuxGME^#QNfUPg%g}QQjeGmOYDULEy+&Mxk8b7XzF!PJt{D~U zIMiF+*daP7q8p9xv$Rhxa-d#(s{FMB&$=mD@KvTbkTqZI#X5OuYX1;KR;j5#3U^^!Ne!8*GbIjhh@(k2N6E)qQ{8@S3~jDVTY^jq`iy+ZM5;+kcXo9x{g5&0tF7NBjsQ`X0t*@b70@Gc zSF2ByNLsnAt*gkC^IaqhrzCEUEmu9AW6*!=>NbURV;k9TA$)j4@k&Iv&af&&NHMBp zlx}KAJ;E_-z=k1Y#(T@kK6%fmFnd5Z4?R*d_DqT-Fh5s0uWfg@ImeHRZty!ZM|AbA zfx&nD6#7YdoP(w}f-^4RA!7hGU#&NsCWaaKwA47)Uqey>(%PKc(+}$B`*DLgyXC+p za%Yoo_^zV%${EJ5%z4??nP3>DL|sZChz~8p?of!la1t#JuxN%|_CjIOW)llvewW_g z+-L|UXXD{fS+A{k=h~RcFZyY-e$GF>cz=xw;G>DMcId_{?^T};@cGPon+$tfYcQ6n zdVDbK*b=h1w0WLllL`I9o_Y~MpRjS#Wn?wZo|ukPxu|PgZ*L4(Z!Q%V4J^pt-#;6q z>lFpuw1{|+V?F7#m!*a|IXTsaM%>u->N^hQNRY-r*@(PKQqctoK_`N@ok}^g-dm(( z8T0G5_60X2@)*duZ=|j5Y;ULd+}>04=?fPDc9AD-?-a_ebRu;bJCHn zxaSivJecn3h0-s-7>7G6fA-nvPQg1*r<*9365tSV`g_(k0|NYsLFBBB{FWw{Jhu%=6lkdA@{paS^cbdNc13y57_*{j`PR0)5Pxp9Mne#KhU& z{C7i6RKG1(f<4(EMowZ*Q2c9Kv~s^mc7Ol5#1VMF|x;~u2;TSy?T+WXORqC=q$=` z-BJ<(d{3!;xAVwYGa16bL7_8EwPP)Z^DQeuT;p2moXH%<+0M?zo@t_x=Gi|?6K%tP zXPV>zQ0|!?Tm%?DYWQ7bEYPH=u`z3N!Ie516To=@!Ydj#=$CMN`K$cM+lNe|^Y;k^ z!iA#mVIFDrc6LK$lwxfY(iG>U#H{}$SFeUkLr8%0>lSJHxsWcgbs&?juC8xqI~y7l z3=NqkXJ*&{(NgR8t*7VYzIZ71#j{yD)%Cnv;h6?{w#V`$ao^j9U0mY_j!5$a0;TX~23jp+E8>*j4{2^>e>;F$}IGCQCBQuzWi z1hFm&wVzsA+3q?mq zTkdSG+@NpR-1GtbCF?ro>C>k%tE;P0^W=F@g3@GY#$nlT21C6X*-Fu270Dzd!2N}r z=NqOed&3@tkf!TRw^rLy{pc9|<0I0gL1Hmg+FS0FHIOdlcWq@inEdR4tz|+gtQLU( z*qfi0$=KA=#>S@y5==)c(kfU+4Sd0PZMRneVUlF8$U^ zR6(=3v7xbtsS8(D55~5{U%lbEsBhxGVKwE(%*>3hSgvWTrb!cbL16F2ivT%{{3vtq z`*9X<%-JtrcHn8p2zo4>4n-iAGP^F_Gn$z@v)B91J@L&Sj@$OutyScw$B#_!W1+c6 zr2~HKrr#V$)jGBSPm(=Q^z0cv-R=#r2%ha3O?Ytcq`T~1r)du__<7lAO&GE!jzjQI(qUyV^4T{**S!U~OB$=#Bd0|Km%te9AIZgtG-FKwG| z5@XkyiRs2CcmZH)+FS!w!|#)`X@;Z~piV}YFfn16nF*|TtaCUq%^Ew?hC#di4R zrz1YZ*^_ee4DuVnV^kGJWlX6q>9IwK#$sp=bS5|pCM?Mt8 zu1uH(5Z0n9Ct&z3<-m)(K|#dkf{FLfRiIvhIvnJVAvA@@IOa!J?Zp{jcAKNr7J2whC4O-RkE$|wzZ1wg+RLe z+3!bj>4i(y4^gbOfLs)r2{WcDtwFqJzT^Qa4O4@*4h~s`6UFIuUJI92B+3jvZ*5(= zJSk!3$ZGWRp2>~*^@ZV%@x`U3ytP5^apvT;hmY`NYeVL4V{EsFhECO}6r|O5+_)w8 z%o^d`JwZBr#goT9!z%ckeZRka`N?(tU;fTcPNNS;K}<)Z(Fd!m^=EZ;XRtRMNx3f*$JabL!zmuM^oE-TF)vX;pHR>nS=uZ?+PAtHPQD@;rm zI>{SiHWzPmXQJrOUVw%TSprRx3Wp}xH&nL^!ZR6t10ZKKfyG>V6A)^M0?_z~96RBJ78{+vxx{T787SHczrq0;B zMWt7HncGCdNAs7gL{HF#m?=PLz=kYgTyuU+RDQIN^^=ARp3Rh0mE%mJK-RQ#3J7m$ zq@U;;S`zrQ($gi}p8l+Fth*7gxoo%i-Eg^4-PCthWMDk}{(TZKdYBo8mX@}=7L$De z4cHT@>7i3~jxT%`Pp@S(4IY2kMz+8hm$!1rC3*g0vs~A2@E@aUGz%zNd36_zeO*5d zKx!9(%^z^i0ui)$F~?Q25G$s+*ctr}d)&a9OW?->j$OewekWmY1L>)gOUb&@%5|p2 zZZ}p^`$Ey>{4KI<`tkWGgSsq&A>Q>&GksDST9LS z*UNj(4>&EoiuOk(uG&zi=c)Vp$c4Cgnm%oLyJFE&xn}2E~ zH!86hw+E8#H)B|`oOpG;k9uDT|jzFXrqSStVpNNXh4T zH7eJ}(n{Lx)?a1OqY1S(oBSz= z_o?DNz0U;Q#y+uf#D*Ew`=^STY}Bm{l-`~%G_LX5<259$qGtRyy%KNU_4O;L z{`;^qoW%d4TqyF=pnaNr==kCxXkLEmT9~`&`BDee#_kl?Kq>mUF6Hn(se1WaR)aSu zt071GWdel)PUy5m{jv_E;oid!#k?+Dcz8&^r`?*d1({dj4q_FZ6bk0Pi-|fA6RU*l9%fmWI$o;#3`q&k)vdr~2 zIXt(ara#qgfP5uhs?LwrkL4FiI!KYm6?B4sk9|(A9Xi&>$>LU@D@jI9p4aBL^jYkN zTI~qE$TTMli?n}jSAcqsW^|T(o2YkHVW*GhDqScYd;GR&nx;eXG#c zbs4+&!qOEV{>3PouXQG?Tz^YbYGpU&oclW%?0U!IKv&rbMZFilPjYgKq$S+}@CB>* zl*YNIBzO^_ub=qSQ$FJSWHIoGW4TMNMNf(=Yie*?{iXpsB4Lq{<9-~MF5w>~Bxn`Q z_Ep7h&TK4=E4sVC{8@hv(8hr0-K@h@4sx$b7*zVXt9CC_x83gZX?QAUv22+lbOhu* zkJGCf{d|UB-#rE_ltQ}YOIr;8v(lTW;v0nb(bMViv6q`2K8$s?#eCbU_#N^Lpl#V0 zuAqpF%*?ychM?Nj9}Xjd)KVAI{Z{qUIg~|3M4}HF zB;D%h_H!O-?(YP|vIybc4({5b*nQ?pBgX+fmNS7MJ&(i$W7lOYMxZ(i{7ie`Z9oBE z4aT~^^-Dq6&d0_4w7DhpTL^X9e4{7lL3jopeEky6q&b>ASw=JE%IVN3$oYV*g?aX{ zT;8OpT;0(2+WMzW23yl&sq#5G-~veC&2|V@Vo?ia)*1f-|4^PuGU;Q2gm7({!ttl9|m@(`Aum*8MoKP2qSPB0hMeYaVGp_D6$G;h;s_658PkVpI+H z)ny&Ifu>-VyvX=pM_8QWmtk=tn+lK+L3;k4Tv?Y07r~df>Xo3^)-!sfv;9*4=j@`f zTY&fY{JQO?;=NIu1VI=cFr@hKZ$SuBN4bXCWB$iQj{#EUZpJTSXjp#+cG{Wcx6q4o z>*ubV1j+54zr^)dgJ1|p{+jZGpx!?-1x9f4B7uLz)2(VEHJ}PxQ*|Dmvj>Up^3dv$ zLz#he^k^Xida9yYtG*Lmqq9tO@vKUjL!KYbI*SpWGymUj=S`5)s$Vha;rV*(j!`9G zet+OZk^#HE$qNbeKgazu917XX#&_F9pS>>#>3m_MwTorYiZeLu$)1FnngPhWY;-8z{Bf6?`1u8GFiX(5HXu zr%t1@iNc2a4W!zuGi~Htq_7)gKsEk{oGR^6la3g6%3hJ$!JQsCx++ipw8yPQZj+s_ zoy8K)eB1CC`SHeU0L$ljy6#IqEpgn1;QzgLMk0jADhg8)!_|1|Ud z#s4Cdk5IiWH15ae0kHXK6zu2JR8mq>07ZI7#%j_SZPw?7bqKQi_V1sF&^77l73x91 zZ-h8Jna7+fr`F#%r*Le&M@m@tJbN68vSLr4R|y}lPBEIEhrNKM^WLlaoH3gohRzF> zJ}9s+Bl}fhNXl)_wA&>|aJtpQv6L{^#nkl+NTmac_K_;`EWCs(sIyF0hMJ~T?wOfv z!z@zN|9JZm{|n-u>EvY(&H}>&`t(>qUjBpx_9szKJx)?$b>OIGsl?-y9s>Q1^Y=a{ zKD37p&--HFMMuC^p`i1iG7FgZeJS+NHpI#1-Yb5Q^1RP8BBk;Re5zRNXaSRGl zIIazSdbu)CBC@`A?V5tDED2soI*&k@=<25xU;?Y9$_jL{o3%mzMZNCK4sYoyE0grG zNg`?L^P+@%%$HI~wtSlh<~0{fc^=sF(V(kuh2CK$bjt;V-VW>kA{*sM#)=PjkHT(6 zJxssrekc6|6*z?Kk`yLfKA`Wik|7u+$z?9)eu^y_G4Tuz$rY`m6O*r76z)%9`Q@Jp z3_pXDjF0nvXdtW}g$JvHb%G@Ft)EVp5_W>>{6&Bij!|)O#AcHx)Uku6KEi`#Va=$& z7Q65~8tX5rd@$SoZvgW!GBGKF=PGdeMqC7-&rwrT_U+x>wh4C{E)6wUL3rvC;c-!4 znBRG}fqzC4n1Np^{r2F;e-yEb&(Ad%*t=EXVjg2sX%)}?IaI){=&b{H$Uip2mb}>u4v*XYJsaf z&sgt&N8j7UYRGHqp)dD@CN#9I9sxqK31uuhtb+|KJR%2XPesWM+G6>)Es|iAqJLpw z-6fwpSW!(%?%zwV;ixwoJ!W#Vj+kccEq|<8>_=8Jl6OePcgTj|mI0s!GAKAJIwoD0 zKK0Vu2G4mZRlvV}l0nO3OP`(vruC=~B{}%{%4_`vO*pcP&2|u@IXtK(o%A4<>%3G>to(u51cChb@tGW)_ulhd)53{BYv;MP;88i)T zi@`f}=Kr+eh|40Q6tIVV4<9%Zcdf6rxGv=Uke9-=mqI1B#}N38!XL#*)}Zt_B7ArW zV9~&;_doQRT~@9glC5+dl&oO%Tnb?}mfWH~PaVwsN4Q_KB|WbM{VqNIVpS+@q~JQp zM}BUDu1t6zK9$5-mCDS!fBdI#=l+_Ssc9kV`$80UsH6gzvT>C~D1ubFg_%`anf>|e zwqqxt1+8I<+xmS|Yu4baWQyw>Ai6mQ3_7-?q~wT4gv<48kLzqkl`~g+Xc6UE!%DG{ zZYEL>5Cp9B{aF!#!0>$O4~B~V){=9xq10g)E}j1@mgD=mUd*4PwN0m_Mx!b=%Gg_k z>wgsX=FSiQEr}V|!eHJPDGo!aqecmJ{xrd;Opp$~iLH$r8IRpie*dacPqS1w^0rpB zF7Pl{eo(Vq5>Z~-KW(29xl|f={sOvb&haBCWX z`-Ulpx|<$1V#8~m&glZB%icay3@nnK6z`B@2g;3LeRK*qD;mFg4%&(jV zz~V5I$0?K-qH_Pg_mwoiC@F~nvFqmYrBMYKH#fT$G5fxEgOC>chCXRZx^W8_+Ua9( zS%Zs&Wy2EQpqu9pVoCY1iQ5M@AZuYxUUwP`Gl>u08~pfi#105*crvWQJBELTeh98W zxF`Zk^dNXN8++(7%o;SHxY*z{&eL`Rfzo&iBEwD85voaszmyIJg=_^g4NCo&PZ7(Z zg4bdio-n$;8cLejG7N#dAmqNP+fp8WL8>%MQT;C6u`Lu=f|19=cJD?)xM5me1OSRW z5Z>UU+e0`7xSC)jH%lC;^Ik){`#`zxfWQ8-S`h=Yg<-@yh&mi4r|j_GDxc#o`;7an zoqaK3pu)84!iZm2()ii6^&9^=qRfpm7eeZv+qD^%O;O|ei{034g??t|8ZaL z%@L{S#_%9WeLng4SbgOx!LDe0ts7H_uk%OoXM+`d*Y-E^$T?A7;1|tL(KtM--S)e@ z&eOC0XL;gg;*(Gskko>-^Cct3zCO;;JJ@G0_HwV+Ry@nc2tbO=;Jly%OUZ)J*L!n0B%J2xfEw}1PE+Ab?4hyn`D%2_g|;U`(Ua9fdydY;5iWHp9hAUwF3FEIKpI5 z=Uy*(LA9$vGC)MG-8b<~tH7k|Kaz9OGG3%#+5PnRp46VdFL4}3NF0K{;25ovS8F8#|i zA%fhwk(4+NQc=5;LSD>26pZ;XUf)7}#q_1d3+6TljF8v8F*{^W7KGZSzpQ(=ND-pG zGYWEx#%%6r`+ln50~JS;n`V18LnFK`;3+F*E^&f{ldd z?5ka^1wJQq# zPR-k&{;2w5Nc!lMjV={TTsoP(C8K0vUfpn4Qdq&F$9eQyReV=fs?&5viT6sW9~GAh zH9YwKW&Fpd!NPrOPC0x5$HaJ=`V%Sk1~%#9yOvAM75sI&ZfDdk69Y)|UZ=I6(AaBw zmI8)wKa1Y=m=wn-m>Q|bL~@6@%)bCxtTBKSq%a0p7TY0sh7gj~IdlYVQBL98z3Fxr zsF`H~w#DHrY~Tz946xkIHG+HN%}ZgE?JkRh@zfuHAL>1irqBM&YG~>^!0Lu>B87o6 zPPJiEE1c`SVtMWA6uCDV`!(2}r19R05p8CMpb!WC`Idv!#w{(@jlx7}c`H!F^L8_7 zKn4=#Wl_=D0CAi`D=7d9XyOoC9^^%&KtwHq1faPQN%AhgcEsKT52-)KS6NjRz2|V9 zl>u!aJs@9qfiXLi15fVZ>PrTb99Yjc_Rs3TE5V%wS0wq;x)m>Z7%wTpa2U1(*rb^W zD9>uR1)5*>=)k!KDyi9DpC24OPRDETP;7F2MxiJ`8XrEfuilgC32|sr5JzKAsxQWq z9b!}ca(J`P$-X%CaZMb``mIbP7{DDQoSCPX|- zRy^Mj5{d2yR4E*yW3Tq{**Lui6WX(OL3hd{a}-k$DlBk=m~NV`;1NB#QavHEl+oYn#@Tm1 z2)|#S7zT_VrvU=mZt@lJA)gQJNAWxc-v{4sV{3DuG>``5B=JtD;tM~(n-KSaXNBo4 zuS9%&A{qO-V5n7Vl9%veN$?Q^8q}}p)SiRVjag5~7Mm}9*^ksUzrGCfo%4)_c%`WC zk$tZ;9|97^{45O`1qrV_E0ELsKJ~VvG=u}8(4HJfW;eS#6tUOk^`tjrvH6@&M0kaP zs_Lg*z{{R~o>Q;;DWf6Dy*ZrD68a3j@)H$^8eXs-5`43>_4eb@7Evf>Vv~)I1?(yY z;KM-~^`(DpWCaldqh2GAI(4AqSAvulOY=Qvng-?p+Xql{Mguv_d-7y0mEittAC2+UcOM~8-r%*U)C*{c0jFTiUBgU@>b-ch>lSB>CzBK|0t|S^ zG%)wuhByX5l3BgJNXURt9)Rd)eCxqB03nEV;v>x5Uc!K$w(h){!zhbv1C3?ZywpUInfMgX$x#x8kapfOPB+ z>rS`(#P!^FkF`0z?7o0pv1CksgbAq!Fx zsP~cgME8*5fYNd)_f@K5zWwwdMi%8F4At9=`vZJ3?@o<~rvDgzSud*;J9z~EkqAhw zvSfl5A{+pgdml#53~>+|ZXaaGDpD^)N#e{(;FTb<6|guU6=IV%jPEr`%H19<&nrYK&KJ?nPQVX7yVO9^Y zof{-N4O2h3<7;4r4e`Hr4m;R>4i@j%F%{*(+sFLNV_Hb)+CQe(zwSGSpk>O-N_*J~ z-W*94GXBnNA#t9XEAXIN0tjyah9_wCD>n%sJ&5M<-O0G6;IaZ~!ezqiHlWJjZ4Yh; z96@YJC~4?%+Tq)8qJB;Jdw$5!G~DIoiy}RBik@eE*jOzV>P7U=H~uz_t*&;KFx^ik`hfpS=3=#1V?GaqzAv==qUpp;gSH@ z-v(%=diH4qX%l>(p#kZsS4_qFyQ6@_J_;uh&$f6l%vx}elO58hfItGS8Ol)3DV!jt z`xzA>5E1$^A~XmheDV?9MlO#4MX0@ftQjN$G2bKYWd;WY9UZ3s5NIOod_lj3O~z*xf&}p+)GGNFB??NQxt)Dp* z-MerxwnT(2ap)5uug;qyR4|;y#!@-_8`=xc>p<#;$?^VG~=% z)==6qc4;-}Rxu6av{ubPZeY&C!^sWI1>{pvn+|*0Z2!ro@Ulzy_7heN`|~1F^E?YC znANIoJu_CGtr24CJSMgA>p!yePJP{^+eV^dCd-FJo!co%{)<45kKL_nK|zTdz}1I| z+kRTLbI^Q(a#mhze_4tC6P);H1NhnTD!=Fa^`EhP|4KL+@%}hnF*2z1RzMNRcMs>{ zd5h+lZa_>0P@TW8D*2b!Xw+X*Hx1qeL^#;ZeAOpcYYN&LE!9hSvf5v~Mo&@s*Anj& zZ3-t&R#5k>Cz7X)P)fJ-fDE)9E&$H~RF=_C_uomLwzBhHN~wVOwomsffy=JWgQCRW zI`IF6pP>w8Ww3OylT~)1^u;ynLx1)H=q{2}cD_jN;@UkR(fPcNg4}-XUur@EvY_kI zK z!k|TNC17DeNu6rltD~#)r`J8Ne{xA;I$FfO1+IJl+$Z9&S)T6hQO8bDMeX=j#9GW> zy$FgW9{^YWRWi%m%{nWo!;y*u6hgi@gjQxZ)$-m3%xy&Ir^o}l{U`1oP|6VIN;U7O zuD+IAxc!wmnO+bN3UMpJ;KbL+(7LsB?*B@N{tH?qn+agIRrZr7h~?HCBgwbmRbJ1k zWWg`PcQXWk|HTGAKD8yTuus9%RLvz?8;_+LMwvV!k2QR2M3gY^Lf-ENB+~|4@XXQlIJw9(Q3RSnVDO&MvF0>Zszi5 zF%&7?)PZSf8T&RvEMm82Ed(^6h8j8pm27;u?Yaw3-5`A*95TyT6H0f;(SL!8R)FYG zfG7uK5b{{K%c}!4AS`{%j!3#U6u=J4KYR`{=(7ImCL96-;|KL9SE$U^Z2V}SJud`> zCWtuKW_>9)v!P+my6T_cwG~y6e%I@D+p7G`5?d?qa8%9Ofl4@>rj)OiY+>BzOHY54 zfs&>Kz>vHUU2YCsJ#;#~?gO>kYHjI$RpKvrutU)Vz>f4EzVS*SBN_y zhW-Yd!b3yK8bz(*_=TMJ99y6zKQ1o7aEZYy~crPH*VP%gyqEaxJL8 zeoLJN!6=ld3R~bvUpC)L8%(u66nU`L!twZa$zHW4tz{bvAP`+CSHp|ifzv|vQoa_Gm8pGBWbZN)QXzX~Cp&wVz4r`dZ?d!R^QAtY>-ygJaooS(@6TVy;W}J+KVQ#rKF{-U zKF;x~lt+7#@3e{f^i(eLvV(Ryz8T~RZ!E@ycqF7(H#;?9CTcuh z;Z$94TnS~>1g9T2YQFsGDMoN~m6tje{Z*P_v3bai5Y`S~<$i87qG)2b{L`dM{WL#f z>(sx4xS{tdigebGii*l|^he-@=ohlGvN+XGe?QRqT7-wBi52)*-ILn+{hwMsBt^fB z!CP4WTG#2!kP;mH@24|E6RjsGi-v?W4!O}2SlCL@aXZxgek{hJWQMf44EyzRje@2A zs$RXVU(q|BX=`7EU4F~CX$l%Ppr8C|Yn1dh3ci7~KY)S;*T31x$(i^KM}6-o_=LOdzD9P!kgN*!>l{?G+>V58c9_w3?#}#R7eI|keAHy&1 z#DB<$4ZZUO=}f`IHpQNkD<=%?ILJs%9bX_NM!wydG2O_{DIobBQeXesxrbPhH`wJy z73Mnn)2uyQkZSOiv#)S=sSI`0m&zK1qrXe2eC>Mi`<}ei1IMo4pYd~)A~m1Iex;|R zZ1=h#Q(c{jPj_6md`A*FWi;WlgCdL&GJy+%@TW$5VrLOb>(|pU1G8wLxTSzpDr|== zCZ`{TvwQV|?g|bOkPFI6WTe6Ou&965YCXO+EE)9!r3?ik8&B$8rh#a^r)XWZYS`xS zBe;#8zEUPdIts?w4~{%s4{AM9TsJ6skm}(anjyeXnvQtKzO6q z_`FN7yIC@w?Qzn}m&17M&_|{nva)VM7o(}u{r_q5^Rg5YWx`^-0u+`3X@gkv2lez_ z;XC<3&hZAM`Ex7bPNHA;-5{g9$~=4VAzO)aS?K<bbq}pR|W9DNq0p%q~^PDU6W1li~aNd95CcQ#( zK(e&n85h^Rw|r0DKFIyYx<;+1=lknMVzXuJP<-}aap-F&n)~_Jk}{{;nuzf9{a;!C zTXceTO5Fc%MJL&DEH}*x%k`xU);HBcfaQLZFe#v+_JRFm%zw8dI!&)}43iXZ#o4M+ zkR12?#=YgwSC9MeZstN={!6lo894W}c zI}5ksdd422%X}u0@{-ttTz%&4M?oGQBNL)XhW4&QX?gj& zu2yK=3uH2M7E$!w?ZLbHJ_)3h1h`Ox;bUN*lZ{%L;1{aZo-(rS zx{I7;ip)q#{WOqWQhP4uE{9qKD%Th@C(c2UP{fu?L|$q2#%46T02S4Q&1%)7e4`6= zR*Vr)6fquQ;`LQzwoUqXkK3a*MjNV_W9nVASid#G0KoKn{(Dl- zmKVl_xT#KdWGKke)6BBQ;X*J*1V5b$9eYST}!EYm6^qlVdjq&3$eMN$g*U-Cmai%&s43{)?L9HaJ(Y-uY$~Gz2#0=}C-&nVK}*D)h=;U(!sCh#gdd@EdZRl`%ylH`$#O7>;x8HfCAKxj8b5u5u<#*OV z?|xcEm8h>Y%pSkKz$H+_MnD*mNM+L)7+pcf&b}TU_aIK>n3k5-?$L*mt=KY8icJe0 z48$HeAxYk3cVu%eG+)4j>K~k_x7W82d6%C*lw?+4x}Dh!3fO!75UF~+ncMkt2{g%R zPJJaMZ{*@nJqWkmTP&LmLBawosy($8pnIv^$8{W7{{$Yy#UGft<~Z1tm~xN1mC^LL zPhdT>gswcvWz)=1;v%bm`^y9g)YqCKy^I3=EIBxBUD2n6sO-cX+fbFdC#-o2Lp4=M z>kSvN-(N1Ts+MK5clI8Ss$xJdj3y7~D0}5u+IH{J+#3Jk$G$$(7>+*1`+J}bFPt`} zkg1L*gNl|tGYam{HDoL3?)KI{P1b|YSW$rfIW1*TU)ENz`}Lp5g^4Qj(%_qw5fQi9 zzqJm}Ee6j=d%bp7lgsfqSUA$ch|kF1E}VOc4Y_~zL?*Xw_XTytLFC!QZF?gU%kDX> zT{zF%c_E-dKBkPDWp%gEFusPOxd0~#`MQ6?f30~9)t()XFO*ehGenpZk<(>IB(lR8 ziy(x4_lMyH>Wk+O8c|NZq}P!Q+GBPd6dQkXjf3j|o8FH=KcAd|JH`YC}fSoU1Nd9BY*1~Qp3xsAz;Mx%mvKjo7G1qHd0{uGx^k4 zCf&`*4DmzVg=-JT{jBkhm^qEa-k0xU)atKEO;P4Blv>7xkob5D=0WGZ)n6eB9mlQ% zpNGRuIv-Tf?OYhY9-3uKkEvD`aO^PTb;ss%mFq_Ha642Ol{>YBmP~z}N?psvr6}Qa zeW8c8rk%1Gx&I|?E>t+9KP3FpPH_8kyaz$~ES^Xg2%#yC3ht{(X~s9nT8B(_%wl`L z1LQhg=Izrmb!}X&y9;9CX`~GmBgEwwccpi3n-%h}9h8X9s1bT_xa$2UEYp6l%-G>Y z2N9@lg2d^O;iIF|!wRp#_Zk*9kuVuGxsFGXgYpfGA5q1%YY^3acL!PIMYz3_3uK#; zcmM~@sxw*zk;07y{*}(Ss=Q98eL0VDq!Gz=_g>vxp$Qo>@#PYa{5YA|yOx+iATHr5NcQtsSYLLagVW0?*1f~A&V){zMu#lOyF@f3O-M8oyBo_T~LS;T~t z)~c|hJCgIqK|C6Wm{PAJP zF$NK+>?cQ#H=f-h*?qDL84+H)yoXOU-smTYDiWxw5%MOvTL@mi%hvTvL_{PCGwDs7 z5reO8S>V@N>U>Nm3Fd3d1Yv2g=VmRJ<_wgs^n7Y*5iiA^K@HIK@0tMbG&* zp~u!NSRd<7rH5>egNV#YX$y;}kJ}rWb>DEu+R~~lev7@BZCZDz*m3_5=laDXfHL-C zBfd%851SEoa9%md#%vl%;(65dq14j%LpDTH1PKsSk!M7_NKPztK`mezQmF2!6;N)r*5S%Ol>To1Vc{2$X#XvFdxF`=-BjOYXZh`CDY055H8e5Eo8l$9w`YGIV$ zQ2x1``JW^HRX_-)_@mYN%PQXJXtiw9!r@FAPfPGDjYuWfaB*qVk^F`=4aVh925w|5 zsP=c0g^%6Zpu0J*VV0!hSWP7idE zhFs5(q9YrFzz*Jc)2zr1`X>acR@3*3J4>q=J-SzH3iaAJXQ>Ev9+42cEBQ0xpX_3N z6+GX)XSO|^f(S0sWc%!9sa(_Bpq(#ypHYUd9 zE4r+A^a<81WuH-|ONot<)OM5YvWqV4A}y!*40~L;r?|fdufK|}!429UOE5@OD0PyT zyCJZ=-u6eV<(nyzf+ zCydR3ljMlFk6!JMQ@&RIMsn-MdE|D$F#4|sagWYc6{NX9jihITi4W2Yh7){4Giapv z0%-(U-0_nWyk{b)BS}O$XZD5NEyBvj!y`Oe{BxJ#N-yh!sks&N@_ZO1EUekBHV5oQ z_VHxKY0>`HiHu30-ZI+z9~C4@eFsVtiAGOS$5T(k>X7KsC(j;pWP`F;%|Uvb(nzLW zyoe>WNt!LYdljl!mN*CaJl7+V$T<%ENgb-^?}%5%NAT7$-5u5@z0rA9;5PMlZAu!O zHo4|7+(iZ)M%}BUT{={^^(F>}>~6kESyDUH$J^{@IW2xxR?yHFUH(x|!(q~u+pKVL zJ?eu%ow?=){*MxRzx_@ZYQ=Ct7E@QXCTMi{6KPh~2r_UDsUU|*7(TKpV|5AZ804Lr zK#H(9;NWmbM5S@x5nTSG$vF=#O#bc$kvgVm{X{;a?{dUEf99#LNBDs&0?-TomC&sO zC}z0^>)U`5Ii$Uo@Lg0D3Ta;%;ji%$>A5B0GxO$BOoZSp*vDT;*F~iG{t)A~4uO#q6YE_Mxkr}E5xJv7Ki7vJN-lB4k+QR!xBT*NkXlJ@lSY&k!lIZ$JEL=; zw1TiXeP17ye>T|Vb-_iaM+GC7`L3y$a~D!VnNt(kh#(9Bm`ejImjj;`8q$ymshdX{ zWTZn!O*W+OU97=dC&St0_}xS-nG`7c|Et#Rw+du2iPX`+UATXK68caSMsB$vP4adY zb$-Trl7b9PKvTy$DEle~xtfB0pL0m%3T*6TH~S_xQkQ(^HZA>&deAOvDo8IX>^8^c z<*Tal&wPF*sS9?#888-L*9>3xKB;S)Cm?+O@es;z2EC`f3#NRAnRJnDL$0h*wcgeb zC(I?cHNpLicSz5ob)c>$kY^#!gjN8*16zMxhut^K>C(Wa1S_^ao=r`<+iUqc;mUWk z{~jeha+IgtLn2=ljNqE2p9yhf<%CvF(4T>vFx35|>D?3Yv(V!e8)1}{*XBKuG-%-`c**T{kI{A5$%$pqlva{UvNA9 zw=I|3=d9uS!P(+#T|o+&g6QhUD$u2O#1W_P0C^akpFz-Gb7{w^!Lxzt@O@1zVdwtM zLqi6FI8r|+Tr{f{0ufdC&doO zU!TUz#d;XGzGv2`W;nilli&&(I!3Oxe%>Sfb@Ma+p;G>#M$tuLQJ1B^nj1Rg;UpJ6 z=JzKo?$vg2=>3v6#`Sb=InE`vNZQ5Y%MN!{N8Ow*Mf?)i(`!ujbasa{y*4fs{bd;@ z&XwlY9=G-M&!F!cD52*bx}&N%_vo2ITfza>DSJ;!SWqtY`Ac-2Ou6*E zv4`(y9SUWh9b)el_+D5WRO_?oXVHmCV6y-EocB1@Iw8mz)FpS=B z@p%#hKV0CGdoHvn2UWuOiX@)Bpl4ZhH7&X(QE1=ox_+C93edz6biSG!XngM1z!~_u z{ag}yQqR{%>T8RaC+lWMlKHCo4-glueMcwuOvb^fa`CIa)y!5{OO-*Ce!AxNChz5==&ou4dDnsIVl$uYZrFiF{=dto ziOV4%C5pz8I46-(o0mPVL1?I>ob3OOPBMy8D`h>)JMfv?X6E!vYNlP#aj{+hYDVJP zxt{t9bg4IHX=w<^dcWx^?HF1INql_fA;0){Nxci&*AA zQz9!gsk0%kdp1}1pEVjnp|zn`vJ_#GDwp)Q1@q|fSX$E%y-rr9Mj^jN;JJG=L=Qet z$3+nTO0`wLNzfouo+)i*Ci~R$34Q;G4x@jL&>pwU3;RH8IL+Y)$Rl`h|1PE;^+@>5 zw)8u*#{stFd9>5K9$|;YOxXiE^`BxD7%9kum?Gb*yeg}sZmrv_)y;9v6K~u+j60I*JuxhPf5Qihu|Sv;URkmb+=gelXa99!+GSW>!J=2s`@`TYS%b zlM;Me*?d3aaLml_IHCN^{%^D2jBUjqF=Axn)!DdQ(7y3%Zbw-Q9TwVOt$4C=T&$^k z=XSETwlGD;Gtnr=8nu_QUwqcSLD;YNB2^o`)SAjfAt%UONt- z;%4pjJK2_Ny(p44Uwh)5I_t2zZ=*y?WTlZ^qUk`f-|2aS%;m%`?VjUU)T={pUmU6 zB0qwm(|-Ks3dA;T3DVBW)GNeE?I#t)4w(u(d#`C*#g8Y?3^V)2-_gz?qcsK}ryi>5 zd_5Y!fuUuIF`~0}B$_+xurvG2g2`iZFQ+#BMe1l%f7!k&Kb?*YqSd4-rZcbkkSXor z=YPqvG$mX=BSi4ZM4ukNGZrVXy(!*pO})qsa^FXGgsXP@iiA&E>35Is7d$l9J$XHanRAps z#hm%)TaWs6uQh>%l3%eoxRrazf7Y`+I*&itxriE@hNwHUf4MM#{&yqKOBlzdcN4SC zx0n^pEcuY=-l{&_XdOIsc%hWXO#W)PrMlf>h4m>-UXSIQkONW)o05;O2b(jfgi-?9 z4TIz}<{o8~utg1!NcQeiGZB#tg*sbuWp);jJ^nz>iK13sSD{Q&=;<|%iMjgxJ`roJ z=_MGYjq@O{5>1=;&(2fe!p`>yyKnK%tO?IqI<5a;DAT5&Ih_g=Lk}MEB2!`Vw?c(D z<+qO7Usl@1@Y3}7DdpZ2c{@;>f#3~ldr!ZCuQj#L#zIDrq>a?%$z@#G{Ya~PM+Gip zxr5{$c5`Z5&>e*b3F@&dUZiIRu_wufF4Wmzo=wW{*s8_J3D?_;9o~qpez_{cS-D64 zy(QC8KDJVf_jC;3r7+otOMCO3 zC)8&8*B(x4%jPIaQ*SMN&aI?@whDbM!za&)! z+V{b?kHHK`hrg%Fz_)vD*2J{vP8WCM(zRy1^}|Vb_!wt603<%WlSrUw=F}Jko4tN5|9SVkd5_5u#LWS-?XIVIkYuOX>j-f&53(! zEA-xrV7VzdCFASA;1?O;?#PvGv=~B3(RPhOXI#=-JNQmaX>LK8vU&){LJ@i$R*BkQ z^A-_=C-AFASQ95LfP}TE!qL~VibbbP1-XVT(6CywmqoUmJfP5Z9GbXmyRQ@uwm_Tj zjwsSl=0Y z?8aYo>Hp+R6b6&l1;doFS;pAsX?xEoFL-jFTgP3-m@E|^j%b}Z9?=wlej+Z+tWk+O zDg`7%*_~SE$Pc(BW|oI-NHQgi_XD;i>Ir+%*%&D#bvS5?`*X&*1A49UuS`5FYf6@t z1{<`&pp%@|2r87hJn#z)5+R&?)UOmcWw_Wi%M+U$9PqOW`5z12K38Q`Y>~1!5mk0wDPRDBEf6q zAsq3@SbD`q3ll7DHFCH(dn8@zISD)`@8#zQuZjsc@=7&g?%v_P0tgcJ56tu9QPg4;y3AxBM}E)WhC;q%0V<&C0a2L?1zLFVf2r0MbehhX%+jYncsSKSc)- zVE#nC{DWF%Ed)G{4+&xGc0EpZwRF2gS2FB*iB~LDs*bmF21&ynH==Z_t|loTobgs# zP&DPJ$kK>GfNC019G#d@)~_x5shW3(ifrlI(P>S$pB8A7`jVn$q5Tnqpy84)Uy?Ng z()5mJ^eml3F$>d1#vXksSK8YMuhxF7lKEWMI-jGEOHFP z>Dpzs88Mx6=7BA**tT)G8NK2bSosdqJff}=!F+>)ur*sK+Ej0fu{_4oZot1v_wCUW z2d{frB20f2hvh?_%Nzv^;jKrpg)jC4s(%N3AhB0G$QTQ{&rcpEe6lH=!_EL%B_}RT zF8gzsk`LUPCj_?(PM;F#^7`1qAO3?U75HSL-$l1!q|L4$D#r1}HWBIpFLJHsRuLpaK zZcJV>V?{<+PudvtI2z=pn?WSB`ch}J2QSz2jURjhX&#oQMvE8x9#3u_3=N+{T(gH| z^V5BAwcZQfE*E&@;;%BpML-(Xan>In0L3ZN<(VNCYkKN^do&KTMxCad;O?jmGwZK} z9c55k_S}ry->X)S`VNDAD|)LH1=r1y;WupDmXB-#AGMwDHEX*`tO2Syd(=KUV$66C z?OVDnos>7SK?uq$W+=uN zF|mq*@R4n6z~gkcKF~iDK`kK)LK=^iXXiSP6#N|ae+=OD;5n~34*|024L~(#sXWSpH*pOd&El&5uRl9$ZJF@uwyQCHrc+7yDqQo;`h3uwqiNYP^C+L zLEtSGvLgsQ0Xcsg_R@KiE3tG1#5v~?PA03{f-E!Z+r%A3WZO9z6$IAr1xc4a+gavE z@fvTt$_Q6G-T)iyDnk-GA=X8tT7jE%$1IpGdsGH5sV9<>03F6Xb%s(cC_mgG0$j6J zb+G(yU-jo)yhqi{eU3h7M0n_vHY7c^Bs0qxSnM~1KacFpL>_;8J6D(Fl{b?Fd`d|m zRS6B=n*rRO9HA&ui4;yzY7B6(PAyANURy@svztZ+b3HwKC(6Q)PAb>ja%Oxu`ZSTD zNs>0k*4!QX(T;R>-4|CH#zOr&*Dl7Wc!5VeAG8JZz+hWiA8yaorLarq|0RGZnDb)DS>Oo&@4>@zMUe7CG3n3iN$4RgVIQr!=<%YW`ad4bshiGP^=4>t!(WiTV= z%q3O7d{{$ICT<)j&Ya`yyNW!hP^r{$-Ec3j@p>jZL#g)aiPLb#+*P}W$PHdKd_XqP zTjlE`FeROC2}n(fuQbPAC}p;e^?$om4TdUhz13J6L(+A7tC5kVodatH9vjW{TDolF zSxRukYUE*hhxK~e+~My#Vw}g#lo!hx&Q4&iriyQgNz6%1-u$unR4+my@4~ZY@Pi*uax-|)NPKD04OR_0{p@VFa0X=B3gM)qCbM;~T3mY0ui@Wu3=RJpf? z*YY26dU?^`aNv6*Jisd8*mE~pteoMR$+h&j9o2#roK1$OpEH@0QY#?D*g$ zrq*TA#7`o%dfrOTt*F(*IWlhRzR5!btB%GhlYLZQwa8z=kx1DT`A4eHnzCh5ud(5z zl3^~8e3+eGJuz@oO1&ks*RQVk)QRBYm?MZqNu-?5v*TsjM!emx)j``i% zub(K>39ad;-nfeYJOu8Km@?HOE{P{7_hgP};KBN-_M>{Ma#u17{&kVW)0aNoa(G|2 z%mRD{RwNnKfuYG-rNhqm?fS^ututwH&YuoJ+KVe9d@Wx3BMVWI44y9{Wtly!#lx@vP!;*+gG!*s(L4_5h-ghhXULj5fbQa_d@6oUOWTK z`&7Un?>7(jjqo4)#{a79>;8I{Ee>hz^!cW*79W;Fh`&v=v9d5TA3aP8GnVgd`SH7f zw}pgN*TTvAVpc4&2UtnZ0WEK=9gVMram$+z-M<6qNjUFtdQEpY8BSZ+an=>AtCR?r zGGL3(T2*`0Dc8 zP#IHS?qLt>k@8;jk3$sFm(ScO@uE&V%;IzZru*U4jqsD-!6zli`*BJ$L^U)u_Xm3& z#&#vP8}=6TNH^k+_ZEM;W=Ri|^Q9<-Tn0NvpXOWfaW!Xn<7S7$VBD6E3cPlzP-0Te3b6!Z< zSjwr^bzjOVZn@$5OSCn>D%~cizEEflUNJO-C-ghv5Z=xf25sWDH{cy#&G2ap0vn{)k;t=9BS z%ZD6r)$yG$Q4cL}{&j1mydKuHIT39(P1^WLRolMaO8Zczy^gtC7Hrl%->s+j=+XW! z8K3KYk!CyS-?yE%qvG`r%X6(J$+^!Prv!)x@Zq}K#I{HRQMjD?R6_%aY ztbq9q0=IE-P7C1boMiUbbAPc1MP*GB*fLx&ckFIvHml0YzpdRLqr!hn)l01W)~s4i zV&daq+}vFus~1Xmy(_xgbSWI+@3lu|FBg8WSfQN$;>Fh{`#2~Ph4W zpxEC;g+@KBV$`Z)6L}{553*R*zlt{);uFh_GM-;$8xFt2|MNlXSVv&~AUDePqP%;V z%dh!baq-6Mc6@Zgyj)!CIEIG%#f@~U?i0V)BZVsu;toz0s}5$}U3ZpCtM# z>JmFRiQ}i85abPbx3pT=X7<=&Evob+XsXdfmyA5#Nl7Z^c>jg@BGY!N_tlV3cQUwd z*%BL@&JWS(952K#2FD23({VAx^i56}F-(Gfgr(RkoB7RU*)U`lxdJ(W`7iAdQ|1(= zS@6yJdHi7Jb=rxjwmmmzi%VvMhRLPY2O5#A~Fpj`tSTGsk0a zgXBEAM<x$x6=E-Rjr$6ymFV z=3yb;L@)G-y87~6$Vsa5FTIRk#p$tegH53kUh5APxH#x9HLf@AWRr%noNR4~Q1X(2 zml?)_o#u-`kJpU45Zz%9oc+01f0T?W5ARRbXECPu2O1a1o9e5nq131suY&?6rQ@w( zi>eJ#9Gm`0_73-2d37syQNsJuHG9Q^TGtyhoHDMGGHP;(WhoYr zlql_*+pX*A{Vra0%H*dDl9%p}kaO=ad$4oqfom%AsmhmxW+zRP&4}+P&D>#(`_f(s z)SQECej`1@gAY2_jUqJ;`TY5s&NZeLx=B|D@Lyek+K*4> z8+*S@)lqtDL72U;le;vx$AF=WjMz-G3^Q1$!U1H-1-j=OMukrWt6mIL4T<-%ECSmL zJ6ko3Gg^To9&ob28dY$O{s!;doskJAH5CwwzVaSpcKLu!(>!z2jb0t`koZ;mYbQ@0qO7HY zCgeRo zsW0McAVDkj;+NC6?D4o zkG?o4sTg!q`Mz|OGrcTEG9Z;swo88C^-GJg!cUgvx`n+4kuwQC)sH9BV23}5{+29{ zT=3i3S6I|}nSr@miXz?@bnq~%vH78MaA=}{d;W&hCW8(xghr0|E{Kj}Q)f8DMtnHm}2v5R_i0PxO3XGFcx&iqy{ zxMoct&Ts%JsrBG~f_3!{?6}|X#R`ZW?~oL%aCUDXlYEoKxFXaZtANXt7@cIL9ANYQ zo#_s`vfkFoCuL=JoF1OZMJizd;#gLk*Ef#sUj~ZlO?|fQ(l^D<=5~)UDihKZXN_m8%`w;kb03CsbaR&g0 zi{sQl;bhf^e8DmqflKnXe9y-{J>91W%o(rCOQ2{us59t4GCUsPI=ARAA`!m@(JgSy zmCxb>*^iKUz1W>;L{+hICRg*7?={WG^oKg$A6t_nmVkIU67n1mUHv%g4Uxb*9enx? z1*7Whw+QTl?*u6SxSwIM{c$(BCiycI(*MVGExI#M8g_n+2;OUJ0KOEV^eQz(*Qchd zy1Qo(F82oz(D}<7`bfM3VrzS}^$YR~ETBCR7(v*HSAKK&A*;H|x)`rop8$``%OlwE zc$K@vP4giy1BITfICn)4FXZk4!7RYe{L}x*OEuwVK*d5t6|MkK&oSV&+=}JS;Jpd{ zZ#w$~s6ZkcB;FjemTkGNIi$gT>tgw>ZMt)5E?GDJnYJz_7aP-bk!CGNRsZdu(-gGq z*aNq+*@a)N#)$v?={Db|3S z=^}1x-E4oV@e{2eYY0(DLy!OtUj2&Ft_p3lPM3Vo>n9dF=*TU9CGlF`^m23;?cL3< zX+-0Q*n&JGxAku4SL^3`XQGF~kO@RvJ;h?uY?{hUYRp*MW@&J7p z%~(z-*1`mply^eAr~xK8?@qBjZMoVFU%HqLTVhquip%5L+T2{rwXcY(tO1sF2C@(E zgRThr1Xy-&KtvZb1HyILP^&$+e#S4@`6nVG41o=E6Zv$8#D+Sv2ic~I>C#V7DxcAJ zQf6;CkP_(=Dh{0i21R$SkNg8&s?SvtGu`x;c~PYEvrk;j`?)%vKHEX}1Jwe-b0Ee! zfR+CT%pf-w3?67F1;>=)d^g6AtEHoxU=qjQ&t;WGyt^9WY$JA2rczqNSmiS+Wjt48 zNis}aBC0+E6o@Yj8i+lQZ%K7)`Gp(sLt;`Z*w4LaExB3LHVuimIl9^jxC5rNK=458 zTmx9{9DhM@aS#-ONj}>oM6AHTE83iyG``EV-7uFblr#IhVSiPcGfAvnrqUoIg7{NV zSDC8ugPaVdi9QIo&r|+c>iJAllxDhhqM6;xyg1|GI9?mc0uoZsgr)Ge-KC&Df&2%( z9nmjGL*U7S;X{FtVpbZ9yGwfQJlONIruDgU7M6BYn4gUEno#{32N9s@YD2FM997V{P`ooY$L;j zLXs7IZow=p-U^I+L+YxExCU{^0Ol3l0 zLhf0F;S#|t;7H@hi^QLK@IGSfJ|H?!ihxM0{qOTYtPeC0G6*1Xf&dL?99NiI`_XQS z6ewaGvRKIZ8C>?lmm;A*eDCb3K$7~~)}VzogC2ch(py-I)C4{@e%m7{ZtkzusH4Ua zHNT0ZVLUgJYMQ@RIrPqQ_mju7wmU?deXPZ_E&wv#0^E>pTjdCD^Hn;-jQO9HMYM=i zYS1z-(;|8|@GOKsuY)~@BY+q;MCgKm2m_wb)-EC{4B|%+unTP^6@zV$cEg9cA>l+H zqlM(Rw_^kU2`3)cHVnZo^$FW##YvtVKqSChfk^-(E37R*vfV{_enGb{`OyzDvzWav z-|)aH!5sq{hZ1J+)+ctBpRd55lE8D|@3Es%w6%yFwb5;U6L-RdYez=M`!nRq8~_iS z=Vq^3NzsA-)^zPY+Og$}xLz*PzwA|$hVxJ?-jkxP<66_af1*xL|%mf zsbEEZx2*CaO3F{H|8YLo{f`zSdvPiC`9}C2dUjjJKhb7jrpSml0s+_AM$5Y8DHK=z z!7msVN1g*opVwY&QzWv3&~*KYxUQSbh$Fl;3}7Ra@*d-vmJ_{!ib$r&siOS*h&y%s zg+BJGSUFqIW2|ey2(v>?&L7w(XY5!K$;a(&;A~p?shfY|a=El_n5(2b(B(p?~FrQ83dd z$Jpj?Qk5`Vyn5e}JU`{|w^QU~wAG9t69`-PsdPQog1G$<84HFVxH1sP1e^n;3b+kv zCs}YgeG&>)k{rH)-9R#aE`R#|$CH=_JC7JoT8BvSE$*=^cjM8M&^6Ct^CQt1G+Vw8 z+1RnF8thC2c6bEO1-~W?bh4G6fyLx5Z%_hxfrQB_4>k7Q0?dkX!zay)*uKHj;7YcV z&YDDleET`obM=P<#PYwCY|^UiY6>8x((V1^7HoLX@pr0uiB}M-fg^@3ePQtYEIdI0 zg2KbniNF&?dsVC zN?vQ+s+E%ymNSt`&-hjMr!h*~PXNxP3k`Bg*Wz?~RK>vKXS!z|3>MPIr-l za5o$vSpRk*tSuqprFU5Xkl{=sE|Fornf7yZxAp*!j1b7Eik@=f&MCgqVEAJ|N{lC! z+rpR_V^}v)4v}MoPLllza!=WR4QHXD zIEvMx@WLLfU|txv+A^o0V{$+1jgF{(3@fjGQKv@1v{ zY4@?~Qi~viL&REN9jeuy)l3;oMrMY^xhiD?^-^Ih9}3fF`YLF-7t{)rO&+0mRJrp? z&GZ+I?e*cHsX^2PvQ-0mXWmD)wCz47FT9@Ap7nTy2Lp&D?6?8lwcz}KG?l%8g zYfuZ|d4dnB&b$oqI#_5SEEA1!R19m_lrC*O3%MnrPvCNu-5j2}9AE(=VfFr?Mgh0+}_RYdz3mczyA0@ z=kL+n_Y*|44J3derq=&HF}w;`llgx)yC;^>HjJ|wdRy;xkR$Y|oYDfNN{*ISUb&`^4)At#p*X%Y#{sGlM6kufe>{NIk zVRtAyGp#H=m|PLAJo7`g@WnFP1){E{Nk=D_2I$B!Z#%84RQdKz2V1TzbefjNfSUnwGm?0R zYSZDtV~_gDiT00DQ1$RQ$VM^775$TR1TJy}pfWGWXan!fyX`5jjYxgEg{ieA`Vt8# z5t_(3=Y^@Kvp89!(!krkdb@=9hlm5h+#CmH8UVt*VzRzy%(87tfON3=`b#7jqFwD_ z+vFBU61~sT!4Lxqum%PT>?V?Gge08qg~}*7e3y9u>&ikdpJL*3{>nS|?)+b@Y{4HZ z3pX@6N8*Q3_#-1&Hz$A-#L0)Dt-*}^l6h7b%~6H@H9QglaWEuZY zBb(gynDbvomdH-ntq5)*yp=kk!E8(3{ytF-m>B?$J8*!oolfr7!V_V-2#=2MCeQ2B zkp`+j?F=IRUwc#&3mBPDH$SdYiDe_l2xzVb_-BOOBW)+(dw=6oIM}ES$jdIkLx3kh z$q2-#LM9jTZGUptQ7@9}@Nav{48lIYVF)-J*c`Z2`Vpfdh|PfnI(y}G>5R72glP1U zGsuoyw9^UmZ&xk&n==L>2*emQZAWN*JN4RgNPq`JY*ufU-qx!rz>1K+wIGMh%>dcr z2?uElEjQ$|L6o}+{imjd5Tc-I8{fJr8pLN-u4Q!RlXT>eT$lWrm9wDW^@;3I#;A}z zPlcx$q{P%~ltu>hfWWEqdRDLA?-QhjO?W?gA0oP`Y3^C0jfR38AqoqI-~sefbpph| zKcx@;Yc4x;P8^UuZ)IXHZ(WXS;{SBs*eqZ#D1P{!zOsl5vcHe&&nug&@w}IDFosTb z{ZGvToT!F=t8JcD%t?8Qx~x4kynOa?oJ>+ZFqYE6l$ZV_6d}o>ki3*VE-W9i)Wd^p zri|H^+w~u1&lOrIW_jHV3yjN2xPf#{=vOKFsMidgSn)iS&ghTnb2Hib8lxVn5=Lj9 zPt`?XrVeA$fkV(f#Zkyt**!iYpAoa92RSz1)huPJlA`pH`J0n)(*ya;&9E)ugZ&~} zy1nBPax$EY7jcfP6sLOR!k8bo3$bl|aB=pkwBxA<=K%8P!F$Qsdlz|X{jOR3-Cqj7Xr~E6R4E!c9*Ua;uQRtTP-SNh zYQw=O6jLw!c>y-8L@oQ1F%I(bcyI%58riadz6JQ=^@N)k(z}4GzZ>EMWejLFGbNMw z*G}Jg-bGIY>J{-%!9f~0>x5V^b-2c03Op0YshuhD=*j#Mc92XQQC!;0r=}5<5ST_y z0q`(@-u^>BU4grA1;z_ejrg|W_gI52(cwHno>mBG6q!`lMS|$A;d0TJ^Uz*ZFxD>c}aF-w(zwsvVFhbbT(tM@k@&_jqc*tRI zy3^VRGi&G3gty&yfKiHp{Q}<~4js#L`TsCwIplvlNarV9FfhPAW8G(`cFedwz0HC{ z_%aSZ5Kt3*Q!k)A1yoWkfCCU=F!M;|2$)+PibT{+U`=NmGp<0=9s#6nA*;n0*5X=2 zQuX3i@5{d~L}3BfgJkWg{RXuPFg+SPQRHUj=i1C%FOo+fh`zxLOIW15x7<@I0HJ_b zeVZ@-rmepNN^pKK8kYe>{Nk&p-ovp`<*UC<&9Y9)hI0ATbw zzZ}2@qY6pcSN+3XLGn4|EMI^c0E^dl)d&@$5rO~;Ii9z#l_E>M-U9CaxO(Ayp*y0Z zCif;_hER<$#o=26E*n_H5SE*Fnx!7eW3PPrbCz!_RpZw{gizc}1Dc5M2U7ur0fxX} za?b!p!G*y9Z{u?mk9`&evnC#3PJ<>AEyC(n(Jt^*yfg&TSqZ2HYXQ_Bfs+Bs6)_^< z>=rpKwPO{zo{GjkQP$Z-dEBgv$U4C1gTF%JMSl~AJJM7h9F{#zp?e83h%a`#xHnqd z;-QwQIudjMJ`$W4eDKkN>tw57Ju1F)i+zhPg-$n2GYwVv_5be1p`l?>sU|&V%ClDm zZvYfM_)>`02h6n(3@yM5pd3I{5eT_B0|BPffd|5)`Z4~W-7vIt{e@3m8ev2Q$R1G* zA|5qRj6Z=$h4DE8XR#n{_w>qQa9@U$df3ou_#NnNBr_dc*@6%;c^PA#b7NP~M8p@% z7(qFqAXOC6yof(HR}Lb}UhLiM242^nf%>GbLFnU8WfuqnoHx6P@@$d1d`le)oH4YY z*nN>8dnhJ*GuirF2wML`T#BEHvyPhounubX-nbI?E$~pTK1LC~l!nWHj$A5BV_1rD zEX?P7yJ~O-^Q@2DlvX?5#XqWK=H8NT1~ZB|&#F8nApc!o)D=V%J_J&)Be%`slNprD z)SPMuSZNl?XYM*iFZ2-5T6CZ*xtaffDtru`08@~ literal 35065 zcma%j2VB!h^EV(wrK%{s1O-7s2%#gzP^I_Y1OlNZ5L)Oef+8J6q@N-nA|RmByGW3x zf*`$v(tDBiZt(7&yXU#*|Ga(-kj?&fXJ=+-XTGzOaBWRxDsm=rA|fIxm75B>L_{Qb zA|hhBQxMRy;J|A@M0Co}Q_(F#E;!5qY;3+SU>6iScmY5|RV&RnVSxF5ow420yj*zz<{a5`^&!iu~$< z^#;R>!T4cUK)bS?tur7>iO>W57X(d8j#xW;3=T9BI{hNR&eIz8i&$$#Pc2^$w6~Cm zga_J4*InIF-s@K%Z#%4qBL@Aen;@JY{>Q_fzHWBET9J0%j<$Bcy1RizdHmj)&?2M= zn1^(zi&`zUmrY#yAlLRX~@y1lK4maC|cF9HQVRmYmzD=UddCTL)&Y+^5= z=Huxs?&YWAW2hl6Zz?XP=AnvpLMe-=sd}SeC?G{S4Ffk}J9QgtZBy$unZ;bo4!J(!r%{eI|&U3F`THqkFuJg`)UUFVG3YuU7Jyks+q=t{9gNU21lC!Up zv#2f@SneiV$;ck&WU4DFrlsSgDD)|{B{1oL31Xc9hz4Tms%ceRD#JeBlKbYaHga*mFM_DEi69*3<6p@(x*veS~66LWUg)z@;>FmU!n!!%6Y+`%?e(sk2Pb@asWg7N1`V;xRgIlI zOpOrQ9=KIjw z4aUe+M;Qp*h`<$PV?(UBgPxHBT+UtJRN27EPtIKruBxS^gfKx{8-gynnjR=S9UFv> zwz!v;xHASWiWSloCOqtH?Tl5owKZ_T-qexzR`T}1s^OfG?kKdgikh`I=wRxE)5AEU z!1~nW)C?Sb9Q7Q*3Ov0q#)?K51F@UpD72lhud1Ges)~q~ArmKy+wu6lW)?V8f=j*KDi^8CR znNv~&FF|o7K_ELogE-pGNl{DgA!MYU?Sit*VID z(l=Gpa&go*Ffdisk@rK2xj5=!Rh-dU_RcQy5-wt3LVHIYRivqkv$&9*j+>gYpQ^gI zo4B#Ouc9JSSjSxz=j~*K)-?o%6&P6s6$vLC)`qYOPc2blGzwv8>V&dZHxRZ}()2*c z+hMeoyo^OmG_&8f|{|guz`t|oq`8iN7+Ql&je_OsgZ-VlC`m%k{wJzL>VUGqo5{?M!VP{d_`T* zc8*3iK6>unwsuB#XgJ#2PEXs*UIL@#Wp8b-tFI~SEnx~nsVa&aNXUt6feow#jz#eE zkCXBEm;~>CcQb?(QL7`&L_|;`6$OMoE^Q&*-~5rmVN2J$#p!M5{s9f1MDoiwAlA27 zAmN)%O z?%gB0R}wtHwUMAXJdnqJ=^lFghjRK_YkX=dbCdf}CIvaU6FGRPU+xyemR-w-C##R1 zxeQt}QYAcc*q&}=N{km@r%UFqr(K!qAgLzCSY81WAa-|m?a0W;M6hD#PxY{0IDdYj ztF3KsapOmK)7g)Q?Dw$2X!;)u(VCshV=}gnqLe0CSy~Pl?}pFX!INvR@bLv$=5|vl z@Jt3*;TGZ)?a5b2rF=_?F!iZ@s+N|Pn%36ivl9I_X5@<2h~ z4@bwgc8#=N7-~>#b`TzE(26q!>yZgQtX^2m_Dq)zwsf1chbL<~3HLYfIDROqG-mNn z4YXIP=lCk-{I97A>{g1N+*)6*XJTSPVosgG)4t5hJ3Dk5kSpJvlMD}KO_P$uIzCF~ z2VDiH+^6EX)K& zLH{ZAF4s43^$&8PjYkTALnv1i%8V!Ta!IN` zUsGJ^O~wQyPA%2H5yFSzTWE?E$Ayn9DC=6 z96?*z=-pZFb8B2nW2DtPf1cGsxZGLP1k0^%`2FK(tHt@lEEVHVaz&bH;s`c_p!_fH zDjX2C_u9594>XOuiz1+xDE#YbnQcOIGsssyggu!q5Gt6sI<<8BhIGV}sm8`ejL_v- ztw<)lSLa4fdz3}ct$hCc+2T=kbv0`2Gbi)Kn5(D`cn$M2Ue+AV*GeyyP32h;_6oa2 zi&}8fA2OD9G&mb&zLg$5;*oD>J*rndd;-cFpXO%xeVj&F`7esJ%s%F~ zm`j|$9<(oRS&b`}=hn*D__$nyohC0VC}_si9f%%&&x%|cx)=ZSL;RTK`?Knn8$&6?MMMmAxleIOVMe6(hkZs{N9?f= zGxhA??#f>QaVyWiz}{AF$Q1NS-E@loLBV%QB7<$5jDoH_+wwQ|ir$m#0nnY{zyr1t zEgyaTpATT?_o-9TJM)0j_i!t1r?|{_Pba$yM-qKcGVrIcC2P&IGYa`@J#iJtK$v@?0e_gVXk|SP4(j zn>QOLzkTfRJ&8CqKARalGAoIHrWiq5*Vfh5^0 zLY%e6St@*N7e!j{hMs(0+GY`igeN-V-Bo=Qv-k}^R+-7icb$z{w zUcj>Y2lX1zj`7Ej>GluzmM3R3{HZa+l^4WBL^hpzH}i{X8&A@(y=Mul8SqIE+Wsi{SbZ8K0phCYA3Q@6Ktz1p^gT4i{{&M?M? zV1QFoeXskjJs;ba+FxmkmfA*T1{$tpG~4J5?AQqqPF0S#&wN-(IqtQ!wP9X{2n!2~ z*G$T#``8xRTNYW?JiJC1F@^f_<%^r0on0C0^X%+qyG}QPI{Ixo70n@B?iv-Ca^W&% zQ^#j#XE)A1DhkhZic8-8WczdKoU}^WZ{A*oHA8ZA^Yts?syB#m~6PyI^egefsYB zeRjI);H4pn{=sO-*Pfnl9&sLPlKA1*x4h7e3q<~?q*tAZyyg3+I6Nu(yBUAR`R~Pb z*jq8y0ruY?N(^x+?Pu|KKS^dM{Zco|Oc?4EP;t3p2$~!k&Rf$RWIN+NGxQONuP;T! znN)VWl{@p)NbppgGU>9IurSZ@88JgVqM0ET&dbZ|ZfBRxS+v)lEOd8SIJHBSscxYe z37@X?_*$vUxda6^`yEl`Y1+#h-w^!f#p^`z7TJIBw`{ zUMna>T`^Qtrl<0bi{a~|j5MMwDcZV(I@iUsotvRjm3I-&MnlbZw0HZrH@kOqayG1$ z+T7wNvifF+ZLQ8~K3V^6s~W>ZH+J)7QQe^_TyHlfcwhd?@%9v0=>>Yr%*?zBoDZ&1 z-<=g}t|+T5Is2GBzuY|i@auRWtraza8cv++-qzx@Ruovd;o1%- zH+KthMfr(?)tKP-eT5cH!1%R9&Ff{*6cKm%B0n(oAPqT&sjLy1G79yer*dxYA3Nx$exzjt;7e6)gv3s*c!RMD4+r zS%a-k$znu4^~Ioje!$2yEPwz0{mhVceCEO4=n?;pM4~EJMeUn6!nv#pE)FzoQan2q z4TVJm_V19pvK{_zAD17ST$@?*ubPS2tWqP&Q#0WN+&5`Otj;Z$9 zTR{s{Aqx=)_+%;D*wPGbMI`@e>dqaD8&mftUfe5*1Dm);vVp0;!&X&`5m~lM%2HwI z?_b5a&A@SQes1oU!@5p_DWoj1ENUqzDEK(hfrNK#Zqlx!b3(qkhyv?)4R;VqZ|OFG zD732M-%0!I;Q7=4{fR-~D6Edu_>&b$%&&AwZ>M!rUp$nNm6g5o{+8_SRKjp?h(M z82n|ej8{A(EN0K@C#pw$&H(!jd?P-h*g9_552#hZYUtQ(u6pvs#`XjD2-fS~ltpG` znkxel0<}E56&Ow3hqz0B~i$X>QS%VXAj~5`o&pdjQRM;M1vUN zxPDSm>d~~PSK)rl6%69k@Z7ay>z^!T6WhdntyDHAZN=gCWseN|dtPsXo{J#r&49tt zVaw#kdGG>{Nq?f{$JFw4U({XPTpkg|lBw*0%+K$a5Fa6O2VK6oVaLvW1<$vE#Tm() zdU^?JV?*vIE4=0n8=O0JZ$0KSnRpZOAOx+bs91*e3b-7HE3RJ}t|D7bN=l*_k@eqt zwcx6pfTt4$XZG0Go9x3l#$JLdAdMpM=a&u($^1(?FZvHd_jCcfwHT&;dk`K>X{fb- z!i1`j)s|6yZ3k?ba#IVx%%pc}Ybn;NiqRQCKOI{~Q!+A^-dWt^nId@ZzPHPwn8haS z+8&P+UUmD^0`c1#mlMXeH$@8WOOC98f8x z2AQvTfI~fgUQJ!ya{%#vu<&hJ8AZ?%!DCFt)VjaOjhbqX@i#0qzk*}mLM)kpky^lt z-akJ%vO=S3b%sI48E*1A7dRtwDzU4Th^_3PDRnjx{CIK<*-5Pz57dhzr*k*O=_(_6 zjzlmKh#abtB6m)MF{7IFpLu+?89d-7B2*FLNEdyj$~}wVC-rs`_PR z?sD%%rVO~H3ysR%p48tbl`8`%I zaCbUMb^Mng#N4m8%ze#gU1pWQDqvXy)1g_>Vleyxf#p`DM zaQ|${sEXuZSL`SPxT2i?)(}$CvDO6c>os&(7o^WQfgTD<%J+A@lcvW%W40$nYC7%*uXsY@M)D0b;G+ftah67)2ePhqmjN} zU+<7BPc$+5ed$cUp4dnT$<~Q51h;mdKkM@YJzZVSqN>T?&N$&b`jV$r4KR{;U}d>t zbl!9S@bp1)SxsC@BM^_KU21;4j`4giR8<%hbaZ%7^D@F?wzFd4i9)G;N97NXkwC?% zSG$hg_CW}2KBQSQvoqN9`R6Agc0u}AQW0B;L7b1laaSCW)_Y4OTI5knMW?S(7-EYo zdL&@I!_q=FZjh=Z;P@<~i87lA-RXR;o97q!jEA=R4r0l2;?hNn^dw6&vq9IB#r>7C z5z;W&r)eLn3M^UI6G8bkp}f4jauD3F4D=S5k;(1>M_>$`AuzE$8q6NN)sLz%ee`)?;33f1Ffq&sU;q7b?)2+E)XU$BPZXJy zLX3=zCf8@Xw&yYBxYdgz6&~D{rS?=Z0Xr~)!Xt}}2EKmzau+yf-MO>oCCjT%YQI^D zpr5LmNrmzC(&m^gvOOa3Ht}Y= z5r!nvtu(MD>Ipa?&ge-IW60^es)=txW4C+kVyh&NZY_q;=y-bak}`yGQNb8v)=xyD z^-ZK0kY}3y+L==rzm+9?vZcM9BR%+LK(W%mZjImZQ(sewF5PO6ofRO5Ed}WkO9}Fk z`;v^D=bOWwh`j+_ZnY-Zc8@6zWes8Hb2B+B>#>_ZtNqhlwr8Tq1kT)IFdQX`R4zMD zn!!L9`qvp1ex@EghwhoIAI8yM8xVXf_%p8eUaWD1;Y^kpM?H*hj)ZdM{TrUC(>T4? zFD9vOkld-}eu{Vpc47F(t1&DVk6#Ks<_i6;dcU<=n}&q4=^1}Lk(PRdT*hrNvIxl$ zBF-~-|8vwtjioapt<#W(I~-rKgd3nC=(_^s|1k3G;5R(3xBAVE+t$WCWDH~BjyibS zm7H5TEzGw`Nqn{Vw|yVYzw&jwL)7&81qH00w)@598Zo1bnLR)H_E##l zDa&h~lK2E{J$8Z0`Qy9U#B5`0u>VL)YA&ME?E~)7&^Pz1 z7e6zHEPO4U$tL?xW=NgZ^_LHA!lezK4=d`Svl+}wT8(h$~Mf1H0ex9qB{knjHo0!AtiY2581DuzZoFpo#&ULmHK+A zy+3=-(PGnc;Y!S%lZyyt|Jw+NukPR3n96+vkUnA!FJuok^`4Zk0>1}ztTJ%bG>Zr= z{|$Sh?+s9Y$+7%{Hjop64Q?63PhL_H~uu_Pzw^8fDi^Ji`5 z3}g{u^nXy+n#_@3SP00!`9C_ne0lf(?sS?GZb1^lcl6t7o0PwDjF|yhNeW5kzk_%j z{E1Ysu~qU%wOdLIesQb?N;nGyD&n1{jXfe8iod!U}!C+#H^lC zup3Am?6cD+-jvyEnD4@)-%P=gVuNh5=28!nq#94pEzjo!-tF4YEg%-)r|(@&)br;Y zee2i1kFIM;bV1Qf%*^Zg{7}Cf42mQm<1Z>88=!ag&C<%--5-dA6$>crd&GZnUe5 z<|wz|sZrh0L^12#%!9Ty#lV+s(wG{a9X6(c|FSuXonk9Y5;!vhkKP*GiXHyM7SqLh zu4i;&EudkK*~ zEA)PzetZ{y$&-3$)l+Hen(ddr9a1*Ey}Lb|G<9=MT$)Q~C3!aU2{$fyF_V4093k`0 z`9x3|mdtYP)O`Ym)^}6>ziEvj6*SWw{&Fuwne}jfh@)k;C~Y7lUsNjMXn5cgDkLq=-mR6K zBxGptKg~vzNC?Z>9Bj(=&#uyw=|K<)Fx9rAH2=qBK0$5A599G0cWBD$W_BuI27TL) zJ%m=O5_w-=Zpg)=ow`HhW9=5=3F6ooS9hbQ}c?c=V|nKY+Poq2Q8 zci1K6BJSXQMwIwh*wE@gpT=IL2>#m1vYYLDA6q^)K&y)WK^`5@^(fcSuTwiLve$07 z1nok0M+>OLxpYPg;wB;(*SUSC5Cn3%CdG*N0 z(&P@%Jiy)5F%*4@M>-Rd5c0+;Ni$6G=5F0eysF)JBDvxaIeOWAeW{HaL5 zOSv{6D?78}^UQK`X9$qXS`^X2Itncc4stB&5S;q}jmQ=b;SekT6gVFOHjm!SCT)B|`Wb}JzUIvvrvrrTixG!Qvw5Zg9k!n)ikk@^j#f3)mx41^rt9G1$E|o zNY_20(ix|~2H8oJH5l8rG7CNGzTAPZtZ}L3%$4BM5!kVW;nBSRNZVdn^+kM)G_VUh z_al^9ee;NkiFd4KsU<}4i!DCnmRdc|q!evw$uE~E*B%=kO}(qrbHN)joFf}ZPXEfi znJZ0~G)r+fQpY~iChO#u}7`h|p5=-M^=Ftu57rvHd z)JFrU(E88Qk2wNTF&@&#a+LWf<04)nXer#6pNgG2%NUQ3 z#Mg568NBwrFTRix-deW#2oV(CM*m_pk9+U=;XCDbJljbxcu*yiLK6ww5AdZqpOIr@x=Ld`!rprJvfd!RgnhoR3CB{Kz45Kd z>s%Ck{*vi5l>fN145`*gxLa5gTt~sKVCQE`oI2a}x5IW2zUky$yxaj)*WM|}Zg7bq~77IoUyb zKN)*2bMMUIgCx~@G5nQqzf3{=-iWV?Hg_jBBFV^@G}9O8pJrX#ML51wvypc8Vcoud z=-7ctLO0oRZZ4B`d0&fyytJ(%M9MBb1p$olifU%cs8tU&#AVDoSYBs`HxEnLfz?!yl76wdmF?S1oj%Ld>we#cJ_e zbe(MPi<=h1W%PuG>!qbD;}1hW&Vu>iXv}*QH6~#p=f?A|_r|&D&R@5X`tXdy_NlZ3 z9bsAqc>Vyc_i|PE<{Rqt+OUp%rxn1Wk2J7bvB@Af)oEo;HmIq@6fIot?f&W1;!T(p zFz9K(NWf<(?Qpppsg-0gMdr8Cvt?`tzqMpVmTJ*ytt%>~1coCZ@~V7gTg6#@~}vv`0?kx&22A@%33-+753 zxc8fzn>zq@{Yd#IiK#_Q$mSR;;Ip4Qc5eHx6Msmeg2!FIOBot)2VoB!Ed?%2txN0$ z>7Y9Zf(uLN2-Xh6ZzVG{ll{q7D#{YM76A|-{BoYb3Qj;O$N}8JD^PzMVQ+do4xm!< z--kJvd}vb(C;AU}rQDGZs16GGGyW|m&$^-VZooE^!i z!TNv2N`%K?c&O>06oC*B1lxq3s;*BLAooQq@~*un#AYMHP5)y?5UFdFSDYL7NNpS| zKGgWGWJU!7V@eRE0ngB;j1b}e9@P-$1i|MU0>_Uz zgE87qa!2%j{>zsu0}n%Sp&Vz6V4M>o(;lk;9)Bv0bfGE!*?la4i}?t&lykyxwV&yI zO{=CE0}i@URK_y@oYaS4%%oDalU!vzv`9^>iQCHW z?0|e2AO%FV)09MCE~?#@z|P&ZrRz6(W4+G*7c&GbX5De~WKq%8;Qck>n*hEnfBBMp zd9;K9sMQHV*q>RrnlOB-HCwB1XMA#AcxNStQPW4+%|3qg<4uCC8nTIhDvhp{e(&V! zYS2koIb#aJS_6AHp`0-)02uq6EzmJW&*K#y?8jsR3iY=7?bpSY`@D1$bNxa{u_88+ z4>LBoGsba~Nn$r%=I75RuCBZh1^Iv{$$@zOuPFTL-}-dc9z3eFmnAK@>6=SqKb_2&-oTh$P9%R?3);N$o)m+PxdaOmM8<+- z0NTkZzGfBp)Bp$BneR0;*L$?=oiOy}3%0v&avoBVovrA5j4{NiEPX2oldF9(cKSI( z#*;CPrDD<#0v{+#I(o(QW;BbDvAgz>bO{S>kbc&gCm#WL^Wqc?SjCdEin6bn!#X@O z5#$*+9*&2f9=WxzdkQlG`LQ@zP{IEcbE-Z>Jj#G|^L)j zQ1YZF$#Zo=k!wwn?052EL09gHRonoQY1YutD6g$$1NT`5d8uw@AwW7$tgxu41++^q zMUQ>tWZzrwK7hB@xYAgGL%y5sAp_C#(1%3wPc~+DphPr0|ozR3%DgSq;fClKG zONLPEt=K(*|AND*L3bY_-t|k_#)B}&>c8N8NjKKGz_kZ2Onlwm)?n(P%>ThNx|^(- zHKBvdR|&Z8&W3XkN!GNQ@$WInpWv@?$R4FJOi>YC-9NVY;t!NRj1>ZHL_HF3!Hh=aY}c8j(j;j)}Ry@J}QK zOjT5{Q#~qXg)85hIOlu|Ffv#A8LkU`lo1T_J@L%IyC_cT5OBeqvq8QVveo=Q#$htWHcdZe|K;VIG+&Nit$U; zrK@ms#fFvQ%b6)7T&vY$j_Y91cGpiGe3DBT*%Mt(;3@UK8TTUiTkL$}E7`88tJ^J1 z7xE%fN9s3e%vL)W_=dpe(=k>VB;lB70RyddN7~+Wi7I{Rg8YnV>FHu>N&@aoKtUt; zh{M`9uJq$qsG?{(c3KY=NS^4PC-`FaF^MkGyO;a*O7Q@KJsn$U`NHQ(ZP7JR06L8gc$KdO2#MZN93AZsKU`Tc0z}sWAe+b<6sSYEml8F`p zpAZGC1px9tqOr!^Ko-D3TA}w^|4o`FmirI?!vAwSxWzyI^5xZlv3la{H8M!Dbq>!| zxavs2S151LK344J?7My+)z|+7UN4zM0fzqR<}kLa8yF;-+1mgZ8+iNb;8aSi-mOF! zF6>PI|3KN4CF=G{Plba1T*oix6D-5ZI{O$>wTo%$ZFvzYx zJ_Uc#T@U5;+$RAb^!u<^bv|~V*gW6<&Mf<$cI}TKvIt8)_7096DnO{kIpz|u{-J>Z z^2MjIzcdquk4#}y-6aWWhw1&@#qFdt3639qD*fH*zYA^>Oj0@39{aU#`#@OCfu`~g zZVNft^GyGXH+K+*-**7V`6YyqU%!3@seu1O8~zw{^Ca*O|L2Imocf0L*Eat=Aej?9Jh|vBBU5oh9UU<+Sh=}@{M22+KI?WNHXkg>1xS>Yv*e0fOn5({?ODq6~AAMSs zx^-G+J@Xx;x?IX?B^dyYH*1|YOLm8YVtD&I+1_hidndW!zL_fK`ka8ehtXIzEf-jX zr*74S9Qhy0U!_bgde=e#;F*02TMU>pnmfFfCjj)TuK3HOyb7})Jfu@hO;OC>7Ehnn zbGaRt%zcZTvPA4FFiVQe)2yMFbA?(Sver$B;>&B!I*V8{a7<}`4MUMF2b=D_8w&(j z!q_H?Gwi`pZDuC>p_Z9^QP5Gd?R8Swy`<>~zraN+vBf3=`a3UwYFv?ymH<)bP{QXI zKl^Q*a8d7(bd6DF=YKz%E?Xi3aHid)-$PX)XkS0Zf;_ z=)U@m`jsx3_aNI@>#O?!p$2#{|Cw7*5E2^3p8r5gTirT5!&jojg%M_$l3%X53KI+) zH*MHkSHp5DasNAN&*8r@F4xMJSK8lpVz3ux7i;1g^GKm)suJ8~AAe9EKn8(~%?B4X z0Rf2xtfY+XsbWi$;ds7xEnt|tnSz&{R%N@`-V>5H&Le>pgU)0jd_4N3Pe5qUe=-1* zpO>fN`xkE4r^w;58lWH%Aa#vbAsTTI#23kKm)F-z9Ru^stD`io zUe!6_r<#6T36DOiTmMK-z_gQ_21#YBKBdj~#DW{Lrz4r?8e%OZ-YUm;n?4d^JaSv* z3DSFXpI0->7^*5??rB4>2T6XrWqh=++b|zc8VyE}^C%oU7+H!WhM3e6*_QUBY#y;V z0PzjM*X0DQ^jAsM1-oeVfaD_Al_C&ysm~TV6)lzx3*D;iQy$;peRa9zh~9H1{$a;l zt&b(v37^)Q$gwxuA0n<(6lYGGeEM*Gl7?dBdf!UM*s=*=%h-X|maEN2XlXCo@Q=MN znn_rq>Pjg^n&zq#j&%l{ffh{c_zq@}pdhl=Vu#{s0wfSJh{hD>V^(mmBsb7$@oq{q$FYI;sSO zfYv(G7*zrln(~RCCkflDZJf4jN#h1t=}&UF7RoN<8T%PlHd>2-8Eku9kA+%;$O^zXzk(=wL?m#NZw zZGMRT@%HRp+ST_}mwQ|b&Ijb~z@h+Ln=FI}aNB2(XD3v_rF}Ukn8CH>WM zNV}=OxHCvg1pWM6A?DR1#(+Epv#7s_ZH@^issn3Kp(Lp zU)8lI8GIl1KyHZ4f9IwC9YUrEU?y1r*2jjIRGvF#5U1Cok0rewGxakN0w{IJ{y1ZXE?6U?Kb+gmg4@$!fKNe?vl(-c=ryq*7wqlqsWL ziJ4=|1m}PG#jFxu2=B=`SKmfXy_SF7K-ki}E%66xbzyzg==!_K8+U9v&DTTPD)>YjtNs`2Gxi{^GH5*WXUoakvel9pOLS>W_b0{Hg)Lp%0y( zpI>c`pig8ZV2T~*38?$OL)wh(k+~g|adK2te|q!)JAd{4FCJn@^6KN{yrLo_%fC1io6`P+nSG9|xla2(1P{M_(U4^8=Kl;X+K}+T26q-h z&8685iJD%*b*_M{0#GvElmcSpF>rlI)?#v2bcGNH#(=ULa05R>gm)I(M-*FeujDp> z_H=~g?f#wkWc8c_aF>^sTeF<(k;mXUcYq{=@VgD|W}E;3Z~~GHP(}0!nt_mvEpbq^ zIwMZ4l`dIYa6Ni60ObXFgF9E~p8PAE;su3ozR{C}Qp#ZxRdBtEfO5}@ zhBnw9q<#W-$p9R{0N{4q(c|Xd>H7wXYXN;R?3sa|k`cX3d;o}<7Grvq!g%#KLGqVa zjTcmq(kuz%!M$8?w=IN#VCze3cxN>|(f!rG(Dkr{@Z8=nh%3n;9vgweVG+??f36xA^F^g z96VC%f(AWP_b=)3*=C*ls5FM`JqLiABGf{h^*KHrJouDLx0Ml>` zW>DHrnxrnweC?L>{K-T>$UQRqXDKF)wzSsQ*XO|vb5JaV4xRZ@oXcWAIg|qnunRu> zE^zIkR%$5#bG~i0^6q>*yr>-Q>|_7x)6+w`*nsobH#D>X3i)ld-=7O2 z0dHfHsN^&Fy>=p$f$uDJ2#b+lt^mUnFx1f?octVJS35^RtRxC{$Q{HMowKs0gvU&%XIg+hnfvzb1tWl;9GKBfHl^B+5zzc@k)RT0j*5*ROtJw1 z=@3FL>BrKk&K?5300;sF++drtv4|XU87TB7G>j)aCd(H}f{v6=g|3{dXTCZUUX-l=fxj)0aAxQu@o9V$VYX>4m;p_-am;* zXJQ1IwSEvC5yIiO|0QqmX_exD^y$J;YgZR$TAUhGU6r$@RaRD7GSX-{w+)2uyj6~s zXU>Gq zue4U;mjZRmr+%m{Ey3{>vSr%_(@eDkB5Mx(1INu>177+1~VJ0%ToavDj@~4lqwsGufC6Tt#N( zgZx;}QR03a$z=bAl1AeRGoiZjnPJk%{RxUCITfLxQ0-QMdgxx*d4^jccko1)QcHd1 zvQKI$o~fXE*omf=s}6jzfoq2GD^zK}8z>f8h*GXGfe)eH8Ic3;)bFA@Nb0Jcz)h) zwsmlB87+e3Z2*^3mamY*^jApLuY_@5#+~SW)HE2gXqC2FcgnqNc6yq1LVP>pRZAa0 z)9l(*a)!qY4EjCHrQ;08qmM~&4?J+B!D2^x@`MH#N?SD7V%`d|eTjS>eVJl|^tv%a z1+hi?sCBF~s|y-b@HoMfw>((xYo&>E05CK{B~S%alP(ygr;EEkbLK^GbBn|iT@wE| zqy{lT|A9i(r(khaWL`5Imym0zq8*P~-C6jrT{~z5&n9*h z##Q+&e*g&`(B4{5Wq^Ao?eI$0$11aGAS*kYEa~y%!+Lj?-Vf2EQ&($jVuLi5)Z_ba zuEnrYYJkv)D@=&!*@J--@udP9f?G|0>6SoPcRIA|zm}Xw{z896D6U1UWThN>G6lE?eR} z4vkq2amg{U8|0wCRR+`~s7ilfdfA;gJoxs8*6ZffK1!^m<;XfXw?Umo+eqPWBaYz9 zAD&E#r^QJuz9=Qi5;2aShU4pL{hvJoSH_<{O9W)_bdWHv?R#LcOrUf??J=KDq)JZn zjpC~V^D$xb^WeK5K-<~ph7V9NtW`$u+tbFsjYK6`avuXU}Du{C0M_j+!EsEh8bp zqf?K2p_PM)@_ZX%z2!1kn~_&05=kfIg)%&({nJ(lfz|&;D2}Ssv0Jrv_jo1}-Yjop z^YE^xB>K06eOqt|;og_v5AH(-zygDUH#>}(_O@j3VL%@VJsW0AKjZWTdFABpXTKk5 zbEM8lLOpo|(IooAZElF7Ah>!=q9}xU5B140LGBC4RrvgNub((PoHqA79o@hqn|Mc< zHebZn&xF074bl&|-+@=?mld{)PF-@RgOv#qI7knAS^B)&11NHkQ z=#WSH;2TOfOJP`nLXrwPlcD&`%%ecJ??|qN&*uVCly^iBvOI^${uQA3I z=XH*IoI+7P3>g!?v$8=3ZYz6oJn!6=>!tsR-;DR{fguK?3`nHl8YXnhI*>#`!lAa* zEvj+VYH1(;eZMG!p|{~{pKc@9=u?j(qr*#5(Yt2J8%Ba~($d@@lRctdqSCMWn+LcR z&`f|DhA!-xkaeUt<5fEX^a6Psu60Vtsb5W(ED+~lqL(I5Ik`?wuE#KP!muVYTCy6p zIxr*LJKfmZu`oBc-F?gH_IWH1t0P%~`ttDxQ^<6Py-(3mfri_N*^DS3E_?-A262vA*_1h-aTD^D;du(hL|%Q{nYbpXYUw#PH< zgx-mk=S>)DSbR8Y7*Bggxb6$Xj{GLPtYT2xXNQKU+}4~Jr?DvZK*|A?5KJT-P6E_Z zy7@j=P({SAAeq6wX-7pIBl;pUy zG;3M`c=18|^-J4jKdOy}&N}BR(W^|ylx|+!i$SWKJ$MHlxv{^Kzo^R%15D`azD{QGD^1JEh4G1S$sa44PDGFLA9sp7JSTYx8yVS zh?Y+*-i@_)x5OC}r%*<0bYI-p(9U!)bxwZUuss;RcrcgH;qJzzGe0B`UfEvv9T_Y% zB~K<3+p`QyBX9vOd^Yyy`1f;{ODHpOaSnjF#K(3dnKi%PC?p6Fy5HQ)Y-X}l<$0tS zFRfVWxv7!&L46`8nJlK2=@@ag)EHwPJR&Yfbl}Oh#JB;vf#o2l?)0958*%*Abje48 zTnj{uKNRz~x!QF*;vFonnO%&kfJdAn(q^@YxVBQ?21>{?2uQhoYU+^iB;K1WuT4Iv$2t3So!2dgxo z!@E9Qp%i*bcJ;fAJx#}9{~MQv;0m>A-FN=8>9cv2_5V^PLhLgaDTTC0#$srO7qcxJ zdA1S)@wC!%El_nyms z+r{ao_^9aKX)i2%s87rwWqH7C8fWA=Bk(ST^5*@T&D+CXwVTz`8^a{~yOT-J0OUB%4`IWQ{N zhVM=;%7gQT1w00Bt{>!KUj{XH(3$4u(xVs&qllZZlomUp=Ts#O*tA+wX}@>cs;yVF zr~kA2nHuwECjFDn_Kju{rqPffi1Oke2OhI*b1}1Jvis**Q=&W-tdL~Nt?vjSGPur+ z4yiMwr{~MOd6J-@{)9LJ4YbwG?>RML$bm2|f~TB^5#mqq?z{MCQiqlUm#Yr3#?TZb zlXk;i?VGhllhElAfYP+TL#y+d*uc&7@mo`NdhU*FwLJ9|f=jgT97Ngu0})EB#NXxE z`7nW$2c-HF&_-w>tL5KH)=4;P%-4CN%aWTQ%piUBlo_B(W%p)X^w-!`9DiO9l%XZ? z93zZh1S0v2tj}D#Um3iST94LuTvxT#i%VL`t}Z7$XQ!Nf=A2S2W}6mw#00l{lHQ%n z$6t7W2pW*tV=Lz9hDza-fh^fn^(UIJ33hC=*TD*IQS>=af5*U%I%q89*9){`$be## zMiwU;?f9w&kMoJI_lNFe+O$Vu>m}0M zy@))Kz1O}kN#5wky2HvR_4JAcPb;3V?*&v_RPyhrxCQQWbs*7jhoG7(CTRK6YeLEy z7EBQsK^0~|Ur4r;3Rb}05J7Dp8adi9Y={&n75R4mux9qDzyZQ|7M#v6c{?%4Ns zj_m69N{aCSl|yJk!h2vnl+xXdiVt5Hp#2c#>AMc4P%)!*THlpNlvFVGWf@J!CE7lf z#I|K>6~o{+d?nhC4jk~7xnd@v*D#@xe#fx7Rb69R@0RB1%wJq~sm54#9Zv$F1);F; z@GeZ72q%_zL6ocYrShl5Y=HJbF3A2zDMa-N#Fz;Ecv%iV>d?7A*oy;QEvU6qWRf(; zi?Ik_`mlq-qOmy%BWAY7W)N_E0K^07E^7eEJ%Lh~j5s#Q@M4Jol7w2w?oe38V|2MB z3%=f!brfLAInWeX1lu$8n3}WSJ?y^WI=1nmB=E%gA?6sRdRpWwI&WnzZ6;1Rsr`Ch z0x7rKnw_gJ4;d1bGeAaLg+3_C1Q!=OsCGeY#`D;r6llf;3I(Nb3=(@9&JIY31X9r& zTB84WDr=DoLCk-PuIUPNwvl1%dtJJ8@*oqaCbRz8)4?CW<~lbHm+)0H$@;z8GQcSN z4Uu}K7Y+G`sDz1;x=)fJ?DTk_*yZoxqM`e%bwx!D7A)6gMBRZqj>5c$_|GUpkTN^s zupgy~+={#wmxst0$eSUB6Uqm4R?Kuxux|~O`0Z-LZGP`ib|@$?MCRFGrJZ=~ zK;J|K-I}#@UAx}VFuG<&1t`cdH7>G;j^xr7FVMHnWBNSwcNzU-I%oeT$ z0`b0R(DoA~sZr;gd@zFu`~p)s%#;5)YnhEro9|R}6v|J~C`3al8hpQ4Q;Aj%o~-BM z_|p6Ik>ml$V9tZ$Evsk8$F+JnLy2WuRTnRiGfWmkl^p?Mdp2@4P)LNT$&Y1RezTLJyl4>_?Sk?D!P95 zp4>3wi_HDdp)pdRVxo&451+idF9C4&SfOtY$b`*S%s#Wi2{1v!IL7<*6BI|52Lr(g zT-!3Da5W*9C}#g7u$0Kw`6cTi^-qSRio+VPG7_5Tm)_TT3i zn#)TNa8+*e)$|>UVSM~JDUzGt!C*f<@(=}7&;lr~q383lB2VKin`)S+bGPL5DwP?A2yLg5JGalmi;Yo($iz$q!c)bnmkN&S^COdZ8zfEyFYzta}-6O0QJD#p|;Md<{D7c0O6d-_i0WMwBK90u!OvxUO-!fZrc3(k>&6>IB&guU@=U zPl1(wk$wK;K!ND+<2@{Chw|kF7n6~}U3z5DrEd!(3_F}BEbK)*UIplA46xR0Z%BC# zV=zA0EFCjWu}OC{RD%0tM(#B(1~(W3&aU^iZ?c-cFg9a3|IoxaSbdT6v{w!0Z$AIW zsGrXTFTv;}rf@b@X>nQpFQlSau60b6Qn>p*tbO+)fP$2Z(krsDy|k7rY<4) z1o}{Ri1!6SG8*bP37m? zhZMw!mWTXmINP2=eHCCFbpY8y@*+or{&|Ow=E|&K?Y?^gPiHp&LoP#W`!nRA6DLQX z7hJ4XN)DjS(a`xN@D}ut8|U^;XNUVI>PxHL0}m9zNk!+|Yfmss5%y;;RxgK@4KJ%9 zUK?s|YdR2AIkF2cggQWNhd_xbpIKp=j_gH@n>bUM(3XrAo`}c%eLPFfH~c>5Ts|T5 z=w7UOgB~;>QuDl*)FS`#r8V88ZxgYTX^-@VC07mi4o?rsYw>iT6r(p5Z!iuCB{Gya z%Yqi#5n#>?AaWQG@mvNA5_@a!4|)v9&DM9a`O-6{Tk?UBpg}t@487%XNrsz3OQ`jK(T~Z$8!v{%Cr1v&P!N!*0$!7_0%<5St z_Z9r`r*#FPDf~j{72P-NwKAp}aatI=E2Ylra)exJrBJHQA95Jh;4dfXaCQZH#l!w! zpdXe_Y5tlBxYzk-okK(*@3wo;!Ef<9X|#TIHqEJ@!HHyTfY>;jqedGXJ&b?~p^~TY zHv}kB(FC{vg%CQZ#DYgLMnj0MuggCKA9O2ogwQDfBvrZ3=ooR({PN1(rUFWsF*;U5 z;4})f$F!a~X7R&ZI>2QZBSz#}akEyTMVz&TLu+@Vx%8ZtP{~<6qCDX7i*J57l4L-T z(8!ZZ6^95wE+W8%K!iy1(eU^CM0T8l}cXr)kl^u&x>F5pLLuM5VZ?Q2 zE@%5MNDB$>8C-sp@zI^#CQhVA3c!8hLP~Q_MHmHE8DxDaHbY)d)cBkANjT6CT||?0h47j$B7sbmy;s-NX=A)@5!F5j$O@{u;gl zDp19QdXXMyBZAayuPgPK{e>0N1_B%UE__Y0HdlA+g631N}4 z>719;ZFB6KJIfyXIU@V=MR8%lP6BSbmIRtSZuv0w;h3-EH*<0qe2S^NZrN6mEEFlF zUAr+D+b+T3>?AkcdP7U|2T~?yU(A;lY@W1AC`~?1LS5`;@&7olVT$D(LvUwVikFpq zE2?Xf(8p?DHN%oK$`iqKUjfmaFTt!&G*eJ-4$@lrMU*>JDRsU>=?VxKQoH0EQ4?^& zke>m#qkO>BAUfHa2u<)kWz%SMxFFcXv$|;|C6s{bRYKjA1P3#NQ1X;%%}$d-`#PpM zA#ob-S7Vcxx3jwNS%0Pa&88B*vQFKyt`$lK?yi5+J^9Y9R}iNGkW1BKKzGdYq)L_0 zx0}pBQXT>yjB%myTV|R(L62;1wMmq8OR6BcdT_PfLx89!c#YlX#o}w# zi?VK|P`j%^X9fwQcm?y=QNrc8NDZ;prT!^pM%P)ArmV*v9l;#f=#Cd-Jg)B7Aj9+{Ha!Je{sqj-y zwG?cXwJzmS^x&^_+I!?0-VS-*&g_%SrV#7!xv9Z+#n6Vmn7E;UB+$ybt~lX&UOKzXLvRE^t10Pa+pji7Re;3M_cl^C~2*> zU02TPGI7rNxP4m1WRG%b-66^c72aA%4el^SxCl5gja7ah?k@X>@3oQxt{fi8IKgMvE^rmy@mda1(rO%Lo<9NF#H5B3JdO zz9ykz8=Ak$Lt|Q~G6w?Qq6L^hW*FI8j?+3);DV4Uq*vdc!~skX>WNT3519JuvY%%q z&2;^PvgwP-q4>+g%mugAF|xbV0HcapT0Tcembpwi{J5z-?AO0GqH&)H{}}Uoo^x5+ zJrVWEiX<%K+D+JP_alKz#kLWI3xK@XH{=8$XS)j{`y&30mF_UWwKr=dc+y1zeeP;Y z!OYXs`6GCFBB(w=09tG)8Q|z{6+=DY^}YZ&sIOlau6m z%C79G`x{kMepcqLCj@&zjv zwAma2O3}$DNB4UvaC^0PsT-c_)R=8F>j@gDmbBfq-yG;Kx0`EAS6OIBZB5f-93<&c zM?b%%&qpG^dG?=o0rFV76AP5iCnnTQ>kjG6Un=+-OPW+N)RlRD+S}dD_RhhJR*&(E zJg}5JpQ!)hf_>e28WmwQ75V2I`^dT4=}+~9XLi+hbM}qrv-X1@cGMImNhv%$8sm9~ zVmzBqbB2rbDyGpU0^a0}PnC@wrkF;=9lX!5A9ynAkxoexjr~V5(IP=Shv4=>Zr`sn zxaXzU-3fgAhw9wzcK?!!N-EL8*$>ILE-kbHYPvp=BFa~94TgPRZg>AlpHQ|QWPP1| zzdXr-pRRMLZu%YJ&5c(`G>1C6a@Y|D@6{={>$TXa*;TLTkmvn-q&~Z{E*M|e0q9Uu zuF|&-(VlAeyVjlx&iojVtPw8}rF@9_QAB}mbT9bF%ZB#_Z#++RkKR5w9pg`*5bHOi z$viEK%U%7`F%{B(-}B(9pdMVsOEp@M&?GpxNOf}jVnOXwsK;&SyofRiMRL~JIGO(4 zf@oinv`8$e5#`Ps3L}F47O$5}ZiAa_D@Fzvu|HsHf2KcECsthD8eYpU$j6>YqqAWl ze8d_HW^ir6G<5O8^*cdC&ZT+PD6IDFI(AfDm81NTG<}0 zvC)F#iJf-rTzBH4*K*S+)Nrqx?RozyT<4^5SztVyxR)03e@2O5kLGfVLN&o~*cA6h zwN9}hy*g@o8Ls2mdAAb52ru$ zJQ=>agbVZGrTmeXxBD?8Z1uTaEoE-0eq}%Q60h;q-T6$Q#|+*ezV2DO@u9Jdb%nY&|U1`OX>Uc?RAi zI_AP)vy9kVrse^rHKI&rR%E0%f*B-w|9@ss?d5KSK^d8X)b0EA!I9`am&XL7h3J0J zM#OdO?vA4QbEiJaf%^5ogJiaC<&@4>Jl|!yPuL%IW!#)S((|RInnVM^A;L!>sle0( zuNgYpbJ-!hvsr%uJ{Vf`0~k{coA4kDUG-euP9rjyK|?jv3?i3MBhd=JeIlV?=%t19 z0txY>G12W6aRWU4W#8cMSX7`%XyFLf5$wSAzJF!JCCO!6)mJB^o;DU}Xg#B4MnzBl zV=ZeQr!oBAyCKrSw)Q*sHo|VSKi<^sj_&APr+v{G(l{e%yKwWF&c7EPZwa)lPF~r| zu#BsTB{;Ve_tx){b&;({aB^H8*xQV0Fe<^2Vj=J{g&+w$h3b6?Szcns>qKnu)yY4&_cshR6EjZ%2y0dk587^7_ z#&{rqRdnfUfUP~?-S@#sBTP=daB!jMs?LX51@lL(5HRMMz*_z)0txHPPfulwbQ3DE z0?aHu(X1+Lbu_CW=`AYS;2_=&v7^NXiJ2FTuL|qQ?>M0J6BzB;4J{b-jEi!velZt4 z=-{@srW^emVY2o+eSHNjeq_2?#?|rB`Ou4UxqD$oJ$-9>@Qr7K1HpLi!gDo4mo*?n z1P!LKb#Cz_^K;1L@8~x&W~ne(8)ZfGzcLDeO`8#j6w(kHFI)A-C~6=@A0x>nMptHO z-JBfU+(R1wAVO898SgJ1ki8lb9XsRMq~F*x*+uC=exnTx;po9bns;W{4>MghU~=mM zG0!eQcnY#JD;X3X_%O#{gUm3WAYlwWZ^NXjm)~~_+M?K`tP`1H7nGXYC{z^7Z?~!j z6ua}^XB?}=C5P0uepJLX#>TghPbC_vdNkJJu5ZiqOFG)_T28)E`w|S9*8@1i}BfJ*pz8Fxn}=_(Z8tqBBG|++yG< z((+MW)ea6)8{hNXOH1!jD*}z+Tjj$pw+iB-dQ#l~VVgIpky+N=FFOwg6A`+>fq0Q+ zi`5k;cTtRv(rAuo>=UMh_c={^r`jM*Q<$BEp*XiV*)s6AOcIu+24 zF8bl-Uui+0*Rk{W-L1RuCC#PS^KV~keVB)*^B!69!lI)&*pUg&cn!?(mb~+Z#r}KR z5U$oV5~17S?zZRdzmF@58uVQ0nUPHFUG8=EEy;gMH0DmgJZA{J!JVI-JujCmob&e> zhcUM9_vW>kg^D{gkd0vp_f%b)Ki;GsXn;7;S3X8D)a(YIpGrapo~yr=XT_WJ4p-xO z)}%8C&DZ}Gnz`6N5CC-$>EWbNdAht7)5 z7KQ@KBQkRC*AFKDoY|+yEIKfco#Fv3FhJb zT$W)&@5E}0ZeANedD%WW+Um1aDyhQ8OK<-7@dkN*PChVEs_boqY5m+D1z+4gjk0aR z;|B_cK3Yg`Hhw;cB~o%ML3lIU!^~A_K`r)T_pK7F-s7yXyddCTQR7c(BtJnXS}S|s zH`x`}g#QLCQabpMrrVol+-?VL@E>4RQ09b6E&WJR&LheSDl$IQwHFoMn}6)FO*f6? z%o2ES--Cj7Hq^c?2j(AKR2O1R`-d8<-2)9<+Y7w{^;h5dBW0`eYL^cx*U@@;d_Fs0 z%vx0^2SGS@6{dkN8()O7bn#VT)`(X{$XQ71mq`=cX z>iq`B?(p~ZqC%+VzuY2j1a-gO^%X@8^gzEp3;!Fo6C>OyD#%jrx&lAk?6+T!vrhXF zNAc9`0AQ5AgUareVu9Op*W6E=L?sUam`F~^=#JiT8g@9I7Hs4zRP15ofCVVQA?VT@ zlcDx*u#5H`qj2kZHH4h<8}dc=->0{F-g#NY=8JnQB>Z|BsnB@$-kpvmJO;H_sn&ha zSQkP4K?UkWY%mIHQ<5k1{nWx_-58$W#PCE5r>kqUo9gR7k@|tmRaPHS3A~& zGEgW{ec$}+Q`}t#b>=4sA5YH2S6o)$?-K93+QHtJk+b4NHgi#xdwuiDeg7 zN1>O=L8+HrMA1LgUqd`vP-8yISB5fLWj~rTV~;YLQsc8VFk8NL20b@}O$kYybFL0i zjqEQa`A9o1J&$MUulVjz;Wvo4*E-D(rj`1&g4)mL;QZt6f0xvc8>W1=*y{prPLq-{ zv@Pr)D68RC;}+Mn0WI!By9sk_b9Ogbdul4*AIKRf>KJ1r8ft&y(n(Lnl|JPtdk9{R zH|(+~Cqv8|5(KYT?8^F`WkaHapPv7~ZkJnk! z^5+*I!$m}IL8WAu^#as7!_L#v-%9|7zq|g&tItXoesprrvj=Mhg2265qFOt5?|xFy z?xj=u8utO$|6Z!)x6JKcvwZbot5I{A@n-e_sDLaxPCg)bK}$LnihFG%^Tnj#v2+DL zS~Qw2=oqp#$z&-Un0~F5l`SI!gBpg;Ni zlAUk=V-v{-vn`JAV30r);n+{{_df%t&#OXpT(Q53 zjymUH8=|9k?7M52)IjlDD=P;xeAW~YHU!+^lH^0(n_{oNg&hDU+%65ZIDdQ5ptGfG zrBD1$9Q>_=M|Qc0?JUG+B!@F!I-<;9pO~^ z2+@NWX-2}Lq9|i{1}%)*Gm)mLfb_ay=6VD|Na^Y?Q^5_9|?wU=(8YxwENegw*F$`<Y2dHg@LCY9ggwRkl;=nuHZNj^ zndU$lqc^7jP#XA}(@%5J9GmdglK$CUPVM*JUYIyqVif>q{Q4cZ8iqlAd2oD;hFGq= zaKQU0+bJ@7R!-iIQM%*vi~6H3t49oH?x&5XJRq9VxbDRx%arnjmP%x|QbAk;AcWgw zDGp)~EZYSDWWnN66jO7`d%hiC=Qg}g9>W_B%|hl0$vjdqI(MDT{S5!J9=~1zXYFMR zw;V>3uFwRpaafLHcNjWFnI{#-1Oz&Jrf*$ z<>ANy%u@kv5cL-x36OY&OWrhXjGHuI^OyQ-zxB4keC9p$;1>WF>R}98<+DC0YZkqDFh2jcAy-oiX&Q+69Zm+CGEj_s#v=u`8h zmV95LlXw8aSQ$3cl12{VzMm{EpK5TDbnA=8EzN zC1=EdyfJ;^79k_*AGycYTPe?ykb<>V#bFuu=4jg$e}=Do%ftIs&kQ5Tpor6GVH}== z$G3nxWEtD<4Mcnb)rASfm z+)fa#a^yV2bnE4MCA;Nki*>v>tSK?N-dIxdRKkAU%fpc@EhnE02Yi>n5OfH}?gIHv zPy*wEg69b}6_UQxAN}H0te!Nnt_OB&2(CsUcdm6t@D2GC?6L*>0Jpsvw4GKh(QJv8 z>eO$W_9mu^lqG?76O1%ezW6>~m|ouI|Bx`7SfGYE{JhMAKtqv|Yd{gG|3#@$GKJ6s z{kmbrZRN^n@NjV8QPI4!o{;Nt^5xph#*ALAoA+Z;rlG}K;^0Ao%^~>)04D&d@KjE3 zo+3_UHD8b(?YH=QF<%Yg5+UN3r^5-GFsSnnSl}Jv0dgYbg+k^AV7Z5HJRt1Sqm#;w%jEJpi1@MhC*;VyM4=^8l)%lW^-)3tOx=$P1G?|2wwnNdY~;ZB{`>wNr%xF!V0V zhu8>T#s2-%FZ-x~lEcV6M|{zAY<2_4=IC}4Q=rTND0}b}vF2w7T5j>pf zBl&>$s(59-kN?V%NU@z5yA=!AgKtQP#0W@gv*%?*k=FrLQ9`BKAY1P8H0;~V592Ea zRsJ@wf_fT&h|Yqc)*n>yZDsWkdn%V9kYNkIL4yWnYKaK4C>@Y@&4I4+n2N|gLa zy1Y1h^~(JT+irx3G4nSJh<5Qe66q@Xd`;zd`U3zQ*7(>!L+!=#Bow1F1a`TaTd*Xm zR1Jn7796K#(Yy3ra3T?(Ki3hXn3i(5aLFA054f-_`e0G?Og&NB1YipnqkmzGTCGy7 zOVNK}3l|7m80P;^Y{9buVGBdhpOr=Sow4ce@k*fY$V|u--;T3IDOQ1A^=7w3IJ<@w z{06idHOsS&JAn53GG+oVMMNYPdIGuofV-H;S+YkNpxZzq_ek&uKY57c4?fIFrA?Dy zB|=x0UavZ}V5}B>23z~P_CjeRYk1Y$^BA$Vp|%q)Mw zPO-ojihI=M^rJ5gU=@?dORrtv1ONo_rc;wiqNmYO#%jv0UjtXh0i*}tGj6Fd!7QuY^O8<5ZduY-y`)Z~A_nr{kD8yR?k$E%R+*0(Raps7WXiZOx-H4= zAnI&JHmu*>H26)*gbSU2*|W_&qRtoXy%qrQ>j0D|mj*2hbAFG}>_B|h zA_N%L11<#AK$O)}^<-(0N=@wUZbFEh0p%Oyc8JDwt#}JDd;qC_HS%m!o!hr9V`7-T z0inj{u_7O5dHmL_&ICjf2&fIPq|hQrA9)|ySORAQam7eFCdD~iIlbqHGZEK&`iYKS zFI(RL8x-HIy0-^Q&cnN$54bcQ1Aj?Hyoju~rKY3wc>niCMmyKqnG{+ExqUkNni#=kKo;b37!p|ELG;(- z<@FZ37i81rI6V=THTKQ!zl5mjmamvS0R~Un>+CyYVEarHRiYa*iC)@7DuJG6^!+Y~ zccdlM*5&ytepDC+WE9n!c!Iz2Jk}c<+ToH7ih-f-XfL5pR}Gz6X-A!B&y2NezV=bX z?Y`F1p{*{F{}hNb@gpC5^}3TmLJV~wj1kqSl7kP7(~@Vzr0Ts4A_fhd*J(&x3IQ$ zL6wDgnv7fm-bR2Z5PJRt2^x`H{%=4696uZh4Gh3I+}vTdQv&{B7|kJoa)F{RUIbUV zz?Y4>9B@jW^#Z_dFEbIUxIs{esr^nj!M1X_34R2<{8?BSXz-pyA|ix>+IxRN)9`*r z2}1(Na#&CfY|g}kf<5;SGmw~y6FGH6fgt8PBqg=;+!vGDulPy5-gCVQW;r*&Ug50f zmx0Qz;r3>U`GK8USf>So7(SS;V;RcJlMMxjl|%ZgqeJPd?P1KhWYFm+Ch@{SQHBfh zeX13H_k1paC4Rr}b&}N}^xWShL4<(};nq$jw*ww9C;#Vcbz4UnL?KiSXmdCO9zXt! z@C~GEsIjvZGtrKxp+Xw^Qk=rdiyL-_A2qEAdXS?$w<&f4z?=~R<5=Qx6F^1Pa(m+q zzl72h)S`oG1vOLW5qrKk66hAxq#?-Ea52DbfsT$!`)zZJ?WOZu0V=|(=h9-R>CH_0 z_?nnqpqPLJ$Of44&x!BB)HgvEg<=q#CshXW5W2C^gTp@w9t%gn*8|3D6EFkRemE^! zJHYV-*@}e7W&q}Ar7BZ10N5byX8S#8OIUXgVWbZciM;}RF}fs^zC0F%@k~pAC;=>5 zLQ0P5-Wz}UvM!JiKG}hzh$OIQ{|wm^9%=HUSql^_@qw3@Q-*-=&;-8$-LqZ$1}q4R z;S6efXeu<*_(5Q#t0Z7|b*7F^YX7&gmlE>d${yHRoCyhDf)aT6rB`2sp(R>Kv~VP? zVcnNk+9waQGG3P#x06DV&~fuqvNM%M&twTSqtG#)s7l1AkXAH94^(2XWjsFa%AmAU zHeDb|fHVu*nj#Mg%WgDRP_uZLjC)zDHCOcwB0yjGEqnIv=u}7X>}4+q72u&mvfnsr z^j|(5Uj}QZ&{7gT`*VpU{61|pdf@j z#^~Dl$sgiwSw;~Z!vWH1O6;l<=mjJ7=5|0~dtWrM4d%qseMbiQ-uK=xwg2DRfPnRg zGMbOwZ?$!BhrGiK;}pv}3mo(?cc*02IjI_15&|_TX@Ptx;wcRpOUy}dA$M(FbNGQ>#)>BY2idjN2iRwOz-nI1|tRH1~v&;J5X+{W1e diff --git a/docs/images/centralized_Bb.xml b/docs/images/centralized_Bb.xml new file mode 100644 index 000000000..b4dbc89e1 --- /dev/null +++ b/docs/images/centralized_Bb.xml @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/distributed_Bb.xml b/docs/images/distributed_Bb.xml new file mode 100644 index 000000000..57c73a0b5 --- /dev/null +++ b/docs/images/distributed_Bb.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cc5263e3aa6f3af52d5f8a345f177cb30ae1773b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 13:21:18 -0600 Subject: [PATCH 445/644] Moving tags --- README.rst | 6 +++--- docs/data_structures/exit_criteria.rst | 3 +-- docs/data_structures/gen_specs.rst | 3 +-- docs/latex_index.rst | 2 +- docs/quickstart.rst | 3 --- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 18a6bfdef..2e81b76a4 100644 --- a/README.rst +++ b/README.rst @@ -20,9 +20,9 @@ | .. after_badges_rst_tag -==================== -What is libEnsemble? -==================== +=========================== +Introduction to libEnsemble +=========================== libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. The library is developed to use massively diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index a4a762ac4..6e4cc0b9c 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -1,7 +1,6 @@ -.. _datastruct-exit-criteria: - exit_criteria ============= +.. _datastruct-exit-criteria: Exit criteria for libEnsemble:: diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 9cea34cd3..b6518627d 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -1,7 +1,6 @@ -.. _datastruct-gen-specs: - gen_specs ========= +.. _datastruct-gen-specs: Generation function specifications to be set in user calling script and passed to main ``libE()`` routine:: diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 742e29d17..1cefce8b7 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -3,7 +3,7 @@ .. include:: welcome.rst .. toctree:: - :maxdepth: 2 + :maxdepth: 3 Quickstart user_guide diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a470df97f..0152356dc 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,5 +1,2 @@ -.. image:: images/libE_logo.png - :alt: libEnsemble - .. include:: ../README.rst :start-after: after_badges_rst_tag From 09e9b04829d30b2770357b799923a3bd45bb709c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 13:48:12 -0600 Subject: [PATCH 446/644] Starting a branch for reorganizing PDF From e771a0c291213d53b11b60eddbd789bc323f87a7 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 13:52:03 -0600 Subject: [PATCH 447/644] Renaming quickstart to introduction --- docs/index.rst | 2 +- docs/{quickstart.rst => introduction.rst} | 0 docs/latex_index.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 2 +- docs/welcome.rst | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename docs/{quickstart.rst => introduction.rst} (100%) diff --git a/docs/index.rst b/docs/index.rst index 440dc149c..f0d169a0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ :maxdepth: 2 :caption: Getting Started: - Quickstart + Introduction contributing release_notes FAQ diff --git a/docs/quickstart.rst b/docs/introduction.rst similarity index 100% rename from docs/quickstart.rst rename to docs/introduction.rst diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 1cefce8b7..f1938e38a 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -5,7 +5,7 @@ .. toctree:: :maxdepth: 3 - Quickstart + Introduction user_guide tutorials/local_sine_tutorial platforms/bebop diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index aa71d28af..96498a735 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -3,7 +3,7 @@ Simple Local Sine Tutorial ========================== This introductory tutorial demonstrates the capability to perform ensembles of -calculations in parallel using :doc:`libEnsemble<../quickstart>` with Python's Multiprocessing. +calculations in parallel using :doc:`libEnsemble<../introduction>` with Python's Multiprocessing. The foundation of writing libEnsemble routines is accounting for four components: diff --git a/docs/welcome.rst b/docs/welcome.rst index 4979ea952..4dad228a2 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -25,7 +25,7 @@ Welcome to libEnsemble's documentation! libEnsemble is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. -* New to libEnsemble? Start :doc:`here`. +* New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. * Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. From 4d3a867b3238d96c983872c9c9359c261f394469 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 7 Nov 2019 14:24:16 -0600 Subject: [PATCH 448/644] adjust platforms toctree, titles --- docs/index.rst | 3 +-- docs/platforms/platforms_index.rst | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 440dc149c..09c27faa7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,8 +24,7 @@ :maxdepth: 1 :caption: Running on: - platforms/bebop - platforms/theta + platforms/platforms_index.rst .. toctree:: :maxdepth: 2 diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 9092a9110..cd6f58f7f 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -4,8 +4,8 @@ distributions and macOS, from laptops to hundreds of compute-nodes. Although libEnsemble and most user functions are cross-platform compatible, there are platform-specific differences for installing and configuring libEnsemble. -Local -===== +Personal Computers +================== Users interested in installing and running libEnsemble on their personal machines are encouraged to read the Quickstart guide :doc:`here<../quickstart>`. From 4164aa6465bdd44941a6ba14a858e71b0a4e8c0b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 15:46:15 -0600 Subject: [PATCH 449/644] One pass through the docs --- CONTRIBUTING.rst | 4 +- README.rst | 170 ++++++++++++------ docs/appendix.rst | 11 ++ docs/data_structures/alloc_specs.rst | 12 +- docs/data_structures/data_structures.rst | 2 +- docs/data_structures/gen_specs.rst | 2 - docs/data_structures/history_array.rst | 8 +- docs/data_structures/sim_specs.rst | 3 - docs/dev_info.rst | 10 ++ docs/index.rst | 3 +- docs/latex_index.rst | 23 +-- docs/libE_module.rst | 5 - docs/logging.rst | 3 - docs/programming_libE.rst | 36 ++++ docs/sim_gen_alloc_funcs.rst | 48 ++++- docs/tutorials/local_sine_tutorial.rst | 2 +- docs/tutorials/tutorials.rst | 8 + docs/user_funcs.rst | 5 - docs/user_guide.rst | 126 ------------- docs/welcome.rst | 2 +- libensemble/libE.py | 20 +-- ...uniform_sampling_one_residual_at_a_time.py | 6 +- 22 files changed, 252 insertions(+), 257 deletions(-) create mode 100644 docs/appendix.rst create mode 100644 docs/dev_info.rst delete mode 100644 docs/libE_module.rst create mode 100644 docs/programming_libE.rst create mode 100644 docs/tutorials/tutorials.rst delete mode 100644 docs/user_guide.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 46b5dcba6..b68b88a5d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,9 @@ Contributions may be made via GitHub pull request to: libEnsemble uses the Gitflow model. Contributors should branch from, and make pull requests to, the develop branch. The master branch is used only for releases. Code should pass flake8 tests, allowing for the exceptions -given in the ``.flake8`` configuration file in the project directory. +given in the ``.flake8`` file in the project directory: + +.. literalinclude:: ../.flake8 Issues can be raised at: diff --git a/README.rst b/README.rst index 2e81b76a4..b8d3c1e79 100644 --- a/README.rst +++ b/README.rst @@ -7,13 +7,13 @@ .. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue :target: https://pypi.org/project/libensemble -.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master +.. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master :target: https://travis-ci.org/Libensemble/libensemble .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master :target: https://coveralls.io/github/Libensemble/libensemble?branch=master -.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 +.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000 :target: https://libensemble.readthedocs.org/en/latest/ :alt: Documentation Status @@ -54,12 +54,113 @@ are portable, resilient and flexible; it also enables automatic detection of the nodes and cores in a system and can split up jobs automatically if resource data isn't supplied. -A visual overview is given in the libEnsemble poster_. +Overview +-------- +libEnsemble is a Python library to coordinate the concurrent evaluation of +dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to +various workers. A libEnsemble worker is the smallest indivisible unit to +perform some calculation. The work performed by libEnsemble is governed by +three routines: + +* :ref:`gen_f`: Generates inputs to ``sim_f``. +* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. +* :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. + +Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts +be found in the ``examples/`` directory. To enable portability, a +:doc:`job_controller` +interface is supplied for users to launch and monitor jobs in their +user-provided ``sim_f`` and ``gen_f`` routines. + +The default ``alloc_f`` tells each available worker to call ``sim_f`` with the +highest priority unit of work from ``gen_f``. If a worker is idle and there is +no ``gen_f`` output to give, the worker is told to call ``gen_f``. + + +Example Use Cases +~~~~~~~~~~~~~~~~~ +Below are some expected libEnsemble use cases that we support (or are working +to support) and plan to have examples of: + +* A user is looking to optimize a simulation calculation. The simulation may + already be using parallel resources, but not a large fraction of some + computer. libEnsemble can coordinate the concurrent evaluation of the + simulation ``sim_f`` at various parameter values and ``gen_f`` would return + candidate parameter values (possibly after each ``sim_f`` output). + +* A user has a ``gen_f`` that produces different meshes to be used within a + ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or + produce a new mesh. libEnsemble can ensure that the calculated meshes can be + used by multiple simulations without requiring movement of data. + +* A user is attempting to sample a simulation ``sim_f`` at some parameter + values, many of which will cause the simulation to fail. libEnsemble can stop + unresponsive evaluations, and recover computational resources for future + evaluations. ``gen_f`` can possibly update the sampling after discovering + regions where evaluations of ``sim_f`` fail. + +* A user has a simulation ``sim_f`` that requires calculating multiple + expensive quantities, some of which depend on other quantities. ``sim_f`` can + observe intermediate quantities in order to stop related calculations and + preempt future calculations associated with poor parameter values. + +* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity + evaluations requiring more computational resources, and a + ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and + at what fidelity level. libEnsemble can coordinate these evaluations without + requiring the user know parallel programming. + +* A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, + sensitivity analysis is desired at each identified optimum. libEnsemble can + use the points from the APOSMM ``gen_f`` to identify optima; and after a + point is ruled to be an optimum, a different ``gen_f`` can produce a + collection of parameters necessary for sensitivity analysis of ``sim_f``. + +Naturally, combinations of these use cases are supported as well. An example of +such a combination is using libEnsemble to solve an optimization problem that +relies on simulations that fail frequently. + + +Resources +~~~~~~~~~~~~~~~~~~ + +**Support:** + +- Email questions to libensemble@lists.mcs.anl.gov +- Communicate (and establish a private channel, if desired) at https://libensemble.slack.com +- Join the libEnsemble mailing list for updates about new releases at https://lists.mcs.anl.gov/mailman/listinfo/libensemble + +**Documentation:** + +- Provided by ReadtheDocs at http://libensemble.readthedocs.org/ + +**Citation:** + +- Please use the following to cite libEnsemble in a publication: + +.. code-block:: bibtex + + @techreport{libEnsemble, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, + title = {{libEnsemble} Users Manual}, + institution = {Argonne National Laboratory}, + number = {Revision 0.5.2}, + year = {2019}, + url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + } -.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 + +**Other:** + +- A visual overview of libEnsemble is given in this poster_. + + +Quickstart +---------- Dependencies ------------- +~~~~~~~~~~~~ Required dependencies: @@ -97,7 +198,7 @@ should produce a file nlopt.py if Python is found on the system. NLopt may also require SWIG_ to be installed on certain systems. Installation ------------- +~~~~~~~~~~~~ Use pip to install libEnsemble and its dependencies:: @@ -117,7 +218,7 @@ If necessary, you may install all optional dependencies (listed above) at once w A `tarball `_ of the most recent release is also available. Testing ---------- +~~~~~~~ The provided test suite includes both unit and regression tests and is run regularly on: @@ -160,7 +261,7 @@ available online at ``test_balsam_hworld.py``. Basic Usage ------------ +~~~~~~~~~~~ The examples directory contains example libEnsemble calling scripts, sim functions, gen functions, alloc functions and job submission scripts. @@ -187,52 +288,17 @@ See the `user-guide `_ for more information. -Documentation -------------- - -* http://libensemble.readthedocs.org/ - -Citing libEnsemble ------------------- -Please use the following to cite libEnsemble in a publication: - -.. code-block:: bibtex - - @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and - David Bindel and John-Luke Navarro}, - title = {{libEnsemble} Users Manual}, - institution = {Argonne National Laboratory}, - number = {Revision 0.5.2}, - year = {2019}, - url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} - } - -Support -------- - -Join the libEnsemble mailing list at: - -* https://lists.mcs.anl.gov/mailman/listinfo/libensemble - -or email questions to: - -* libensemble@lists.mcs.anl.gov - -or communicate (and establish a private channel, if desired) at: - -* https://libensemble.slack.com - -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _Python: http://www.python.org -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _SciPy: http://www.scipy.org -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _petsc4py: https://bitbucket.org/petsc/petsc4py .. _Balsam: https://www.alcf.anl.gov/balsam +.. _NumPy: http://www.numpy.org +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _Python: http://www.python.org .. _SWIG: http://swig.org/ +.. _SciPy: http://www.scipy.org .. _mock: https://pypi.org/project/mock -.. _pytest: https://pypi.org/project/pytest/ +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 .. _pytest-cov: https://pypi.org/project/pytest-cov/ .. _pytest-timeout: https://pypi.org/project/pytest-timeout/ +.. _pytest: https://pypi.org/project/pytest/ diff --git a/docs/appendix.rst b/docs/appendix.rst new file mode 100644 index 000000000..a1561e2c4 --- /dev/null +++ b/docs/appendix.rst @@ -0,0 +1,11 @@ +Appendices +========== + +.. toctree:: + tutorials/tutorials + FAQ + release_notes + examples/examples_index + +Bibliography +------------ diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 4ad76d96a..21d4752d2 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -9,18 +9,14 @@ to main ``libE()`` routine:: alloc_specs: [dict, optional] : - Required keys : - 'alloc_f' [func] : Default: give_sim_work_first - - Optional keys : - + 'in' [list of strings] : + Default: None 'out' [list of tuples] : Default: [('allocated',bool)] - - 'batch_mode' [bool] : - Default: True + 'user' [dict] + Default: {'batch_mode': True} .. note:: diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index 71c2398db..258f96ae2 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -14,7 +14,7 @@ This section outlines the data structures used by libEnsemble. check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) .. toctree:: - :maxdepth: 2 + :maxdepth: 3 :caption: libEnsemble Data Structures: sim_specs diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index b6518627d..c3250e3ae 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -7,8 +7,6 @@ to main ``libE()`` routine:: gen_specs: [dict]: - Required keys : - 'gen_f' [func] : generates inputs to sim_f 'in' [list] : diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 6fea28898..9f2271f66 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -1,11 +1,7 @@ -*********************************** -Internal libEnsemble data structure -*********************************** - -.. _datastruct-history-array: - history array ============= +.. _datastruct-history-array: + :: H: numpy structured array diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index b2076ad2f..ae7a81885 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -1,6 +1,3 @@ -********************* -Input data structures -********************* We first describe the dictionaries given to libEnsemble to specify the inputs/outputs of the ensemble of calculations to be performed. diff --git a/docs/dev_info.rst b/docs/dev_info.rst new file mode 100644 index 000000000..c4f8c3cc7 --- /dev/null +++ b/docs/dev_info.rst @@ -0,0 +1,10 @@ +Advanced libEnsemble +==================== + + +.. toctree:: + + contributing + dev_guide/dev_API/developer_API.rst + dev_guide/release_management/release_index.rst + diff --git a/docs/index.rst b/docs/index.rst index f0d169a0e..56786d7a7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,8 +31,7 @@ :maxdepth: 2 :caption: User Guide: - user_guide - libE_module + programming_libE data_structures/data_structures user_funcs job_controller/jc_index diff --git a/docs/latex_index.rst b/docs/latex_index.rst index f1938e38a..72a4bbb5f 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -6,23 +6,8 @@ :maxdepth: 3 Introduction - user_guide - tutorials/local_sine_tutorial - platforms/bebop - platforms/theta - contributing - FAQ - libE_module - data_structures/data_structures - user_funcs - job_controller/jc_index - logging - dev_guide/release_management/release_index.rst - dev_guide/dev_API/developer_API.rst - release_notes + programming_libE + platforms/platforms + dev_info + appendix -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` diff --git a/docs/libE_module.rst b/docs/libE_module.rst deleted file mode 100644 index 029004981..000000000 --- a/docs/libE_module.rst +++ /dev/null @@ -1,5 +0,0 @@ -libE Module -=========== -.. automodule:: libE - :members: - :no-undoc-members: diff --git a/docs/logging.rst b/docs/logging.rst index c0a244e10..6301600e8 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -24,9 +24,6 @@ This boundary can be adjusted as follows:: stderr displaying can be effectively disabled by setting the stderr level to CRITICAL. -Logging API ------------ - .. automodule:: libE_logger :members: :no-undoc-members: diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst new file mode 100644 index 000000000..ad8be08f9 --- /dev/null +++ b/docs/programming_libE.rst @@ -0,0 +1,36 @@ +Programming with libEnsemble +============================ + +.. automodule:: libE + :members: + :no-undoc-members: + +**libEnsemble Output** + +The history array :ref:`H` and +:ref:`persis_info` dictionary are returned to the user +by libEnsemble. If libEnsemble aborts on an exception, these structures are +dumped to the respective files, + +* ``libE_history_at_abort_.npy`` +* ``libE_history_at_abort_.pickle`` + +where ``sim_count`` is the number of points evaluated. + +Other libEnsemble files produced by default are: + +* ``libE_stats.txt``: This contains a one-line summary of all user + calculations. Each calculation summary is sent by workers to the manager and + printed as the run progresses. + +* ``ensemble.log``: This is the logging output from libEnsemble. The default + logging is at INFO level. To gain additional diagnostics logging level can be + set to DEBUG. If this file is not removed, multiple runs will append output. + Messages at or above level MANAGER_WARNING are also copied to stderr to alert + the user promptly. For more info, see :doc:`Logging`. + +.. toctree:: + data_structures/data_structures + user_funcs + job_controller/jc_index + logging diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index d1f627131..dbacbe475 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -1,10 +1,42 @@ -Sim, Gen, and Alloc functions API -================================= +User function API +----------------- -.. _api_sim_f: +**The libEnsemble History Array** + +libEnsemble uses a NumPy structured array :ref:`H` to +store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, +``gen_f`` and ``sim_f`` are expected to return output in NumPy structured +arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` +must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output +from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will +include the following fields: + +* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an + associated ``sim_id``. The generator can assign this, but users must be + careful to ensure points are added in order. For example, ``if alloc_f`` + allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` + should ensure that both don’t generate points with the same ``sim_id``. + +* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble + worker to be evaluated yet? + +* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` + output given to a worker? + +* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. + +* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` + +* ``gen_time`` [float]: At what time (since the epoch) was this entry (or + collection of entries) put into ``H`` by the manager + +* ``returned`` [bool]: Has this worker completed the evaluation of this unit of + work? sim_f API ---------- +~~~~~~~~~ +.. _api_sim_f: + The sim_f function will be called by libEnsemble with the following API:: @@ -41,10 +73,10 @@ Returns: .. literalinclude:: ../libensemble/message_numbers.py :end-before: last_message_number_rst_tag +gen_f API +~~~~~~~~~ .. _api_gen_f: -gen_f API ---------- The gen_f calculations will be called by libEnsemble with the following API:: @@ -81,10 +113,10 @@ Returns: .. literalinclude:: ../libensemble/message_numbers.py :end-before: last_message_number_rst_tag +alloc_f API +~~~~~~~~~~~ .. _api_alloc_f: -alloc_f API ------------ The alloc_f calculations will be called by libEnsemble with the following API:: diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 96498a735..e437be618 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -205,7 +205,7 @@ circumstances where libEnsemble should stop execution in :ref:`exit_criteria` function call. +Now we're ready to write our libEnsemble :doc:`libE<../programming_libE>` function call. This :ref:`H` is the final version of the History array. 'flag' should be zero if no errors occur. diff --git a/docs/tutorials/tutorials.rst b/docs/tutorials/tutorials.rst new file mode 100644 index 000000000..c1ab22b68 --- /dev/null +++ b/docs/tutorials/tutorials.rst @@ -0,0 +1,8 @@ + + +Tutorials +--------- + +.. toctree:: + + local_sine_tutorial diff --git a/docs/user_funcs.rst b/docs/user_funcs.rst index f912ed73b..45533e3c8 100644 --- a/docs/user_funcs.rst +++ b/docs/user_funcs.rst @@ -1,14 +1,9 @@ -User Functions -============== - libEnsemble requires functions for generation, simulation and allocation. While libEnsemble provides a default allocation function, the sim and gen functions must be provided. The required API and examples are given here. .. toctree:: - :maxdepth: 2 :caption: libEnsemble User Functions: sim_gen_alloc_funcs - examples/examples_index diff --git a/docs/user_guide.rst b/docs/user_guide.rst deleted file mode 100644 index 960fc7864..000000000 --- a/docs/user_guide.rst +++ /dev/null @@ -1,126 +0,0 @@ -Introduction -============ - -libEnsemble overview --------------------- -libEnsemble is a Python library to coordinate the concurrent evaluation of -dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to -various workers. A libEnsemble worker is the smallest indivisible unit to -perform some calculation. The work performed by libEnsemble is governed by -three routines: - -* :ref:`gen_f`: Generates inputs to ``sim_f``. -* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. -* :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. - -Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts -be found in the ``examples/`` directory. To enable portability, a -:doc:`job_controller` -interface is supplied for users to launch and monitor jobs in their -user-provided ``sim_f`` and ``gen_f`` routines. - -The default ``alloc_f`` tells each available worker to call ``sim_f`` with the -highest priority unit of work from ``gen_f``. If a worker is idle and there is -no ``gen_f`` output to give, the worker is told to call ``gen_f``. - -Expected use cases ------------------- - -Below are some expected libEnsemble use cases that we support (or are working -to support) and plan to have examples of: - -* A user is looking to optimize a simulation calculation. The simulation may - already be using parallel resources, but not a large fraction of some - computer. libEnsemble can coordinate the concurrent evaluation of the - simulation ``sim_f`` at various parameter values and ``gen_f`` would return - candidate parameter values (possibly after each ``sim_f`` output). - -* A user has a ``gen_f`` that produces different meshes to be used within a - ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or - produce a new mesh. libEnsemble can ensure that the calculated meshes can be - used by multiple simulations without requiring movement of data. - -* A user is attempting to sample a simulation ``sim_f`` at some parameter - values, many of which will cause the simulation to fail. libEnsemble can stop - unresponsive evaluations, and recover computational resources for future - evaluations. ``gen_f`` can possibly update the sampling after discovering - regions where evaluations of ``sim_f`` fail. - -* A user has a simulation ``sim_f`` that requires calculating multiple - expensive quantities, some of which depend on other quantities. ``sim_f`` can - observe intermediate quantities in order to stop related calculations and - preempt future calculations associated with poor parameter values. - -* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity - evaluations requiring more computational resources, and a - ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and - at what fidelity level. libEnsemble can coordinate these evaluations without - requiring the user know parallel programming. - -* A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, - sensitivity analysis is desired at each identified optimum. libEnsemble can - use the points from the APOSMM ``gen_f`` to identify optima; and after a - point is ruled to be an optimum, a different ``gen_f`` can produce a - collection of parameters necessary for sensitivity analysis of ``sim_f``. - -Naturally, combinations of these use cases are supported as well. An example of -such a combination is using libEnsemble to solve an optimization problem that -relies on simulations that fail frequently. - -The libEnsemble History Array ------------------------------ - -libEnsemble uses a NumPy structured array :ref:`H` to -store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, -``gen_f`` and ``sim_f`` are expected to return output in NumPy structured -arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` -must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output -from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will -include the following fields: - -* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an - associated ``sim_id``. The generator can assign this, but users must be - careful to ensure points are added in order. For example, ``if alloc_f`` - allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` - should ensure that both don’t generate points with the same ``sim_id``. - -* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble - worker to be evaluated yet? - -* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` - output given to a worker? - -* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. - -* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` - -* ``gen_time`` [float]: At what time (since the epoch) was this entry (or - collection of entries) put into ``H`` by the manager - -* ``returned`` [bool]: Has this worker completed the evaluation of this unit of - work? - -libEnsemble Output ------------------- - -The history array :ref:`H` and -:ref:`persis_info` dictionary are returned to the user -by libEnsemble. If libEnsemble aborts on an exception, these structures are -dumped to the respective files, - -* ``libE_history_at_abort_.npy`` -* ``libE_history_at_abort_.pickle`` - -where ``sim_count`` is the number of points evaluated. - -Other libEnsemble files produced by default are: - -* ``libE_stats.txt``: This contains a one-line summary of all user - calculations. Each calculation summary is sent by workers to the manager and - printed as the run progresses. - -* ``ensemble.log``: This is the logging output from libEnsemble. The default - logging is at INFO level. To gain additional diagnostics logging level can be - set to DEBUG. If this file is not removed, multiple runs will append output. - Messages at or above level MANAGER_WARNING are also copied to stderr to alert - the user promptly. For more info, see :doc:`Logging`. diff --git a/docs/welcome.rst b/docs/welcome.rst index 4dad228a2..7decf8888 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -27,5 +27,5 @@ libEnsemble is a library to coordinate the concurrent evaluation of dynamic ense * New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`User Guide`. +* Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. diff --git a/libensemble/libE.py b/libensemble/libE.py index 089c87e07..83c55783c 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -1,7 +1,13 @@ """ -Main libEnsemble routine -============================================ +This is the outer libEnsemble routine. +We dispatch to different types of worker teams depending on +the contents of libE_specs. If 'comm' is a field, we use MPI; +if 'nthreads' is a field, we use threads; if 'nprocesses' is a +field, we use multiprocessing. + +If an exception is encountered by the manager or workers, the +history array is dumped to file and MPI abort is called. """ __all__ = ['libE'] @@ -35,15 +41,7 @@ def libE(sim_specs, gen_specs, exit_criteria, 'user': {'batch_mode': True}}, libE_specs={}, H0=[]): - """This is the outer libEnsemble routine. - - We dispatch to different types of worker teams depending on - the contents of libE_specs. If 'comm' is a field, we use MPI; - if 'nthreads' is a field, we use threads; if 'nprocesses' is a - field, we use multiprocessing. - - If an exception is encountered by the manager or workers, the - history array is dumped to file and MPI abort is called. + """ Parameters ---------- diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 054a60224..1d64a2eee 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -66,9 +66,9 @@ alloc_specs = {'alloc_f': alloc_f, # Allocation function 'out': [('allocated', bool)], # Output fields (included in History) - 'user': {'stop_on_NaNs': True, # Should alloc_f preempt evals - 'batch_mode': True, - 'stop_partial_fvec_eval': True} # Should alloc_f preempt evals + 'user': {'stop_on_NaNs': True, # Should alloc preempt evals + 'batch_mode': True, # Wait until all sim evals are done + 'stop_partial_fvec_eval': True} # Should alloc preempt evals } # end_alloc_specs_rst_tag From a074ae742c4fa1cf04fae31031b59b3f7e085d29 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 15:50:32 -0600 Subject: [PATCH 450/644] Whitespace management --- README.rst | 6 +----- docs/data_structures/alloc_specs.rst | 1 - docs/data_structures/data_structures.rst | 1 - docs/dev_info.rst | 2 -- docs/latex_index.rst | 1 - docs/sim_gen_alloc_funcs.rst | 3 --- docs/tutorials/tutorials.rst | 2 -- 7 files changed, 1 insertion(+), 15 deletions(-) diff --git a/README.rst b/README.rst index b8d3c1e79..686f6dd1a 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,6 @@ The default ``alloc_f`` tells each available worker to call ``sim_f`` with the highest priority unit of work from ``gen_f``. If a worker is idle and there is no ``gen_f`` output to give, the worker is told to call ``gen_f``. - Example Use Cases ~~~~~~~~~~~~~~~~~ Below are some expected libEnsemble use cases that we support (or are working @@ -120,13 +119,12 @@ Naturally, combinations of these use cases are supported as well. An example of such a combination is using libEnsemble to solve an optimization problem that relies on simulations that fail frequently. - Resources ~~~~~~~~~~~~~~~~~~ **Support:** -- Email questions to libensemble@lists.mcs.anl.gov +- Email questions to libensemble@lists.mcs.anl.gov - Communicate (and establish a private channel, if desired) at https://libensemble.slack.com - Join the libEnsemble mailing list for updates about new releases at https://lists.mcs.anl.gov/mailman/listinfo/libensemble @@ -150,12 +148,10 @@ Resources url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } - **Other:** - A visual overview of libEnsemble is given in this poster_. - Quickstart ---------- diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 21d4752d2..61cfd54eb 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -1,4 +1,3 @@ - .. _datastruct-alloc-specs: alloc_specs diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index 258f96ae2..dbbbc81ac 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -24,7 +24,6 @@ This section outlines the data structures used by libEnsemble. persis_info exit_criteria - history_array worker_array work_dict diff --git a/docs/dev_info.rst b/docs/dev_info.rst index c4f8c3cc7..5dac3013c 100644 --- a/docs/dev_info.rst +++ b/docs/dev_info.rst @@ -1,10 +1,8 @@ Advanced libEnsemble ==================== - .. toctree:: contributing dev_guide/dev_API/developer_API.rst dev_guide/release_management/release_index.rst - diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 72a4bbb5f..baaccf059 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -10,4 +10,3 @@ platforms/platforms dev_info appendix - diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index dbacbe475..fe2d4b96a 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -37,7 +37,6 @@ sim_f API ~~~~~~~~~ .. _api_sim_f: - The sim_f function will be called by libEnsemble with the following API:: out = sim_f(H[sim_specs['in']][sim_ids_from_allocf], persis_info, sim_specs, libE_info) @@ -77,7 +76,6 @@ gen_f API ~~~~~~~~~ .. _api_gen_f: - The gen_f calculations will be called by libEnsemble with the following API:: out = gen_f(H[gen_specs['in']][sim_ids_from_allocf], persis_info, gen_specs, libE_info) @@ -117,7 +115,6 @@ alloc_f API ~~~~~~~~~~~ .. _api_alloc_f: - The alloc_f calculations will be called by libEnsemble with the following API:: Work, persis_info = alloc_f(W, H, sim_specs, gen_specs, alloc_specs, persis_info) diff --git a/docs/tutorials/tutorials.rst b/docs/tutorials/tutorials.rst index c1ab22b68..4d212ea44 100644 --- a/docs/tutorials/tutorials.rst +++ b/docs/tutorials/tutorials.rst @@ -1,5 +1,3 @@ - - Tutorials --------- From f36ee902fc40f282acac2dcc4d54b3d6e965bb6d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 16:43:01 -0600 Subject: [PATCH 451/644] Trying to format the HTML similarly --- docs/dev_info.rst | 4 ++-- docs/index.rst | 32 +++++--------------------------- 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/docs/dev_info.rst b/docs/dev_info.rst index 5dac3013c..4c4fdab68 100644 --- a/docs/dev_info.rst +++ b/docs/dev_info.rst @@ -1,5 +1,5 @@ -Advanced libEnsemble -==================== +Developer's Guide +================= .. toctree:: diff --git a/docs/index.rst b/docs/index.rst index 56786d7a7..2d7035654 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,39 +10,17 @@ :caption: Getting Started: Introduction - contributing - release_notes - FAQ - -.. toctree:: - :maxdepth: 1 - :caption: Tutorials: - - tutorials/local_sine_tutorial - -.. toctree:: - :maxdepth: 1 - :caption: Running on: + programming_libE - platforms/bebop - platforms/theta .. toctree:: + :caption: Further information: :maxdepth: 2 - :caption: User Guide: - - programming_libE - data_structures/data_structures - user_funcs - job_controller/jc_index - logging -.. toctree:: - :maxdepth: 2 - :caption: Developer Guide: + platforms/platforms + dev_info + tutorials/tutorials - dev_guide/release_management/release_index.rst - dev_guide/dev_API/developer_API.rst Indices and tables ================== From c5fb9d3a3a6a50ce6cae52b8c42387f6d7aaa085 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 7 Nov 2019 17:51:43 -0600 Subject: [PATCH 452/644] Adding FAQ and release notes --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2d7035654..ad20caa98 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,16 +12,16 @@ Introduction programming_libE - .. toctree:: :caption: Further information: :maxdepth: 2 + FAQ + release_notes platforms/platforms dev_info tutorials/tutorials - Indices and tables ================== From 1961cc403501ba7e8b42577f1b259ec0cffcd7a0 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 09:29:01 -0600 Subject: [PATCH 453/644] Using a bibliography --- docs/appendix.rst | 4 +--- docs/bibliography.rst | 5 +++++ docs/conf.py | 2 +- docs/index.rst | 2 +- docs/platforms/platforms.rst | 7 +++++++ docs/references.bib | 12 ++++++++++++ libensemble/gen_funcs/aposmm.py | 3 +-- tex/bibs/masterbib.bib | 15 --------------- 8 files changed, 28 insertions(+), 22 deletions(-) create mode 100644 docs/bibliography.rst create mode 100644 docs/platforms/platforms.rst create mode 100644 docs/references.bib diff --git a/docs/appendix.rst b/docs/appendix.rst index a1561e2c4..b1455d669 100644 --- a/docs/appendix.rst +++ b/docs/appendix.rst @@ -6,6 +6,4 @@ Appendices FAQ release_notes examples/examples_index - -Bibliography ------------- + bibliography diff --git a/docs/bibliography.rst b/docs/bibliography.rst new file mode 100644 index 000000000..8c87684c0 --- /dev/null +++ b/docs/bibliography.rst @@ -0,0 +1,5 @@ +Bibliography +------------ +.. Note that you can't reference a bib file in a different directory + +.. bibliography:: references.bib diff --git a/docs/conf.py b/docs/conf.py index 00aaf27d3..f6a8f9431 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'numpydoc'] #extensions = ['sphinx.ext.autodoc', 'breathe'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } diff --git a/docs/index.rst b/docs/index.rst index ad20caa98..758f7b98e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,7 @@ .. toctree:: :caption: Further information: - :maxdepth: 2 + :maxdepth: 1 FAQ release_notes diff --git a/docs/platforms/platforms.rst b/docs/platforms/platforms.rst new file mode 100644 index 000000000..bb0295a50 --- /dev/null +++ b/docs/platforms/platforms.rst @@ -0,0 +1,7 @@ +Platforms +--------- + +.. toctree:: + + theta + bebop diff --git a/docs/references.bib b/docs/references.bib new file mode 100644 index 000000000..153d16cdf --- /dev/null +++ b/docs/references.bib @@ -0,0 +1,12 @@ +@Article{LW16, + gurl = {https://scholar.google.com/scholar?cluster=16597238545742984308}, + author = {Jeffrey Larson and Stefan M. Wild}, + title = {Asynchronously Parallel Optimization Solver for Finding Multiple Minima}, + journal = {Mathematical Programming Computation}, + year = {2018}, + number = {3}, + volume = {10}, + pages = {303--332}, + anumber = {ANL/MCS-P5575-0316}, + doi = {10.1007/s12532-017-0131-4}, +} diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 84b53c949..e1816a00e 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -1,8 +1,7 @@ """ This module contains methods used our implementation of the Asynchronously Parallel Optimization Solver for finding Multiple Minima (APOSMM) method -described in detail in the paper -`https://doi.org/10.1007/s12532-017-0131-4 `_ +described in detail in :cite:`LW16` """ __all__ = ['aposmm_logic', 'initialize_APOSMM', 'decide_where_to_start_localopt', 'update_history_dist'] diff --git a/tex/bibs/masterbib.bib b/tex/bibs/masterbib.bib index ac2095634..f5f638b52 100644 --- a/tex/bibs/masterbib.bib +++ b/tex/bibs/masterbib.bib @@ -289,13 +289,6 @@ @article{Amundson2006 year = {2006} } -@article{Amundson, - author = {Amundson, James and Lu, Qiming and Stern, Eric}, - journal = {dlr.de}, - title = {Synergia: Driving Massively Parallel Particle Accelerator Simulations with Python}, - url = {http://www.dlr.de/sc/en/Portaldata/15/Resources/dokumente/PyHPC2013/submissions/pyhpc2013\_submission\_12.pdf} -} - @misc{Lusk, author = {Lusk, Ewing L. and Pieper, Steven C. and Butler, Ralph M.}, title = {ADLB4 User's Guide}, @@ -399,14 +392,6 @@ @article{Amundson2009 year = {2009} } -@article{Wang, - author = {Wang, Yusong and Borland, Michael and Soliday, Robert}, - journal = {aps.anl.gov}, - pages = {1--7}, - title = {User's Manual for latest Pelegant}, - url = {http://www.aps.anl.gov/Accelerator\_Systems\_Division/Accelerator\_Operations\_Physics/publish/Pelegant\_manual/Pelegant.pdf} -} - @phdthesis{Wild2009, author = {Wild, S. M.}, number = {January}, From efff0e85ae62092c378d2b414cf351081eea921d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 09:37:34 -0600 Subject: [PATCH 454/644] Adding all appendices to html output --- docs/index.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 758f7b98e..4d02257fa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,13 +14,11 @@ .. toctree:: :caption: Further information: - :maxdepth: 1 + :maxdepth: 2 - FAQ - release_notes platforms/platforms dev_info - tutorials/tutorials + In Depth Indices and tables ================== From a2e5db1d588bf7b0896deed569934f1bbe9ddf7d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 09:50:56 -0600 Subject: [PATCH 455/644] Trying to install on Readthedocs --- .gitignore | 1 + .readthedocs.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 135bf4670..28292b058 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ code/tests/regression_tests/output/ .cache libensemble.egg-info docs/_build +x.log diff --git a/.readthedocs.yml b/.readthedocs.yml index a8f801e04..7ba231848 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,3 +5,4 @@ python: version: 3.6 pip_install: true setup_py_install: true + install: sphinxcontrib.bibtex From 91dac1a788c4575d913664240ea45cbcf82fd214 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 09:56:12 -0600 Subject: [PATCH 456/644] Trying to install in conda environment for readthedocs --- .readthedocs.yml | 1 - conda/environment.yml | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 7ba231848..a8f801e04 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,4 +5,3 @@ python: version: 3.6 pip_install: true setup_py_install: true - install: sphinxcontrib.bibtex diff --git a/conda/environment.yml b/conda/environment.yml index 57e16c744..35956d3b8 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,2 +1,5 @@ dependencies: - python>=3.6 + - pip + - pip: + - sphinxcontrib.bibtex From 0e7299c2100b3b50108c8d710bc2b27fd73fd3e5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Nov 2019 10:08:21 -0600 Subject: [PATCH 457/644] adjusts affinity info, clarifies distribution options and limitations --- docs/platforms/bebop.rst | 13 +++++++++---- docs/platforms/platforms_index.rst | 24 ++++++++++++++---------- docs/platforms/theta.rst | 4 ++-- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 167b02914..3e64c2085 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -39,6 +39,10 @@ Bebop uses Slurm_ for job submission and management. The two commands you'll likely use the most to run jobs are ``srun`` and ``sbatch`` for running interactively and batch, respectively. +libEnsemble node-worker affinity is especially flexible on Bebop. By adjusting +``srun`` runtime options_ users may assign multiple libEnsemble workers to each +allocated node(oversubscription) or assign multiple nodes per worker. + Interactive Runs ^^^^^^^^^^^^^^^^ @@ -48,10 +52,11 @@ You can allocate four Knights Landing nodes for thirty minutes through the follo With your nodes allocated, queue your job to start with five MPI ranks:: - srun -n 5 python calling.py + srun -n 4 python calling.py -``mpirun`` should also work. This launches a worker on every node, on which every -worker can perform stand-alone calculations or launch jobs through the job controller. +``mpirun`` should also work. This line launches libEnsemble with a manager and +**three** workers to one allocated compute node, with three nodes available for +the workers to launch calculations with the job-controller or a job-launch command. .. note:: When performing an MPI libEnsemble run and not oversubscribing, specify one @@ -69,7 +74,6 @@ for thirty minutes:: You will need to re-activate your conda virtual environment and reload your modules! Configuring this routine to occur automatically is recommended. - Batch Runs ^^^^^^^^^^ @@ -118,3 +122,4 @@ See the LCRC Bebop docs here_ for more information about Bebop. .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ .. _Slurm: https://slurm.schedmd.com/ .. _here: https://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs/running-jobs-on-bebop/ +.. _options: https://slurm.schedmd.com/srun.html diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index cd6f58f7f..3716ebf47 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,6 +1,6 @@ libEnsemble has been largely developed, supported, and tested on Linux -distributions and macOS, from laptops to hundreds of compute-nodes. Although +distributions and macOS, from laptops to thousands of compute-nodes. Although libEnsemble and most user functions are cross-platform compatible, there are platform-specific differences for installing and configuring libEnsemble. @@ -8,11 +8,15 @@ Personal Computers ================== Users interested in installing and running libEnsemble on their personal machines -are encouraged to read the Quickstart guide :doc:`here<../quickstart>`. +are encouraged to start by reading the Quickstart guide :doc:`here<../quickstart>`. +We recommend installing libEnsemble and it's dependencies in a virtual environment, +created either through ``conda create``, ``virtualenv``, or ``python -m venv``, +depending on how Python is installed. -High-Powered Systems -==================== + +HPC Systems +=========== libEnsemble's flexible architecture lends it best to two general modes of worker distributions across allocated compute nodes. The first mode we refer @@ -38,15 +42,15 @@ other workers directly on one or more allocated nodes: .. note:: - Certain machines (like Theta and Summit) that do not support child-process - launches also do not support libEnsemble in distributed mode. + Certain machines (like Theta and Summit) that can only submit MPI jobs from + specialized launch nodes do not support libEnsemble in distributed mode. -Due to this factor, Theta and Summit approach centralized mode differently. -On these machines, libEnsemble is run centralized on either a compute-node with -the support of Balsam_ or on a frontend server called a MOM +Due to this factor, libEnsemble on Theta and Summit approaches centralized mode +differently. On these machines, libEnsemble is run centralized on either a +compute-node with the support of Balsam_ or on a frontend server called a MOM (Machine-Oriented Mini-server) node. -Read more about configuring and launching libEnsemble on some HPC machines: +Read more about configuring and launching libEnsemble on some HPC systems: .. toctree:: :maxdepth: 2 diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 06e95a550..aa0286299 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -32,8 +32,8 @@ libEnsemble and mpi4py There should be an indication that the virtual environment is activated. Install mpi4py_ and libEnsemble in this environment, making sure to reference -the pre-installed Intel MPI Compiler. Your prompt should be similar to the -following block: +the pre-installed Intel C Compiler (which should support MPI). Your prompt may +be similar to the following block: .. code-block:: console From 0ec86518ffb2800f18347d54da5b3679471279f1 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 10:09:34 -0600 Subject: [PATCH 458/644] Trying to install in conda environment for readthedocs --- .readthedocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index a8f801e04..b4acdd5c0 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,3 +5,5 @@ python: version: 3.6 pip_install: true setup_py_install: true + install: + - sphinxcontrib.bibtex From 3cc665be91adca7471aa4558a0524fa333fd8484 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 10:19:16 -0600 Subject: [PATCH 459/644] Trying to install in conda environment for readthedocs --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b6439cba8..cb55be1d2 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def run_tests(self): ], extras_require={ - 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py']}, + 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py', 'sphinxcontrib.bibtex']}, classifiers=[ 'Development Status :: 4 - Beta', From e7d92751ddedf8792c9e4b0123e3eb13d3b728ad Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 10:49:42 -0600 Subject: [PATCH 460/644] Stefan comments --- CONTRIBUTING.rst | 4 ++-- docs/FAQ.rst | 4 ++-- docs/appendix.rst | 2 +- docs/conf.py | 5 ++--- docs/examples/examples_index.rst | 8 +++++--- docs/index.rst | 11 ++++++++++- docs/platforms/bebop.rst | 4 ++-- docs/platforms/theta.rst | 4 ++-- docs/sim_gen_alloc_funcs.rst | 2 +- 9 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b68b88a5d..cb40685d2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,5 +1,5 @@ -Contributing -============ +Contributing to libEnsemble +=========================== Contributions may be made via GitHub pull request to: diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 6a9a52aa6..5512df3f9 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -103,8 +103,8 @@ does libEnsemble hang on certain systems when running with MPI?" For more information see: https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe -libEnsemble-theory ------------------- +libEnsemble Help +---------------- **How can I debug specific libEnsemble processes?** diff --git a/docs/appendix.rst b/docs/appendix.rst index b1455d669..a99b081ef 100644 --- a/docs/appendix.rst +++ b/docs/appendix.rst @@ -4,6 +4,6 @@ Appendices .. toctree:: tutorials/tutorials FAQ - release_notes examples/examples_index + release_notes bibliography diff --git a/docs/conf.py b/docs/conf.py index f6a8f9431..575bb506c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,8 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'numpydoc'] -#extensions = ['sphinx.ext.autodoc', 'breathe'] +# extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } #breathe_default_project = "libEnsemble" diff --git a/docs/examples/examples_index.rst b/docs/examples/examples_index.rst index 10b7003fa..b2cc588ab 100644 --- a/docs/examples/examples_index.rst +++ b/docs/examples/examples_index.rst @@ -1,7 +1,8 @@ -Example User Funcs -================== +Example User Functions and Calling Scripts +========================================== -Example gen, sim and alloc functions for libEnsemble. +Example generation, simulation, and alloction functions for libEnsemble, as +well as example calling scripts. .. toctree:: :maxdepth: 2 @@ -10,3 +11,4 @@ Example gen, sim and alloc functions for libEnsemble. gen_funcs sim_funcs alloc_funcs + calling_scripts diff --git a/docs/index.rst b/docs/index.rst index 4d02257fa..6d2860a58 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,7 +18,16 @@ platforms/platforms dev_info - In Depth + +.. toctree:: + :caption: In Depth: + :maxdepth: 2 + + tutorials/tutorials + FAQ + examples/examples_index + bibliography + release_notes Indices and tables ================== diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 2a42413a7..31cfe5d6b 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -5,8 +5,8 @@ Bebop Bebop_ is the newest addition to the computational power of LCRC at Argonne National Laboratory, featuring both Intel Broadwell and Knights Landing nodes. -Before Getting Started ----------------------- +Prerequisites +------------- An Argonne LCRC_ account is required to access Bebop. Interested users will need to apply for and be granted an account before continuing. To submit jobs to Bebop, diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index d266a347f..ee6b86b23 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -5,8 +5,8 @@ Theta Theta_ is a 11.69 petaflops system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. -Before Getting Started ----------------------- +Prerequisites +------------- An Argonne ALCF account is required to access Theta. Interested users will need to apply for and be granted an account before continuing. To submit jobs to Theta, diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index fe2d4b96a..cdabafa7e 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -1,4 +1,4 @@ -User function API +User Function API ----------------- **The libEnsemble History Array** From 8558f47307b6007bcc00de9d1ddf2d79cd30648f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 10:51:11 -0600 Subject: [PATCH 461/644] Stefan comments --- docs/examples/calling_scripts.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/examples/calling_scripts.rst diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst new file mode 100644 index 000000000..9a975ecb4 --- /dev/null +++ b/docs/examples/calling_scripts.rst @@ -0,0 +1,4 @@ +Calling Scripts +=============== + +Below are example calling scripts From 1776f628e8da5142af922aa7cbc20f94e14e7fd3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 10:55:38 -0600 Subject: [PATCH 462/644] Trying requirements --- .readthedocs.yml | 4 ++-- README.rst | 4 ++-- docs/conf.py | 4 ++-- docs/requirements.txt | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index b4acdd5c0..6c550b1b7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,5 +5,5 @@ python: version: 3.6 pip_install: true setup_py_install: true - install: - - sphinxcontrib.bibtex + install: + - requirements: docs/requirements.txt diff --git a/README.rst b/README.rst index 686f6dd1a..dc00bbfcd 100644 --- a/README.rst +++ b/README.rst @@ -152,8 +152,8 @@ Resources - A visual overview of libEnsemble is given in this poster_. -Quickstart ----------- +Quickstart Guide +---------------- Dependencies ~~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index 575bb506c..d4574a772 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,8 +58,8 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -# extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } #breathe_default_project = "libEnsemble" diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..c3e8a7601 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +sphinxcontrib.bibtex From ff3900017f301a358c749a7df7507b7d6114b553 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 11:17:10 -0600 Subject: [PATCH 463/644] Reverting so everything builds --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d4574a772..575bb506c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,8 +58,8 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] -# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +# extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } #breathe_default_project = "libEnsemble" From 4d837ef8d71bfa510c4769334d6e7058c255e573 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 13:47:22 -0600 Subject: [PATCH 464/644] Trying to do this on develop in case that is what is looked at --- .readthedocs.yml | 10 ++++++++++ docs/requirements.txt | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index a8f801e04..c69fa2936 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,17 @@ conda: file: conda/environment.yml + environment: environment.yml + +build: + image: latest python: version: 3.6 pip_install: true setup_py_install: true + install: + - method: pip + - sphinxcontrib.bibtex + - sphinxcontrib-bibtex + - requirements: docs/requirements.txt + system_packages: true diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..a8e61ba18 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinxcontrib.bibtex +sphinxcontrib-bibtex From ed96185d6b469e1e92f439899a7722f33cd26d2e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 13:53:35 -0600 Subject: [PATCH 465/644] Trying to do this on develop in case that is what is looked at --- conda/environment.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda/environment.yml b/conda/environment.yml index 57e16c744..8d2d32006 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,2 +1,6 @@ dependencies: - python>=3.6 + - pip + - pip: + - sphinxcontrib.bibtex + - sphinxcontrib-bibtex From 96554822547e0e28b32b9d4510bcddcc4954956f Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Nov 2019 14:39:30 -0600 Subject: [PATCH 466/644] rework theta python, rework launch-modes, combine diagrams, other fixes --- docs/images/centralized_MOM_ThS.png | Bin 0 -> 42582 bytes docs/images/centralized_MOM_ThS.xml | 109 +++++++++++++ docs/images/combined_ThS.png | Bin 0 -> 118488 bytes docs/images/combined_ThS.xml | 240 ++++++++++++++++++++++++++++ docs/platforms/theta.rst | 87 +++++----- 5 files changed, 397 insertions(+), 39 deletions(-) create mode 100644 docs/images/centralized_MOM_ThS.png create mode 100644 docs/images/centralized_MOM_ThS.xml create mode 100644 docs/images/combined_ThS.png create mode 100644 docs/images/combined_ThS.xml diff --git a/docs/images/centralized_MOM_ThS.png b/docs/images/centralized_MOM_ThS.png new file mode 100644 index 0000000000000000000000000000000000000000..a3489071e0681e6f5df493d7abb19e64ba9ad914 GIT binary patch literal 42582 zcmb?@byQW~);8TBQW8=sjdUs;;Lv^Ojzia>ySo$-5D*j&EuaE|NJ%QFfRrF8EmG25 zg5TQe@4olmcYI@fe|%#I!ajSiz1CcFKJ%HeNJ3~SlM>MrVPIg8s;VgHVqjoVfiD^Y zJaFYwE!8Fl1_6<;qOmV3(81Zw9)nF-{?AWrLIO@6KE7yZ;?a(<9i~ z%?-&Wq$J8G0EXh!MA|yL`+E2|u?fS$@2c*;_HN*Ra2b5p)&t*+!Iz)_pP=ZUJG}kD z@Zth|0$kv_vc0Vi*)=mS=9g^D?2NDR~KCm zcNJj~6%|8Gt3UVn+k5*sd$|9(O;CtWh)?*>2fjg`_J6M0+50=&+W)!g2^Qt^_h#sd zup$tioiozW8;Jt5aQ-VCc~?mdZxlk+Rz+3B-Oo+#rjm-op8?#FsK3J^z)&~A)S*Gd z4SnrwC4D@U5CVb@enHxA$M_KRZTYy4Q*W~T?ZE-KQVt} zaRnVMQDs9TH5*%9O*c)rvxu>iwuGpyoTQ?=n@g~yy^^q#3)0&@(AyEItcnaUQgU`M zbvE(z6BHHkk`Pc)Gl2VfxH_52>#FL@%LyaYD3*-CrQc7OtaV2H0>qyO>FHuzRd(^6#3uwZmg~(WalIb22(VI>o~}1YHJ4~B~=t1#nhd>+z^U}3PvbR4{v{4Z6E(Y zR~r>A2YpdpKQ|>g@Pw+5C_>Cj(8(BS>*k`O0GCh)7lZ7@z;_)(V{x$O`l32uH7X{4 zMkavKG=LKtD*p)>%i)#>v^x z*geQtT}{Of++!@EryJlI;I9y*;^^!l>7uFVq$J>n(gZKCafGYb7zWt-=qdzzx)=+> zwFOLEoaEdQ5+3>jhAJL{N`{JXCjkuwCtr0FZxLS?M4+j%yQ+|mI6~7Dn4t|5XYe^^>`vw5}gKN7Yf|PXiO;GMy!lK%K!QhgP zhrg({uadf*x1qYciMyh|ww;E7j+2|JvAC8AbD#+eNL_lA{%U{*i!OO@i zK*7}180C%%^z{_-lk`vz^l`C~7ejf82WfjEMO{?X;99y~-p)$MKt~5PVkWH zmvFE#G_eg*Gx1kZb5%D{cGdFnaQ5)=QdD(PQx8xP&^4A1_R$P<6H|2+Qx_LBRzNxG z!_|!t9zjY7ZB;ixM^i{YR1{EN0h(ZQ;Y#if1{!L3%AyE&34k>YCOWR7 z_PScS;ARbZLpK3sL3y}6N-4+&r77kp;3NkZ)KPO+a0^h;FcKCp5RP3D$eBI&x-fAi$BEsHsg3fSLH*W_&VQ)jYgq$thMAb+gfl!9asRtN2`zT6i znHb8Uf}F(tJq_ixMZFCJob24yox}}woC7=!)Ks;6T^Y}fDeI7 zIs_?d2P&xgz>O3gl{FRplu_PJ@^YF29*Qc4a6=<~F$rB|r$8MsTUSSacOx|u4SgM7 zFjGPGKxGL7Jq=rwqMnJbr-5p)hk>4mjhZ1ee+@4KWnDvCi6AdeWU!ZlN^pR_i@lzM zo1~nVfd+tOxS=CL$W~p!z)8zUz!dIc6D;SV>WDM}ve5w+=Vu@wYKT-fRd=!R7Vy#6 zadbrb!L>A7!d2j2?%wKRfyQ!58iELAXHgvyZC5{W5u^^%%g0m2 zQ(Mf*))m|rUo3myikL(;@Nn_4D(V)@3kcLbL~02L^kd1%!9|L{ za(?)5c}c2yN7mUN_RuNXN!O&vB+F#8U(cCIaInD1#IW4{?91kK+VYWk`_|ss^sG#e z=(BnUQW9@t*UzD_@q=bH+J`yymZ6_RHvOtYSV!AMV|z$N+)KPDSyWce65F;`&O|1N zLdncK9JjF0*x3cULGNu-r45ZOE!)a#YrVDY?HBWtlan*pGYGM}3SYnesSmEq&X@3I?!R>YRL1fkfEcbAv5GYvO4|J1EvVaa&Ibbnc4h(oBOirj+>m@$RMdqO$`RXZ zy&l=7V`9=&`s~^I7I&u04Y*>%HT_egviGn>YR#y@Zo00^2r?5`?3j`fohS>ekP$(PsSeGzFR{?wgvb5AU>F^aXagM`LFb;k zbhCS{$fziL@Dron6GJWea$j1w-quQbwp!2QZ+PZWgPoYiB#DGN$L3(6Wi8^80h{Ji zQ&V@%%*?!8U3p*;7gC$+>o_=XPxh9j&z>|j(GBL^ZZPxe>*KMKFL9Po^PAqtYLF^X zH`l&Kc2}gIds}iF6WszN85-T?n0_O{z|e4WK3Lwrh0){dOG2w=-=w2FOmpUJK8vnL zVyG_vmOQ<$$;rJa9385JeKw?uYty@gM)5)n$pC!V9fobXP#Vgj-KLi>_2*s1JoWS_ z+ky|cw-~OjFjrZ&CO^u}l_IMA*bF=dHOv9h1~d-Pc=d1 zW~PHpx8=l;+maHPXiUM&H83C2;+v)~33NTkJr8~uymlJCM&_)EW=nTSg6mMPs=it6 z4)7y=LD)bTGuTNC#F`Yjm*X0(4Q9Q?KtXHX7(Y>_`P7Ylq;lc=m$6|S9l9vr%p-sB zL>+hn13Zutx!qJ$WJ6)JyBM;7`LiQ4mN~d&gf7D-tG8C;$W_Gt=;VIZEys7R*m!8n zZ8~zE=%bu4#=WT~A0+L~#HQeb%5lk--Q8sL4VOo{HZ^QF(j5r(OkDy~1X~_9u#>>B z=Y&}daW;H{f`YhiZ;*V(L0?zCH`nB|92|JEm>mW?eBLm#E48`$1kLndJYe^`>TByx ztV<{qDrWcfuha4N?JHKHQR41XiK)yo_ac6sAIW7O{yf>UYA(D4h8!2w*BWW@yrWbeCX@q=iXqa|QB z9NQ5Wd(7K(tVhp~hu+U)f-I|mzFm){{kg#WYcA9ci7Z?eWMyTI**%>J*^=&Yodi}F z37cK)%l6XJx;HK+M=+n8OaF5j9YI$4a(&Nlad9y(siL%rB9qMoNaH<_A*F&n&PaBw z-$y>aa~_-gY6f66af`xv+OU(@ovD~r!DABG5x!N}0ks${$>Z~&i=QXQ?H6n9(UZ%& zyWSkX2+>A7qlh!HsKzux~HX9Nx#@;_raHAEp)mdyb>E7w;-ISp3N83ByD_#1ML1R z#-sh5Mp;=oU=mG9L4oJ@i(p_GU{C4_zk`6xi(?8J*4VYRwHU1lpuocUJU4E{3X^Vq zt#?Hn4VP&?GJfL{ahsCKO8>FOS6!QZEk$1EWFyYXYiYeAwQHeyj!r0q%vjW5*UrgM zp?u)^=|SVGS2PK+v4NYli}G|{(0lIf951a@%+*ru>V%rwh&ity3#GX?n3tC)!OveksqGWf zWYyug-NX+=%S7bwNa4j`TD1mK{{qttoP33{oOBS-)+Fw6JS@5$TJ>C`bUT3{PENmt|?heSj=GSK@bq%UG#LL*Y$<73=VCF&|goh+lX$V6CPw{ z-YjlUTv~FNiPKb3vHtl8=Gt%@IsV*i9JNzEI&hn18Np|E zijKHwdbd&I@R7b}@Wro2hmX14#}@&!Vd5)iNtL8Ww1s7bw^w8yLJCqZoRRqQ zheT$tApI)$*5bS#<1sG_`u6QY@65&a}YHnaKw%{MQTAEEO^J!*g;5gjEf^p$Y`QBB!(R1J9ZDf2+ zs+7>d*~-e)$(uz=^R#4c-AA?t93@Q@(p<)_O^){=WyadgAwEp(w%&p~#-=u!>tYZ6 zD=_7?d6PHWN+?Z9&29hnj;DstZ9-9NpPx??-S(J~>REWA`Dlr*_w@W|wl%)5OdGwK zcJJo?-k?ls1dwXz>IfSIf9NjviJGXLgAW-TAp>0bw*$x$;^G7?UA2WHTwPpf+f${n zI%n8{eld)mkAD5yOMW=4{N#e1T68{X=>6-a%R8USI8(AlXNgR>D zh~GQ%#ZqxX`dSUstA0^gnY?3;7sJXK))v)zO3xhdbb;dVXg0!;Bvm(?>x3dp`av+T zpdKId*5J&v`xWT(rAHpLGFKQGHx5pFQke(2cyD7|mGF$*@u{z`zq{3bw%Ic6o&aFO zeHQ!dfr@r}bvu&3NC@?HpC>IXj<~qkXIYuEte@pBUyRth%Cdp63Mt`ax^d}CcEGvd zeiyz1C!+cd2Iz>>RgUT=N&?o`9z`JmI#UdeSU@s{t?N?{B;GJp4wp?x zN{Afn0-Mf?O9busoxub$#&1#S4kR?2Ob4f-Z{9-kBNYpd-_j&gG`38+u-%OS#$K#vdUY@rOJ zJA>u~FbJ}I#B2B^pI20L;$LF=Zdc<$!H$FWAi_aHu7<(m_3-nW*4BX&ur>K;`|4kj zRcy$rwanEn%huqrwXX{`H3Cxhy;K9^-v&2OUnqsIU%TchCl@uowO|Wp&#p-$%*cN- z@QIXz=y9)`nV4VxKwV<;^6rh)8f-z0Q0!w~DfAZtX=oaC{B7rlW&_K6-G>{K04*ag z9|DT7O)q9!{r2-T#S7mFb<swB(ijwv-P+NGygpef>gpd)Pa&V z3JZ{l)}ZCQ04xy%N&a9oc~NL2@=XH(x&xmXwqX9L$%# zTaB7|owHN-L>t|6a&q!`Fef7;2JF(LB`WpaJ+kV(Xez^}nd5tYzpd4(K>KH!{g<9$ zjtlZwQ6ka-1+h*M>+LV%>$82??;rp%Lw|XK3XYbpfYFB1#MWSUHq3_KX>gxTW?*30 z$+~H_J#H1TF<$ZrjQtZR$Gqkvv7qPJ!<4gOM~pK7Rw1{uQ)SU|8fw%5e8*(~LX7Pf zr)$T6z8q&;h0lkzcoUl)yZLs#Rn*bJP3^07TlgNV96v5poo}{8HW4iw?c6cvGlv1Z ztxdECT9JoWKw$rdn@ySDYxAx5L$n|1D`xm?s~85!U$r~-+-b(d#>R$%qP8$RWR&G_ zaZL%2k#h~*7+$tvWka8xN!c0xs_t#44%uobF6e9+WZ@m+3BLYp%QX-G`~Ujj=jB(+ z-FmcLhvQFf{ns;9%Er;Z;VUQtG#oqY}auc3mkyFIST-6mLTwUiREjtuBe9ZHyC%e3yzIPX61iZ*A&s%=)v!9BEY=G&DAq`he%PqFZ$&8 z$-=w9*%|nOBulKc_ma>S{%NODc_ENM_edlr5(9RWcjxSIH@tw^Eta19_z>J?Q9IB1 z>hngYy4b7v7_0O6!Cybyi@s09X}z|yeRS69F424Df2F9##;GIH69auVQ1~?{FhqwH zW_TO_SIDH{PZ^GTN$FMr5%*HZ20Fn!cEAmFZ^a3nbJC@tz!(4v(9WB3R-p9|4@R&?)xI+ESg01^sLqtVy#hhEq zwOX|`w2;S9%VXokVjoWhl*@J=b%xgz%<#`nS|q~oV{6DcW3gwulEGGYVTNwr{{1U- zy_233IAFc0k&$TN6_jU=q7bqF8YWs4Asca66-h4k5F0lRQL<=eEGS|vD-k&u#sRME zJ38P(BTuP5+&X8$L3R&CV$K68G7?Y?p(}_8kgSN%3F!iuHV!!A4IfA+Ii=6~OR^%-ohb-Hso3KI*=x)KXM zVU6lwc_zW&S##QdW`@JR^7{QCNmFR2hSvfoHoL-IwGemdYc^vzq&>_z3vw zp0@bsOm`WeqiU|9I>MOhS17B`I*G?aX4et>`Tl~*n%0#^IPX7#)eQzLGL-#yy6sgK46I^)&>C#4Q0=E;H- z5MDrjXRxzz{}nle(OS<4tBwz9#7{p;>!Y-1CI6in|wU?GeQvP1&>HV7Xauz8512cy=(md13I}ac|VNoGtllYEs&XzQ4K|Qk4H8|X5w}(%N|Jno7r0@ z8Uq6*|4;MmeiArdW$}6opOpZtorK^&t+b1D_Dqfdf}sHauHI<)A1Er2(W55sf)`*Q zyGJ9D_HzG)A(1sUE*-$=Ab|Zh@Q^hY6tgxqxgXK7e6g3~G0uY6-tzea1#x&+8-GiN z4ETL=P8Ohf7{JTn3y=efE{GY`74RE=WPw+UB5uWe(`7j4Q-J-SmyNJ#)XE{EOF$(@ z&l$aXxHw?nA_tx*ihPRsrZ@WAp5#MzIuHJE+CTQGE_Sv1bF*}#yDbCkc+3Mr>}1<3 zx4fo%#2{q9QguoaOC5y0)FQb{2igA4@p=*^q< zVoii?$++$vLbc|QwE0jkYCJee~^2;h2HT2H9wWHne8fF!ms?)o`l54&> z18f1SJeh*-NN`zB3gbQrs;N`2^(PQ5aVm zheCtn5P`vQ0TAp3PUlY6fMwIj{(CVruOU0xo3HsvD zai*aG0hvn73ru2B{q392i~%U{5=6myVfYKrFaN^`4oEJK4_y95?zM6Cb!#~}b7kJU ze<*<4alUcSl~9j>?WTwV_nF2U1mN5EuOAt*-Tv?dU~aqz&KwWfUup%!5 z7Ro#Up>0^y>1g;moCER!OLIo;NY`m5^S^Ka;K!I7u^m>!%v*lccl4KYxE;f4_Ar5) zVnSE^-#va^)Nk88q{E8yoSR$*-rN`)cPtP&Pvs?QEml*@a(wkY4Br*F(sc`f2Lkv+ zP95J|JIp(Jqk&nRUhRimavc`jn1D(^fw$~`8TB_kD7QCr6!Tl7{tysga=*Q1oa}u3 z|BV~`w(H-t=|8Pn^bf{L6>_n;!fZ`urgW{{=d~slk6w3$rV^;$K|o z4*=mUZ1go;kpW)*<=;<6F~uIbTWy-ZV)$d@z?$ZPQ$C*f*>pTR`D!zU-T>7BRLr>}QUIVSbKBXCeE@JQyaWs{ zNonUUZ8!azZ~5OwfA?RX{ikDh29^)BSPnwsNEv{`_@_SGr&sTH`TP2A^9@<=Eu_lq z0zL~U9)my!^PJBw>~qP$FftV+m$R9v={AHa+0HCrn9Fyc0dV>Ao*?VUi>r@e_|IxE zWFRvHk6!wJk6_v=K!Fbbl(^A%m12tlmz;fh5bgN>7RY@?_!HjtN+YKiI3_uUN;HJ5 z$Dw*bK_`}{cbrhD#jrORT_yGPnXK#AQ#D#A@@jS}3?(m-(uU&w1i|#%Sn^OjBxbv0 z=yA<#S0yRmO?l%Hk*kFj0kVK|WBoInqGGr3{*~dWss6Kg%ve<)(AP0RNSRFLN%&M` z;%=zuTy1S_@HX8xJ`{k#w@=F6F@>zNp5A#V?!mOhfJe>_^60h%kJ4c$^885Q=L=lO z(%=@}G!gam^hzaCL#Xx>0R>&GAfEx}+@b;LxQEBsHBG*&=|?0st8cS8k=H@mNR^Oh z{I-yYQk~M@s5OirWIKY_+j}7w!3qj;-@>plyKF$( zhQx$9*!RbY)9Uvt*D2Yhnc4F8HSB)AXNBN3VqOAE{WIQ;KjVRNX~_q>)69$L+}zy2Swy6O zlZ%VZds7{q9T^?LORwqk5XW%+pWl~@N+{9y?)F>JazC=ubC|3h!*k75uvdi!$3dp( zR%l$;sa$i9?UlzX28>=d018{l(+9<`#kwn%uY68V-{n4(dohKJi)b>NmWhQkv-$p1LKYU8`qxS(kk`7 ztJK^Zq62Db1UU(0o{)UE=?I8oSN7_YWP%z%9r%`?%c6+?SCf}cm$w+m<`NX|&;*rQ zSp?57K_eGa5?K|X(RY%2t4rB`BE}`uh1Uh?HNS+ZSW(0yR%_&3I_88iVAw^dR;>I#$G}h_}x- zV!XDh^bI+EKue2c(W5HDNvn@nLh(8rS)9l#2lJC;Hb2nnZON862Rb{idR5&ZdE@$R z`N!DP#~&KXonWO1wtsp7zKZ*rtY%+VRHT+-s(NVp_<_TKKK;!Kih50`1WFMrnjEC6 z+ldlEp6z`yZ-B`h8h^Pm=i>>bB;YHsPw``Fzmo3lO><1OF?F8oVm3mI1FxuxlCqfA$Q^E-xQiA4UUa#DqT~APJ zfAKX(b#zh1;*E3J?dj!ZJU1+cV-hIkSWcjiHmOT`H?37_9qUafB82+!{e$A|rWx|E za@2IQ-|_bRo5ttQN#mz+-h&#Vmzfy@Ec95wvca8kd5CTthuOoVC4Qs`ln7FFGikU+ zp5Kr@X9s0&wME)#(Z@@E;qC`}GAlGf8k@#_$f&4b^ljcn&?oTSQ-gliIv?*lg zPa;5H#Mb7_4f10RwB_NW<&o<^8OwdVeNC=k-@BruxHulqA#JJ!j0mcg3FeA|Jb$#x z(|da*R)g!vy&*Y?SxEYI7_THTU_OtZ3Qs4geA+H4snl`OcYx7n3Nk9vI?~n=<@WXU z{d%-TO1#fcO-=pFwm)~D>-K?vUdE$+jqGQ*sh6@ft8{_+6BhRsfOHzDh{nCOR=;nb zvpN2!;-6S`Fi-c!Ic7Mr)+3{Ka$s^V-_L;W9T z#4CechdG(iciaz1o;&wnCqFflN>&LWk7MY&W|LAjG_mt}KAo_trhJONbw0`u7gw=` zImqtEQ7Y`#K|+O|dc;zQ@;?3}%fxODqj+nx1K$}db*mgP)Ts<~TemV(POw*3Y!Nos zYKF`{Nsa}B*dEP+>5mljv7P>CEC;mz3B5!cm$`&@S8CM`{7ry$WU)Nc$Fl$B5WDT9#97$gjsA>$M zXubvG2A$e<%{P)GTe5@kTb+(BATY?G5agtp{J5xrM zn{;pP`KgtPnirNQD-54}VRB|GVH070Cn&}3;qPQOGX>Qob#y566Co#)C*YNKmU2aE zG3zL|1G7U}yXTc|*DJB2U&?&wnzvhwgA*t4gs=RR=*{$(o=AQDndtKJ=yy;VE|5s2pZPZ8#LAZ9JZI+yeeEz=d)2xS!3S1HXB<0&^ZqzVTM2BMFNZ}#iH^=mec^D^Ue~pEgM^vX6K_hXGobW4`q$GkZ?(e5Y{j&DC%exQR>zPtgo%#n zc3c^ZS{8P7O5wVxo1+9hl~iI9UBomXTkNft@UG;e?~{42LrCxEb%N^N6nVSCukS6z zD=iQB5I?qBn_=-~mty7Jr3wA1KT7zjW|Qjik?uHx)l91@PrLId#OBQm{z?5Xm@uR$ zaY{=cN$>PNr_=f_dHvfHO&xkSrTWg?GJq4lIsOLE{2kH(RwQaSyw!c<#GUDEHy$MU zDerv`ZcvX2FmrKV7--(jo5gA7izZZTwnE1yR?*#ANlGZwFYqvcTfVm3i}@gozH zG}5rqPVGcxb@T&NGS5JcLu!S(XTVq-z+D3PWR^%b$Dht6Rr1WsiD{u+CKZahfxkq? zFTtLCBr!$<@i3wYPA+Bly<3yNSXkVxkDpwueYYG0iaNYFAcOa65LS{Gj1Syx3g}~E zXER9A1#Kpjs&zNmyLnTz>w9&+iwH^WL`5gY{&X(3VGiUqPq&(<34AOKjA))F==GT`2wzedOFIXgp*$rFU&5Zj~)j!`=R@YQhc8%H)Fgh=IH-+9u_1xBPqom;LvEN ztN8ott!+C5**cndV>WEchYnQUjH?v=dcWBsR6U=F3a)u23^@jzdc2#CJX zp8D;OfRW!;UE2)}4OqKz)zn-cWHarEvJ(MrZV${oN(LVlQ6O@q3BH&?PLXKbwLkGD!@UYm=x-{-zZ1SlYsBKrnIHMBDBE`Wc*yL@nxk( z0$b2-??`p(iZP__bZx~XH5ZVP&RK=DTf zP;uQNs8NiE${-CQcmTVuuBvyDJJWCG&j}xp$=FfDv6VOzLt(8hE18?-?q;IX7wfwS_{0wCKA~7*y#7bD8KFGu-z1GnUJDl{gkIq zZ)ibj1aR>IaHFZfJbKd00+TwkSjHypLC6#Yrjs}}ppiU$o0a~D+!whA9 zFiR7LEQ#(XEEXm!iz=B#-F+#=GWb$krT^tUBjBZ}R}FO)Z;fOssaJlT8*%Y@=3-H1 zp?DrXGrKIhG6`xb!dzNLns1+&F6y>gunSI5*r@7-0>2~%;-m7nYnLwM4h+rde$E4sZPu$ChmlKVl$zC9P+y`=S?M}S|Jum@KFfd!>-%qHpOk4Avk z+8jXX7Vq@@;QDeQG6ytk%kr5qKiJhi;SEX(RUkt`C6X7K0apezjO#XHsPl0i+f3QbDd%uX{<=8Hb*=7R{yoTeDXrh>cdOMV-S&k@{}F2 ze%6nfoH)SWRZ~14971=m30r)1UEbJr_iDG^L!>rVRJB}orq}xLRNwRsVUbXm^unm1 zDd5ErQ=v+(RJpa8`5KQo$lq_jBEFCrH1*_+4>6BjGJvv4NxkV;!COG*6IPlbV>aj> zsO9~1{(vC#&CJq+huoRAFGGw+Z55JpOQpyg1R`F+ zzLP{BVd0;uOS`u65vsq6()mk^e&GQ#rv*k=N_R-xYrk!BpY0a9Q0C68qnhbga)!)! zVs{^oqFwSMhT;#doWzRI_C0-l1(^T+9;vRdpbUo65&D5Eu8cX&Gk>-yNb$uziY!xv z0_ZWTcvPbTf}>H<4$z#L25=*J@JJ=$>r!I6t}PSPta;61u@wrNNbCzWR4WRFo6GNx z@)(0X&^dVwZvWBm;#(ok*YF(FYp6LB_0NnCYg@tAUOm5h^O$#qB6CoUZGYK%J16(8 zocbGIuRHD=w-St)zJ=F~<3R9I3OFHStwKU_1PJejdJGEg^6g<$aQ$*y~mD|onk8U@o zciH>m)Up$_rq^@JB^D0I=g=17A=sI_ISX)ay3i%=;bW3>$;QyKC&;YEpay|O#(*PYkN_vTFK9S78a+=rrk@XTa919(tH<3wj`O+E}8Fu?o%gMzK2M;ict{%HO)6I z>>qrk#Ur-R#4c#WSt7*)z6zsni)yXWnPaI$vglP*>Jr57*v|ywz9>ggT7=M3}0$L`xP#tIN*QgCE;l=IWrywg5Z;KD@4b@qRYNLo&8K zYrWI)^E-%VR?FX;)jRolW(so!4l3?}phXXOJ_8W`DC9Z?_!pNHD3DXEpMSgj;3ijg zIVQ(bW@=ej%OiE|V#>CQk4&X3x*N7oHiGX{RXWH<+?AbgdOIGaMKjZ;)`*ovhl(;}N z*-%qH!d1F*yJh}RK#jGpHYHeL=j*I3QV1FX2){Vyei}lbv#f+3vhhD+z<(6OUwn3+IfAC@ABzRB zfCZ6K!ilv_cQ@8p*?=>Zm+6N^u|M`SH#Ar9Ba}FTPlMNb3x7Re$O}?+UeY+_xK%MK zga>SK>WNgZd=+VXt>b-S((R@*ib?=Noy$zFiIE`Rb>7F%h}hHA`=>s+nVmQgZx*nSA$;*{pZUd1N+XdO!^=2!#$ z`njLHhcd@cx>A)J7EPcDsshkKkQ|5=dBe0IxiNpDkR3p5BWBe$jf*~O#|r}~Ts$;( zMf_tF+~eTYhr=5i&=%z=W?s^f?;dsg=AJ8HVaQp&R>3-bPir63L?=2U$+l zC#F%3TfV|Jcto7#ZB+Y?N?+Le}hxR3*#4-VxQCGnYr14PXLhl#cwlbI9ATE zyls@+s+cmY6g4)4rc^bm8nG4z2}x9FrNMyImRoa%-&brbTzxe0sj$ARLBMK5^<&GF zi)X=sJyZjQHcnlLbMp-zomJVU(5eru!R%rFp`c6n=&r87!0nhm#JS!Ox)7@jPv~5C z9k{UBJB2pnG(VbM3zRrGzZ#OBOA3Cvl}}$<;P8$Eus&}q_jeLo8hMA2(alxS7bMgO z(fg~}-}+rwF6~_}z2r?s9gNK1) zwo>tGY$fM&=1P+u&k!*OLqdD#Ygp}OpV9R!W>cEF-40z7y)$m#O!c?te#z3eEl0x? z9LLq0Ki;;|IPnBOXrgZ#wzr?L9khVYo|k3umsR)2OJzh@8q<*vkIu z_)y*Rl$DbVp7?fOx1fpiH+{vHC=;3%n}|HSgNpc}1#>=T*TmE04WUKh#oc9o=b6d# z+HZM|JWMy|&8Y}r!*U0DauM-FFX^MFyr;frbxTo|UYU&p9#FT!o&-3`D}a}$tD5yz zoZhc)inVa(F)e4wD8q~-KEtp zxyOm8z>u!Qz3O`_J8nIRcxX{f^XBg@R3r?kQ=R~>S;dyA5TUOK2;MPMoS)QB1>LMx zg`CyEyV2>HE*%h)@-7xm9lBjt&cMqCx^fpmbkqOAZM%Z(NAn$O&YtOe2r#v$uZ_UA z-UIl2NX6;j{_O01-y&~FQnas@LB^r7f#khwfQR0Is8Y`@?}-3ceGvABG=?)1RfMMPkK<=OYUG>+r~1E*#8}?NLem&Doeg0<|jgLG*mL4cj_Nkq6 ziNU)(_p4tA*w$U^h&D_|c4gJ#ZEe6;5zyrO_Fs1a6!L-S1*&{JP*Ug43H_wr2I85!Kto^grUqpSBX|wn?+=@6 zeel$Hkd}qNl41l<8}N+h3MjpQ%Y{LwctDQV;^P(AN4bNh0vUN+{TI30^+-{>W=cTU zSyF;<a7PyhM2U2CY*JsIO;ne&%p6eb)yYWWbJEv+Z9UmJrN0>5(|(pC_ZWsLOa>?4uY+TQwm_V0hFwzKe#9?(3+)Z+_? zBLajv{3YuTd%o;TpH;^xf*olVYkB*`?y1zLmMXEA3$TLVqy(__xFYV6g}NsvTCK%A zi0{+U&5z5XajE$*3Z6t_0u!H0=G*wF;rwJKY6O^9eC?2MN_nRGgs5oHqFT&H6_6&5 zoyjRxX-MDy&I9<0{+~;SvFl>DnSJ2&^8?-kpIXet$4&}g zIOui!U9stjahN%*6oW^pxmV}Y8To#Cf;iwo%^kH5;w%jU;sK=9)KmT&Q(tO?d4{?8 zkhA$2G8Q1#T?E-O;Jzu8vZHQsTmiINuPzSM(m71zGRKRIDhQ))p7XESuYMW+j;U!^)JE=v1oEapl=Vve8gDk94yp1X$H$hm z^Z?6+((Kn5SD}&>6zdcsK@=N47%N)MDw0}$KeX`lsRSIil`yrX6=gs1DU{hRa;}q+ zvsZo!|Cd*T)uWop>CWaB1hiKy$uo(qu=x{;S3z1c`UYk4Pe679#Ajg;+B%Rx0WFcD zf!%oU_T9eK@>nhAQ*4vM)8b~orstS9zn`&S<6LqyD%`pcm|6>d7MTxPi}Mol9+p5= z{x&r?WJ2aIx?2egi`rjSD{x4}d^4u#~^;?b-&f&gZ}XN z=-nrc7U87QACBbk2$%+{6lQq<6D|=-Q2;0n;@C}T?I>aW<1LYWN6x4#wnF#$k>i7G z%wvH1eJHuf2C_8ekX_rWLe9{V!vNI9rY)O*HpIoh%*=3LJYjr!^{WJ*h-S#LE$?y2Kc>tvDmAwt^|@_Ka+dEL@B<%jNxP<`ZDFIa=uB@Ef8?S0g1~7 z?jOo9-mPZ}<-fYA z*iKjRT#{US&V>MG6s72rsDK6*61G?@WZ_%gmj{N89{7yFp4NmmD(yzH;O*g7Addjd z4G-x{Hwxk};N_mcaWO!S?g8~cS3+a3o!P=r-ve==6^bC0I9y$e03fq@J6KnI?#Gxj zKD5;aEW8hllW91e8YITEV=sgJ3{(LHhx_QC=H)18+WA;DcxVxE{ipRc|l72q)nGEE?Q1c$J00d4~AB#37317`8{ z>Qhr&5!V&@Nf4(MgCZHYK###`*v#~B-6!_jSM#e)Gigw6v0l=G$3TdG+72g~;pYJz zp#23@1jKA*MVkF3n={6@NIcvP>UuOZ~!)PF4mg`1p8sF4mR!qPoyTENdNm$ zMUbnJ7WA3gcs3b(^oPyJ8gmL#znf+N`5o;#~|Ij*`32(!W3*y8f;uO3Bg!0N}+9I2X`lT7%6*y?n!2yz__p5d- zL81=>LkkcPppKyn3wF)bvV~&CpMYWYxEeR{!@ppTnncM4CSo_(1nTQ7gXZA zu|%gg3;X1!268uj(UW|P@jxN5F!*&X0PMws~yG{enoL}3b!Pri@9l2?CRoN4t$XDmlg5`LuDTyXf9_Wal zln>MaiK}|RjVLFz5J&GUxcje9E?{+7dt&2@Uvnrgh$Ci^MQ-&)eRGISzVHr|RNJQJ z1azbp%s}@}p)*Lt5>O#Jjl&7m&kGT2W9kR~Ie@Z3Im!SFTg~SNTZGi&SQvx*o&^|K z1cw*JTaH!aU#zAQ$(t+exzfE4-V5{SBmnND8wCddE(g}}xfzqy_Y;WqC9-Rj8I)Qh zir1}U{^FLc2)t49Q|vvQecso>S5_!=XvD%uNO*XzM; zer&;2DAC|D@U;X2PiG7Q98O2f`MEFRJZVqY_8%BES|<>?Jg96>rr&AaF_uo=;zKyy zbs&W{quM3wMO>Q?Gx?L5c^FG-%e1srpHFZm5e+sq_fIma(I`d`Ez}q`Jdpx_0_phsI`=*! z$c#ab2*#OBwnlX2uC*Lv3DgT6`K`h_77GKT6a2-Y(|i{0>kok62W78ofRUZCsNxcH z>lZxI8DC8eM=N7SVG&g>N6TVjL|RMMQTTzw-AT~VQove$XCu1*lnC*ix%C6dM#ZlY zPyq!EtY9FAi|1J;Rhk(UEo(iO<-ckr7`ijOIIM1#r928n$leiop$8YGyW(xe=`{`> zdHc^>{YDyo|1!%%=&=dJKWoKd74~~SZl;*{IhpkLW|a!2FlBXDf#t10KP&&Z{@I67 z1UR-kf3xz?ky40W0I*Q{b1t7&jla$R)AJ&;>b%pHEgd4DO^EQ%Y0 z13Vg}hkc=JK5UG89C#Xtt3ajl(i~xM=8K2x?RII>8T-j1VFtC$}S-7 zOagZCl4b-N@t%&;kgT-?I)Cf(NC@DY3dt)R=%*zEN+ObgbeQCL{Gqbyf= z5E`x603ZRFK(QeODChtmCkYV)5TpQZ#tm8^aQ#{=T}VTI)N%ng2oz>JJF~7#V~3g( z3N-u*?txkdMa)fB@Vpf9S`gW1Z0{ie^SFzS7Q|F54oCNqK?I@QAlUu*-!6$P7nLHir1uWeLWg&4^mopA?(^I+?t91mg8?Kvd#^Ir zn&0`EbMN@N*{h>RaE5N{qmOAAS(YUXXBdqR*6sL!1)`lLJFA(!p>^* zWcbgF5TBu8FFa5Y1yW4V^D`dY8$(f{f$@XRdLRKt?}<{VJaK+As+$OzIYc%=Dh^6Z zgS4Vn;x!@*ICQH7s+NNTWk5nX|MY1d4Z!sw81e61=?fj)%Km5Y@C<}NBB4YgyW8eG zx&cECNz>DeeActo6VMjWv#lh5G%fk{`_m(nRA1+Hga}H&$F}Hq{6EHZ)+5{rV1Fg| z&nJsOd&st9z=yo?l}iQB*%H39nD9xO=lS+9%!vf=9BKcQ(}M9%)!XS|Z+yd_Pkw3t zyebT7DByhd7hbJct3wN+*Z&bAiKr01$1JN<=@mSD8ZWx)JKe^lJPO2B_wt2B z=V!2<|4iY2{u?|GAI=Z?bw>uw5NEQmRI5qm^>9uL#Q zTd~&(y|$e*>%nQTber2L+hbdM?$@5m3k&5)y|K1Q$?XYB-uaqZVBGD=r~QX<+_z9P z>>Wyu5R3Uj8@J|u!TSuE*u@;DfRW0oSJAFdCr)(HR-3 zKlVgG`M794ZKk-W!F8@YQm5^?WV_RALOjW5HaTLOnVTzsvyajUk>9;wYdly(O5;-U zk{SU#1U>;=t4kYeI&%GD$1G`8j4&Qi*m*PiMo#VA$kP4-zDo|q{EJoYKl`;Ac>Pb0 zM)ht)w<}#N=v&)K&#*GsU_U;Lt8+d{o7(-&mc6^1W52s~Z2h}m!SmZ#${YEO9Bzj({$4m58=tD^n21oOSIsPETnb#)8=QtD#Jc z{=K!jQLVqEWPp^j1EG;)F|u+kTB(DM%ni!#gX z{F>gwce!u&RwvchC$cSfD|Bca@O;A8^mwM~5(+1sjByfn`H%X_jT1PGMF!`v;`ce9 z)Yj@idx4yGYuu8`l3DJFl)=TXdu1W(`SdH5Lg1-9mm3{pYuYF?x5 z6&)KJMSAsFHjWh;&0^U}qU{xU#@9@y&(FUhN#~vlQYqdeg>$|+M$j%bg#ju^=4uuJ zg=0{t|g!#kYLgi%=`mFbliizgpVlJ+Dl7fCHA?BwMDQ4KM(ol zrf7{G%xn*&TO@4m9fWw=!gN2z?b`#4`k(NK0yYl-^-B4h{(yBx6z?=(1|NRnI6z<;U;zj3(qVt_!`^E?R zM0R8D(N|Uogo1HpJ5;LEueyWqd~f!P?Z%(tieTmu<>&m%t{kW)P`P|6!+b8gj^x!A z9g{dl4dmnS)-LY4ihP|*nY*SwH_==l8v<^8#<{+D`LeKos!HnUx~FaTBf86%YxfQo zre~(7uDhwZ9tAg)_r``8?<2AOrXeeqQ|>Po3Srg7bz75~4V~b@2jdh6oi>sY$E7}% zwK9Y1lt1nU&RJUQp`D!0e_{+~oV);MTl7CxS4NvAfs%+&#d15KcvJu$DiYCg0FTwt z05MHANN-h;4y1a?pSpe-;*3SZ_U&cfe9LyvQCe$bAlaE~nJXEFT<34jnU>+#*OsYo z)7%)So?4<96Y%uUmDVu0)xNYhYVKTNe3k%7q|U=V=YI2uj|_31G{O6kJdG#j`L@(c zz^WlM8Ex6h3LXyu5d)z1fxE7TN)bS~PUoE@MH&k&t;C$_2(kW1SDt9>#SSL1?>y&ugfxK%PxP*J@xoV{B4e2{i`kbq~B&aSfGbXl9bVGSHziXZ6m z20I3|XYJ5Dq~CO5%wL+%2JC!S&8lBB^5hz0_@Fn+X34 z-@GY+1;T%B_wqODUkU2iK2ft4)m zz5Kg%H*1f#A!}kQ{T9zG_F&c3jsnLvBUFdI`6Q*uBh}I}5_6v#o>oq@U$Pm(*jFC( zQoF+7P>YrjM*O+}^s)f`60W4L4C=}>p=u8ZsY5wb!j2^Uu!C z^9u*R-c1R|#6&4JkFPWRjnz}`7Z&}cYdmv;aq(y6O-G(5*el#pzG$_#OfzR8T(x$K z^@n0^0$*pI6D`71c$32Da9K0K#ClUK;rHs`c9&^l)DdZpQw}y$;1C-Mm~JnSBv2s< zRD+?}N&}Kk_G2%y#&7R%bstzVoVtO}qq=p=B^C=NkDj!jA1>dq7>%*?sv*Vu!4SW? z=fy0u;4&5RV3$41^1x%3pqP>00e~f#M?|6LopCghqRT5>R*D{8fg#wiZ+@ zWkQI)I%GgcPBpQ&QkZWjnI6TB()k&CrnzTbkT38%nC&a?+i$odOIgQF@05ctLE^}aTs_{`pNUS9ZPg-FHyp)cDE8c zrE3^G$M%nd8H~@mU7qX4X2*Izni`6+l`+E1jWOm7`FII$hR=n&-@i|Dy#IaYhzfD) zPn^1|?+~DiW#?#&(?b$bdFpHMS@j0`mDT#%d1JE#*5_g`J1h#5vS&Emaq}gCx!vm2 z4xn@Cm%LU4IciI+u>c6{Kr-q&@|nTBh3lgA3mFCly;Pp#~K2ppN^| z50*xs42!(%o_%FNT)#mw}v%0>_j%`$-zUV zIM_H0uHG*>x3}XX@{Z>E8f<#rn z@4&g?Zcl zdmTt6Zp%l9RRs1_4hM(J>bR|dL@-Us7xg7hI$E1e+oGK_af6IRAbvN8%SLU`ff^zD zD|ft3f2h*RMeRA-I8dvDS2GzNUj`*jeN~-sKMbZ?QYd>lz5yPGB*tfm?Y*mb*(lwIxvguN@sLDfC z`l%(b5%PJi4tJ9nbq@7yeC($q-n>xJZ*OFd^{p1^^#(sR&g$N;q`Z@!w6fnx{9J(D zYqi<)ePH@tH5!Vdp@Kk=oeN4^mYgkz2flW;zLfT7Ys?W{x+-EyQ#4E8Z8#{?sGIm( zFjmd*&LStULAL~)G|>clfz^G!AUXc*H1K=5yfQZs74%mCz&N0gNTUBn$b4$7`eS3= zmWlJ`)jk2|X0;i%elQs5x>Rlq^8;J0w@6{SdGo^Bh+~rnN@A13QKu6a2$-@;?n_56 zT8%pGzk21~iLR}%#?~ZAudg(#l;*#2`6bK4qsI?D5eaaUekj~5ha2wf%y0g&g3s|^ zM#~&Yu_D-=mfNGWOVX2^sMrCohNWwF62`g=K~v0(S{8V=UYjUXtV!3z=e{DVorTe^ znlqM4cXxaG*}D4qFd%MMVF#BEh>6d2@IQ9(OP`-NZ?Lx+8S+&`@y}X!Vesp}QZHev z%oMMSC}>ehReW$7bA1g48CbHX^?G`sG@pG>Wwy!p@1GD4o~VJmSilc$iujq*1G5j1 zVuovl2&RCD<_cECV||qTZ!4j-O-|mn%``8&E?37R;mFTe|%eV2P_b-67!+(SH!FXfDR097=_ug?9oiO&5Yo0GX|f+&XPef{yY=hnM> zrO9z+a>7U=UDbq%e#a;ygHhAdskgA8;%883f1UW_2rI7XKH6$}63m0`o0e{p5SlFW z^jznS-TL; ztp)U@(`|O7-)`s^K8{{~&nolLIC6=l(rJk4urG6aH=2NFe7~M^U21fQdr{xUtKabo zur;FUvPWEy&Zz``*?sY0g|SqHS7W4e?^mJU2NptZvst@!d*M>!af~Z9oRPOD`i(r! zt^t<_9H*uoV#xH zkO=Qp4F-H}DCU61dveXoZ5H+T@o_D5(+v4W3$fA8ZkP7xQhcn)DsCw%76f;vXV6^J ztB4={fP3qXWW^(j9z6%ceQ(90gr^J;yPa=K-5MP=kxAXGUC zDl?)jG`AL?L&JAcf)xm{SnSw2ewL0c5d0CH(W8^W!hX)j=9(`a>p6S^axU&4llS!% z9W9qwm07}vs;aKND>x#b(%XjQ zu!Gd(HCouQ{!Q8n-z&J^L9vI-Zbb^t&S{K094UsM6wYAq@Pt#f5z4Fi>dIqAWh=OF z4j*TWa%Omt@yH}m{eCRHRKOF{2xT9IYA&jD>;f&W<%?ZVwF`8{i3bxlYum*t|{z2>4v z#{TkcF?QJlk++ubvOq7lN74@WaZ6Cbs>kzNEhz+K8y~lZ2P!QeAxbF(Dhe41Kz3sM zgFidk^&E2IN!hUU)mF1GYqXpd&!o@Yu(!7&I_=)wFTI0k;9lRPE%=t8)*8lkB^<72 zpHQ%U~b zoKCG#|CUDc8^u{bV~+i5MejXi`ZH3MENZT-SJ86Q9RbACRvrvxUIjj(5qi08M=_MA z|Jy&pe`a&d&|~o41PA6}L@?=H-W1==Hrp0w?E3-&A+H5<1x|^-!(MqKXr`<(+-B2A zq^#e$X{Fx49Q=NJYwhEvWNyR8u;jBZqi|Ss3;go?&u+n-@P+C$@X1#UU=rR4(=3{g zayq$9P`Ns5b}Ps-a#O>ypZFS>28R5+pG1Qhry5W*ZlOb-!Lxz+){^wRwBxgmIx9gz zKW8+>H2S-7yHjV|)onWsb?%~oFKO@M5fdEPjlo6e*gPRW`wgu9BcGvnMd?saV>@%6 zT3mRRS<(AtrI!{-&S25yD1(Tld_Y8i_l=zl#`E?lP1|W{bJ*y!1Pk~roG=opexU-Z~YL)qjAXGV*XA;CKOrY()Ayher|B|klheOkxNN^c3Wrfd=cjG@c&?*6siP zK|^vhVTnc4P?;#dl-VaPuY-N~TvUR!PS{?=vshElEU9L`F4;XT@57 zo#_b(DZhEfDTYI-cfxWyJex5zw{w~4q-y)L2^hRYYHyUYO!+Taru0gA$Gzr$i*@r=2~|S ztvYF?fWDXso4D)o&mHR*C0y>&R9O|&4XYN4XYsNYS2=c>ie1@dTvC63oyDRVJDX`m zTn`!*keeH_!dF!#Iqj9}rKYIDl_6RgwCTRjz5q2A*akH;oiDW!;H_w0R=<0`Ur4PB zD6Lx5ZbY3}iF05`h#T}&Q^k-b#y@}PvuQ_IJ}{}^GDTjEHfzZ{atrq4 zXW}ZVob-I&`P3gdf#~p^WW4)PE<42~H(=q}eeAqohlfz62A;HI0Dq&d`oDJX`Nu zS#{FQ+#zH>S7c1pbD`6zVFFWA`?5H+qeOKIKCv{1v&T7?+L&`bEB{76y1Vy)(G7fa z(4X%JcuGkeq5R^6Av|eDJROUgZi^Z)S$pd-UHIWpz3y3uDL=TAe`(?2nDv;&*XntX zKTG))30OrYqWX$bW7S~dRc_y$P<>lDq@~0#oHH@(h?71u#}}h(qia4u;+M=4OHVA{ z;CpM-boOc*SYDm2jYBJ#hnMP37yhgGzis`s<`~^^)iLpPYVp(ng+O|s`qcLGLN@q` z!;D;{pHC7()8nz&9N(AG;+M}z>t{O4rTn372IPwWwna&WOM!WyIrcDVh2|hmFk10O zRpnd(HQKHHa3&^>9F{5`RWLOZy!}pO$E(VVWKLv7XzKZ1CU6YUUNw%q2G15Lxu?zn z@DtAu3QQg&$3xBC8uuYSuqz0QHzZAUhRg?}x@t6gHnc^on-Ok1cMnD+`g!8&UDR|} zW!Jmk$j7?7nOG-GS7fT6`jB%ZzDa@!R)(sM)AKKetj54n7>MmIhq@c+|Ns6}b+!C2JlSj)VLlAQ~*;#|F=tt941|GzO~Nou&&<3bNfbkRXSFt5)s4q*XUqREvO`;907*it-$Y z7Wahj?PT-(-M+zQbp|ufn~r<2YGd`iVlyz-v~RH8;kMZXW~jN|l%D)A3w` zpZX!Zc|K^rf*cJVp6dm0g!do2V>k<5x^}wpCN@GZ{l7T8mc(!JxE)qP{YU0s!G$>bY(_2WbjbgxB)$1lmGpwrR@BHc7AIAf(nMi(f=KRK8 zrV`%UYvC2xha?j12h+no(LBs<0L?)T7mpD5W*J1zF;JcME^B2j33L*aq;|RMWa_L{ zgUB$n&&zn#(~-jC1g@DJJ==P5`d7!p(F3ai4rV7cwHr>k({9^~AqPU7O(UFp(;dwL zl_sQ5`O}g|iBB8ykN`Qsg|yu!^p|B+Y_z@o6G(+Sq^qKfI7dI=^zq2>$zax((O;xO z!KqVn6en@aE9}p5&5{Z~2hvHQCUCd^eHaCF*e|rQrbejR;%2x0#LewJX7JBwj_L=h zk71t#9IUL>Pp`@Zh7yS}&#aKg!PnS8Z^OaBXlf_kpe{-5pPZy3&fq}3H}!i%Di2*} znSE6K)1p-wR^oJ&!J+~)n6$TeDP_ai!JmNQnbgO|FC(MD?Ye*7KjT}g;+K@MN+nXd z^m*vr*Yw%(oMfN*IP)qPWMJ1WfLY>zx9bkQksLORmEb~NS$udp#n;Y(cW#1ry6<)O z5@neA5JR^5iVTQ`GSn^OB4zoR_2*kUh2awQ2TI`2nr698?b8mK;DIv_^k_M~`x%>Y zz44M(Vt-Q8FIw9}X@U%qdu)nf|6E{S2f-~AN}JyQ{tY%`S}#9$D3Z8btRL;6R5z++ zH?^hL<34y1|22+G$4wd`D*i{Vd0dA_(khzn-Q2iKep}P-6EpdL3FWWMK0?`^abd4um|lEV>!Xce{+_Kv0vU+`_FgCv25@T?nuKM zz*JPP0U>Ntw!ANHH~0Nq&v;@}#bG4hFWLJme^g@UydcK>X6MXNx6s~6m z76i;dnXwQAWI!K#okKEx6Xtw4vhrNv z-@Uy~20PR6;0z}h$8|xo#V!VhQHIk^>xi7l5ATQ388Od9{FRA`zhlYhjUwV*q09qu zME8F?6r4lW&Dr^%LAVCTzu=3!?~NMxE4auWUW3oh*ZJBuOHxd#7d4?9Og zX)e2gR%FV+9vWF0@tk~2{0rCKo{ESwd@W5OW#*^_8Ikg?yus{ zd4?&&DilenAGSnm2%$+vhoOKbF!Wk4?+Dios;WGjGF`9+Q91}9T|jc7Xzg0 zB}<^t<3L4-Ys#Y-dLcMN@@t&^$!_7aTD6xhQT+*a|B6P-YiP3oqvGM)NaCl(JVAf( zhVLx0`!!B@M~P|9{>S$wk)6&y^;wVNjDY7Id`poXlf%^x+qFj2=y#3Eih~i9(<=t_ zK*Lw|BR=4MpU1rIx@rpS)|r}t2M(7HJ1MvGvCq;|a#BJZcWWs@-dZHz;`%djuUF!w zhO(<(#&;nd5CNhm;YNXyF$=4uhH6hMDKLm*Q+|_kG1`?re}aSl%$dqgog0^JC6m{m zHOC%U_j)P=Jzv&u@Z@0mzXt{+@_Oio1t;H)M;M}@PNgR#em6Vd->6|~wq9@T49ppm z5R24MAbMR~2jP=?*?(&1#60t*^Mx5{1T^$On=$A9PBAxi9@#f`FK@>QXQZsSaFqw7 zyl90X3H9gbfw~Hzh7<^EpwdL;NoHN&j@QY-^LRzxlM(0RS1aq1l9I8G5C1cuc=V#= z=zj?)h~(LPnUNNO#DN`2uOAhpD$o2hL-gKYM1R_FLtj1C*9kk4Uvlv#zE0g%%~+QA z>AeUA>_jnHT+=kb2Pi=g=PXp7{KI(mI!C&Sicuq~0i2W^Bo6EPqc#jU|r+8!XEn%-W z>hQkTdq|hAo%tbPH}jKQQzOERWEPdeb@`n_73HUgSFfb?vB{YS@tw(xwb6^C{EUk7 zM*(QlHyO{A^m)s;glboT-6G7(_iGD*SO`jM-%Bey*@vETn=|Hf%rSbI=njIpuz?2H z-stN_SbbkFiRDSQPEYZa{S{GH{|!@{CH7fX&~u; zokp){7;`IcP8rY_utIK}U|puii?yfcVh%;%v=QyI?ahi>oFbh=k;sWf+&oX&*M&SxFINQL^(5|{zq-FTJJ9PPT~%}1-DiS(@IaubkiR1Z{gdWr z-${`i!DqYLvyZm;p$5^0P{D9L-!D<=2}@j_`tiQGx=LaVh2W<9)lwEGLHJr}w8bTk z3Y}0@(Z%C8xp|~7yjJkI?sA83Q^nnnbOFf-bgTsGn@ig)L!N zHVnFsPbJ*K?Bl<@2R~IIBjdnf+DWmsX@>AywN)Q7BCMP(he8~Rj1*Ih_{G}TKyFjQ9mm2BRN@1I~jLAYtz zNXKn7^_%8hu#{Ai7*Yn1k5=88%78+UW?jhsA`ZLxlBJ+0nOA75&5ZLSqi!Yj1J_0D z@3{2QNa}Zfx=kWhzw6sE*uRdC9&4?ywCYPyl=*jZ5r$gLRF6qfiRe;=Tl?&O9xHDH&h z#U0D;-+r30;j)=1127?3BHxY`hSgf8*7{mzDC#fdUzD;@Nq{AQldeL9#1x z4>jf3qTtY`c)+_O&qB)_jb;5{%SuUa-Q6VEu*+K{=?&06KN15z_+j0Zr64*2X~KY% zSNG~EkEXJ&SW}DpgE~W~(MR%Yo3JYdVebVnHn>vUWHw|hE@Hu&a4+V}qx0IpDFdA5 zka4=*EQQ$>Dt{mHL*AstYBJOAa2M2Qr}W)o(hT3!{BYas#oOBlei-66aM!2>mpdHH zg69Pbtk2yGy$;4#LE}bL@^!F-ySU?cpW;9L+(JhU5Hr7tD;u=0d&xV><^AF>j9cjkpnl?nw0b@e(0Dtp4?o;0+7A@_XKSJfWjE;lTR{C&&lJ&eGS7p1e-V z8j2Ki&50+1Z3T19a(w;A(+}<*l1)+hlb0BKDc`m2?A&=13_6*fYL~aZ5vx!_yn=K2+ ze*adbudEHlTeg5c43@hxnX#z>Y8siv?OBWh&h*F#p|&#Ews(`RJ;>Q9KFq$YJhgyM zGO}9@LAUbqnN%aozwyD?v2waQoO>1Xa7S~XmEbx{o1i#8*jtaUn*K&*kB;3Q zHqoExo=GCb;7N~zhHW`QH0n^#DC)};JUK?t|6sX@=q{pCBAykF74)y|Hh>+RT1eO( zmxnK@R&x2yw@#{_nV5+7w8T=V_KO${`2x0!H_%=89()#u`lV*VLaE92^ zijyH8(v3Ib23QwuC)mJjuYS{;9iea%Hj4^4ro_K%Dhx>uQYyAxrB5CW4zQpf+q**Z zujpf|1@bY@QF;mA+H6q`OCN!ezQxm4a|X?zr-_p%1c?a#unbkE<-9eLt51oZFw}{w zrSbdiS_IYom_;K)VW5e3$xfWz03MU>=9KPoG&$ zH47_SO*@?(Vil-))y*$4HJbV*ZKIP_*S@2&pPb8XQF@H?f2+pJo}wX8M?)O?+(8C9 zJ>s_huec^K6tYFGCtWEwl|V)=K~rS=iXU?4R%*0pm*~&+mbGP9Hc>IJTE`q9nXVaB z>L3<59XG$%XuKEXBXzRhO2MBcUSXPcjz<-?Cxo60@f2eoy~TZ_MwnyT#oi9sbz@dl zDCi>ajEYodu!lFKSzA4IFua!*)IPLibarjMG%g$FeXBVipQaV=o+a`gw=I1*b1M!;Q#8V}Fud2ggONRmZDu+IIcmn(G zH(cn|yW^X&P23_J&LLWp0$m*H?JahdB%rc<{hpBOAYQ8918PfaYl|&oenDL`Q%TYg z{d7@U`3tS|Gl#QT5|;xN=NT=+i91Fq4B2PFMt?3?mxLhvU`#h$%%M?nq*=4az`nGO zD^4Q&m5ZJm5f=j+ym*fynfhQa=s{wjP@3P3tN#8V2^Mx$MGMS!`^}k@g@vjQzqvVH!^UI(Nh6&Gnd170p3w^+q_`L&Xb$avvq-~#RJ+C6_upfp_dQH_ z@FfCTJO1Xykgq_b%XuV^fcE@v^EdT?5se#veph0j+6ChR0jxI&UZHpvola5@S-U>s zSJ{S{Q-nQ^5}G$s-#=#{gUJ90DLuXwfEuoTFo8<~EWoQEsoxI-)uVubyd;6Tn(pMA zG!VJj=vgA!x#CxI&#pu=WOLAPn0cWLl_+tpmw^mz1;S!*8Z-)|e{~TNPxmbF(FL1;3du>dVJ=uHMIB8VUYV zn7?h%^-bEdJG++_>c(s}%I4(eP3HP(r&%Q7fk)NbW%ggAoA_`3z@755Q2&33LGTSo z^45c7>ru8w9#cYy7V1|1#d}8Ui&p+7&?LZNs)nUO@_6=WLg)o$m>3(|`wHC+ ztZPvXl+|7R=x#SREoW3BDzMktojYMg&HD#{bZdj?lqhzT0^P^_bgVb{C7qe!65t!t zfR34m9QT0X$@w$NIjBOa{Bq`ip?5y%DOlrL=%8J1SfPR!ATIJ|-jG9cHHNU{b50H= zc^T1+&c<9*vy4t^KqR*u-|0TnxB{@3KW^R(kbrrJLY-4fAdi0ZWVbI?qI_C3vHXi{ggzy?TwopOJ0$eX0VsL&!mF|TmVME6H3K`-?BWu6v!lzmwa{!bW46PZm8vr@FVTYe9^55?HO@e-M z9|_(B!TQhAasbEZ@lTB*8pSLBLVew{gbH6Bjvigqqei+pKl0`Oc?!O+-PqScL^02H z7&mnIKKj8=@}R^noU7g?)r!X;RUUhyGBC0-@r+!v^V}&b44t>fJVf6IaIoZ1)R7gw z))tSXYJ2clWuhUNZD#u^#5)14JV5tKjIH! z$HoH++^|rTVq(ovrn9_+dlIwS0ZJc0qWt5qFgD_rF)kM8|1Y`=M+35(?@qOsMntPG z@~5U@gDC%$E1u$C3K$zTu`Jyj0LwKhrzzS~BpQw~u#(P|bJDzVxlrpA*%6%~@+%?o zD`<_R4rq#_plBM|7dNP?lnj9|`7Pm4Zl;}o9~em_Z|cImCh;$v*yi_NaiW`;=xMIt z^kT13|D}R}sS9(;{_r4@44^z9i9&7O@d3g1fQ?)4PVJk086J(tdH>T%9y0VBEPL#Z zD_H0y)E4T^pmo9thV6YcHI{Y+r&JhXzt`Zn7@d*qPWw}~pWQj-?r02*^D8~!ITo-X zflUU92hzWQAsYR+iU5M^|31PGIkC#T^33)I{0~1Y8X79z<2W@CUT9oHtzdqrH%HvWg3gLH@l85)qokx@{o@TWy*Fw(BqpiBMN z<4)iHze=0LLWF9xqc;qSvG<1h(}f$M5Vj3uD(N;&kCaC3E$`p17@4z;0MX^+8TW9m zNHzDLPl$IxRz9WJolr#cYlQoz%ay(UlDWT_4GU#%YTm_0Tt+9ofZn%1>HWcIznYFf zgYc=2bAI`Mlg*E@Af73Jn55t+kR?+@Q$@1Hm~20StENI%WZ7TIBHq26^>O#AVFL+h-7*vK=9j-c{>uW zi}2L~Xibz19cN!iLGup`+>(GMfk$dAiy1_<_k0KK$>=V!gA6#RNeEhr`d@G!lo*US z#IE1mn+vipo&v~K37}DyoljrNS>PqlrDnwMe2ED6x~2UdKto0O2SydSz54$r5Up7j zzo{-ecS!@-OD&Yivxy=$)fAR5UHp9;srMWzZ*ZxP-dk(fBzzIFv;J)93K7URm(cPz z*jwd_Jx~U6v3Tehx1HhWH<|nK!1?QD4U$Aer`+3qh)rK=yfWdXSj{t#Hf2HbZDek?Yc{r!GTZppxSiuPbKyeW8T3cLX=)PD^#T)7G8iwq)6T?8+)gtvmFbbZl z|DFAt1=Sps1I?BmDlxCG;7^h-66bsPC~eJyY*<+tGei!he0 z&7^*$IyIJ6LVMlQxj9;UVyGi0iKHW!bimytZf*aQ!~0%pq20=p-i9-r+?FY$83&vN^&wj7d5&{Ntpx@V>ne7H2UDF>(q!-3Sr^a`$1kmlsZ3?CE`+WrW}`OXzHO$hzS z`-uE94x_IjBDkx*16{rXx_lg0%1fJ}4UMOxHHWd!iid7^v!-zDwzeq}Jc zIjM5|B;@qB@W3}IAr;~D^YLGZ1&M87LqM){mHh2W!rE9Z`*#-F2pd+pQAzX5c8XH#*1qZ-Lya0M8a2?>D{ zsr(xvkQ7=2UI;4X0Ob+Q$3^OdDwd7;Y){p$Bs47bnyVv@ms$e2NeFsncy!Y%>MZUHq(f-i#-KEX)X$G8CW;B7=Qw) zmW}%KA!j=jC@|}H>(;2$Lx|`3`z)kBbmgGQf9UUV@e))y3XkEsc&T&CB1Q_(i!A-X z^b^8f$M(y3uiil}U$iYlB1SBhc70X+`dpNQikXNzcXAv`_}9XQzsgLk>4r|n)bPpT zyNuX_dlbFX7ftU(`a&>#KO41GA_5Pd)g175Y47*21%R2EP3yenOzyI2GCLTCb7j^yqURzYjz~zg$bA zP=!Hw*O4!+_kp)e{-CBh8f5o)p|dIdl%QsaUq}a>q-o}j%;{NW>B}4A43g-9(t)bM zWUj$SS>cwOY&U9>$AaGq1(87Aa2buI1GdlScB@G%jSMvI29PzrtWT6`>jaJ-e_b}C zgl&?kqanETDSnlfDnjHwrc2^3V6g9d7S9)MtMc;KL19iaj zgCHy~aiqlIx*z`G`TLMYIRS^nyUdUo|Kbc|`buhV^B5mXfwy(}wEncV)wlMx*Q2dR zQq_DUAJ!WRL*=G^17qa|sX-XXCZ_5chFbO{{Rghm3rs&86-aWcnu)DO)t>uL~E!h6pRcz{QY>L{3F}NLo!Po$;J)S_ehK2AVA)tIBhsS-b=uI*iX;e zypVP{su#K)^1z>{$8+j|(n*ovJ>W|-c~EFsj&V1IOl zP3gOXH)_imQ);{57Oa{jXkFbH|39&uNC5YWkFFG!pV5ha=-E3&{a78gZEism@X+-J zP1q|{ysmfm{c68Xoj*v~YWJhH;_-RAgpZl8My`YZ=rGPaU&gHsFf@BQ=I(0G0l6f@ z2sS8!l6Zpprz&6)LyK8HUP&sRIAbJ(D-?f0{q`K5hujy`hghiQKeKce$NaW5Z(bS! zmW#puGHo#GW^+PUMo0JVji41LTAH*0w=g5m?1 zgUd^N?J%hH_0SY8+XS`X*CxwBS8`EwJd%y%u)25FUI04dmA-~*`5y<_R(OMk7LEu0 z$AKNW<3#n-VFkfoL7DwlEv+oY1@=G`SQMe5awGO8Ceff~01r=hgI8W7BufaMUZAeb zh;y&C;mMsEIG#oXKB4;>-*@1rp}e?a8H?JYP4(vR`OO&=w-0f z*r&FY8<8~?DMA2LbUk56Lo9}sI9T`(SO?o5fnsTD6{^N9h}X1bUx9 zKJ}!rA@+Ul09#tY!SfL#B64cVcn}LGIiJSD&;N;qwY@cYktR}1_AXD2NajQtJWZ)~ z{ezoH!6uhJ#dS3vCHF+?^t7-TRF;7Clxo{>!G6>b3De}K;m`N$*wFxqO@m^65ME=T zqWzvc?eBZb!IW3eU57K~a?BC=c@FysSRzfZ!Tz`04=>Vztv}7P`ojE zz_H><)Um=_(`V%i)YTX}I9P-h^9Y=(oBn&9Tx@5Sq_G?hZb|Kny5;Hc?!~8i`qj4) z)do#f!Tt$jp9x;q8pU&{^Rvo7zwjJN$w&tpQGxAIf&Ha=$;$iYp`S2tv%x(7C5=9c zreyfXK_Q!BaLL*BT;iCs6bX ztdhL&;}W^su6{=j;_;+j+ms6Z>xs}Ffy8)SfbL`e!Kg1=D z8#TTrhmAUbhk+&P53oMf)cR#T+TKF@OIlXg6IM-bt9@6fDzEE445*o)Nav;CkEUcelt9oK(SQeEmCJuY-Di8M z`IEC+KjEH1holK*`SKq#LK#Dwd}TeS@CS>pn}JaQ_67D%}kL+)zcL*|wA4d=1ryIRXR zvrXUcs5BpS{f^wQ7P4*+>U)Mk!hBGgl3l@zVnTx5a#DF@H#2&&@(9C zne$-!qmUVOnH{eDbnV^h9bqPPqIYo9!2JZMvryfQEhiSMwlTXrX_R^RamIMVjV-H= z{`E9P8Xv75q=ZE_*-qIVi(%d6dcl7g8PE1{PCE@<+% zI?_YpXE$n--ukm?w#!nr``LyZiq<>1AI^oiGOBDj4jM%}LW~*W!Rh?#z%beM?E_0PuXB1FKPy%hUX06I3C?H={@qXW zTK2L-=inQ$7qogN$LH`8ogQ9vE8NyZz{}GkpS%!%;-$$srn7)UzAe3d*P#SLb9SJd z)0GRIg(U|So}r+g)q~@*YRkKhjmMMi>}$r9U|gnG#k%{%CS*^PtjEM;z7*is0S&cV zM-Q|m2paC!F=QFRpdu$_5^wWfY;Um zPL{r{g;c4F#QFld`6@k>_r+2`$`zxBt%l?nB$(9Rhyg-^7de#|=>rmR40849*?d$- zkw;2aJA?{KKT@ksw%=Sk;=3DZ6*j_1H`Dyx?SZ^k__;vCy0Cym+t%-2^Hp9PsQFfl z8^%9YO#)lJKbE3+gpY3+wAh1gN7FowH086D1g%o~Y!;u@f$>532(F&6sVLKW#FPIH znRwz|MP=aCU+eb6S`0T?0$c_TR zJ^EPnf)nuCjGjF5%ipC?-9HjiKIc!|%5LBLR+?IWoDzA`&^X}0dG z;xR4b=Ho|Ca1kf6KX^RnRap-Y?{Ja5Q)s8nii%MCVt4tv;CTlS3GtgpG2MdFqXAHj zGhSkU-coAbs)`H37C-5mYHs#61K*Xlcz2-$TR!~%Dmn9LsJlOoPl;y+V@WhLYAlZoiKvVa$vVl1 z3N3b(9`eXfvOI>Fp&?;LmLXfVJgN~z%95>YStiL+(S&RvYh&wouix`K=lAb#&Y5%O ze9xWve((K$?&rQguX*3Eg~^2ad6H2Lj9Cu&!<2ZBa8G83f&WtLiPrBVO4@Y;`9Qyb zgy>IB@jmKqbxIUMOL?+Sg6*C&jLM3afn)>NqJr#bqb4FVpD~|-DzX$tuFqQrG7sQS zd&HLRnXp!FJFX1Y(8(5&1uF#NBcH+!tH`7OI{x5T>VfX8pM`7@?VVn2cwTu<0=Fk( z)%^XJ>M9up3i8OJLHvB?1$h&pr6W6{7TJkwdcMzj)&m^tZ<+@*{`K$}R$t5E@qyRS z$;DGGQ2@mZQmb1W=>HU(lXG7>(uWqU0ER_WMN_lmgvAfF&6yDk$lyu&6d%)KM*GCfGrcL|vNm2pP`hwl_6| zbsmtaabe-1YX@4Pfcl17&(r@b4wh=!u@9#9K3uiR3fPJma-51h$_YJwioF&K>JT_@ z@hbZ9GPx{6baa}M6xa%w`h05m1=xS8qw8|5Wu5NDD!+woDZ(8ZSsZ?;8ZVSf9vx^^ z%gtXC)OpU^fln!aGd@_K|1qxV$ox_;J5QZgH%OieS_(l;b2<;RW>FTwpR|Kk%uSW< zGtrFjE{B6!K6Pb(*B|a?$hm&k{h=K38HEVjRlIA(dDIppDInu}EK0RFW-0BiW z_wJWOm1QTzy(jj^Q`1O{I0FK4?$XS}>}n1ELr|l&XH)N;CHdL~qQ$q0 z$yLSf)5FVuYugj214bL7ai5v!B<8B0)5SnmDfX%V&74Pe_xC{4m)zFjKMW9*dja9DgCLvllb&^YMXR($XuSAe>GmDGcY7csg z*&g4>%@=4SEhfI66d*o}+_Hh!Im$i77QGvjGWkvWYHem@hcZidysH1qYx#l%f_Sch z`?KsZ^0&$tTuyH;E4XnZXD~ofJhav=xjbrH@MGgY*tDsLKB|IJ9Ap7-N z770xuqJD%Vl(0VnLA&i&*l8)N42CA+*5#E`8xq6aLk$+yoSW%!WJ;*>t3Tf%nwvg^ zG$>rMP*fO=ZuIn7;oUhWW?WERa3}DYa|kng-#p3Z-p%#*Bkz4W?3X_zvUtBLdMX#4 zruQR}9LFA+@^wFtOB>|t)SSHCJaecCOiaU&*0enGTGw@mgDk}>(6VFU$L^Baj1Kxl+cX^f7sIqk>P}U+`>e~c; z>{K8Rz>nX>wfHO`uXfIt$;F3nX&&9e?Y^+5=5F8iX1aC*_o;Rxx0?rx9wP8RSb(U3x5CZ|0c(Kh1S0~?&F@d^sLXn# z%IK}}|KVd|)AK6q%WHV6`5FoYrzMmUow1<^n|M+QV4))N zTcQdC?qI*Fo^*{o7Q@HKeA{MCk8^7AdVC=!q*VozBlQzFQ-*dP49b4C*KJ|I+r^55 zl&3xYOPe)QlnFZEJ?9KWF6@s5&8cN-i(Fv^D1TB_7#-%Ww~c`z7iNg#SWSne(u4Y+ zk|84mtI-XdL<$Xsm__z8qU}P>(4}h(kk=WlXllx5L4bjb;dMn;&aL4bCNc*ijUv+o zeijVdkGOVEZjRwfUc0oi^nn04ER(ZH1(_vjo)qP|L2Pwe&ghd7mn^3VEDVax#^|A$ z_O>p#y&kGujU%^3&Yq7XK6`<4xQ7p8slxTJw@1|_Eo9tUX?Hf9 zDxwi;k`HAMu>vI@RY|56^7Xlj!yWNpMy&w(16d$+juVflwU9QhR}OYs;HeZR87;!i zsh_nRU-FN{{C3P(CR1Mn*VpDAMkdA{Qcwf;TUZ!#4J)SMIyw~g#5O|&yWBGLY9Uzw zoqktIq!}~4H&dJqT#}e@I^0nYkdXppjlc*>fOHUW0gd`yf|nlC9@!Z!KR?sUI$2~x z9(UfVMS69qQR-4x96X~>lkk?i-M>3X%&h!$hwe4Su^<{7rv!aH67V8#KXobKty1|kpR(2 z!^<-Q`Wxm#Jg^`^dZz7FM3@Txosf3+m=Xf9F7Pr?a_V#Rmzrm2YH@?ner;jsnW|5^ z4Y}>3c0pl!S1YC&W?3}&MZTQUDLcbSh~q1Rw)025wx)Z$$h`TpcxtI ze`0358k(#g+Sxp9i3Em1FKG1p>q0}XGU&b_6*Q}7~d=~9U!98*_xd?T%U~m z^i`HlD|JLQfzoD>n^jH;dU|qSijI^e+ob411Lz#oYbq+Ms5JKnN<65*-QB4$Gl0DH zDD8@a0$8c!=r8L+MDajd4g}m)6N%Bg;?L)`RU9~|+kdA>_lKE-2ln#4ET@PIT}po2 z)0{S5b1gw=-OZbl9sDt<&Lc`70mwB%*MtC$KH&Q6FL$z?H}_T2ExXXm>lwOCaR$ zY1B-J8Pk0i(mkp>MEL6*<#F4=Bar#Ai{}VT;ovSqF%o)J_xqS`$3`0HStok;buqKO z7cSR@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/combined_ThS.png b/docs/images/combined_ThS.png new file mode 100644 index 0000000000000000000000000000000000000000..651fc4d7ed5f5e41d6f479b3ef12732f4aa4159f GIT binary patch literal 118488 zcmce-i9eL>_XqB&r+P}1>QR!EsDvC^%k3WOt;$sgpk=km5}zizqRvrVF|qHbPojxQuC}D80s{}gXqolXK}m~peEqG1=F8Q2fx8-@M>)jUYx*xb+xnV zn*ZIw4gkYztDRL-1n14jL>hP~r0@>#qYh3Xz1d_6D-fI%-t^xCko`TV|2?d)m7_Vv z+>5PdjD&NXtZcpADails2_UmM-Yn*Sx2Z$ULe4_}yWr3FCI5GpL=NyKlK(sF3ns<+ z-_61^P*bpYBySHY+k*j~!ux-VBS7Fe3>1#S3Gm>PHN0_fuBq97128=p{~Oi@45b1d zT{ws~7D^)eax~1$)jXWEJrJhGmO40lk}nD3;py*VLx6Y^oXlxSg9sNj`l=R+`|*?Ft^(#*UtCU{LXAEuTy#(_bEd%`H14w`0xIGBzDg6HVq6kwxf z3Z-ePnd7{SeYL?T_B1Y8Ef9@jco0qbIzH++KYK3?9UCw#0zzPLNY*56GLq|SW`eN~ zw9#VVxZd^{j2+hrP8YCPR2{4b$KM`_fT>$R1c7KA+yq9n)bewpQ0!r3Uk^u!8jH=h z(T0QJoXl_(1i}tNuvVwhu}l*Z7ptL(qC1*c>o9cScyCKnEvR|`$I~2cZ$;O}qRIZ+ z=12`~hE<@a7se8=Nu>z1Z2SXAUfLLQFdT_Y0Z;Eo44{AwL+Zddzi7r3=(p9(Xi`N9DpzO}uS2C~yaJo+;Xs&+v2fafDgJ zeP|3Dj!l4>BVGVS3-A_vOD;jj%i6&TujNBf_d_y0Ob~QSZMdHqou-CR!`fP4FacOc z0s%usnLB!L2~OJntN^I53Co6Rg7EQU<8@&6G&00q%TmLRfTxixP}(e}9Ztj2TFnz{ z&cyIdJxw^aV9wSSS`-`!#wYoh`QllgP)Y#L$C{!B!!tSHDujr$*CzSH$Yi1>gvay+ z28UF)g9gwEPH=Mq+EGJW+suiHrFffwu}okHHJX#JC(R4av?S=jEHMmIaKEvsI+MZY zAfX(tH;ODUQ>R(Lp-g)c3KghMhns0aITl_-iZPxXXzAqPZHd>iQMcrxaCQ_go&^&I zg*j=PBFtGtxUavNHCYpDh6M0}L?Fm?KW!bfJ&i>pF=>2^z{1JXI4}T*HW5$+IDsFL zZEmbd;yL;_VZf8oP;9t44~e6}bgZzpd@RWp&j>Jq63hZD!C)Ay6UWENJJ8kxZp(Kt zBO$bzM5u3|51PP5!pv}XOqe%Xpn+l7AZ@j6jD5YR9s*6awG|nH2Sf9H1-@q5NQjRK z&cu&|gj?Y-bbE6imH;K7*j99UAlHt8L|J*E0+D1~psfxKe8$wriK$^v)S%+AJU<6u zfhN`vGRc}~jKX0#NK1ckf@KCVK~o9lWF!sdMC5vE+nU*X__Mf<<}5$qGuu;aH6e6r zAh14PD#q4~?}PGYW2nMQW{vhD5K(+kQI9oMXpozJ^IZjQDLgjMcI7@RH+0=#{$l@@lP7V-~J=f3I zkBA1(XN$F`qXi_AhX>EbldR67VAT--KTUk0cs}^W6cBBkVBo;ZjtjLEV62@mRICPt zYtB+PQD@MIPPXReNCbmt#WOKAc4P-q@WxJP4onS><2VU{9%5`|tF3MqATZH^I%tsz zUR*~o8qydAr`g)VJs4&zk~PoXnujN-JK30Pf-hpjy=?qo2qznLuDK~1r%6@A8RL04 zJQfjX41r^iG*eTgH5dY=j^&toQ;dlmo{yG4I7V5U2M9m!XralV+L<}zXMy1;Z4TSEYvv0;JQ5+D$trtqRAa3Lhe+}hZZK=L<%Sd!s3ffNrr z0h#4!>S%0ghec5ZesFtxPa7(chb8imXuJtBK!;}+fN*46ku)g&1c1B7Is~M^3TtX* z4hs&1dYc-p=g?DD#Oy2#-!_5qj5R}zBiHX;DC&a%f?+S-_s*-mUW1*c|-GWH>I5ng^!Ur&GwNH2yJ zk>#)E39(@ly!a4x2#D||)*cLX8yYr%siy94V$QX5G$Gh=5Hxjvgawmrs%b(Xb72Ax zQ@WO~lQu(W@RnFSo5=7dc{0g32F$|~;zXgLuy{|t6Wobx<;S#Sli_p=ws7a1JpDls zGqH!+dfKb0sgW(Ma0G#dhkYO#=|zP&VAZ|gEE5KusIHEO+A#S8Qqc!1_Uvwh)+^qZQBCUKsYZ{_hTTZAWz+ z6hUWr+o8CDbSDi>zA*|7xQ>Z^0NsxX543S&s$tlczBX`sECO#0v%(`C5O}1Grzh3h z)SLnw-Gtx?!4cE|H*+wx!IL-wJGiwa8#s`O7tWr?^x#s|b#MZlIzeDV;WD&6INB%# z7f+%RkqmF4Im4KK+6)U5UozRu-&Y6e1M_5~Y}9aYdzOwi+76)|sO<=KQgcw_BJ8v% z>JI80A2NsR%QSP;W(r_|Tum<&luEHeqs$-<5Dv$QON2T=?E_&bKc=4=)thAO9f&h? z@IZyYHsgnN_5aM^FjG~kkqUgObb7RjhT-Q zhe{;F2pAK9Wkj_=D=Qj?#&GmCq0t-}G@dyR?%?CcLgB#p!lkw*7*oie!tl%0aL38G#y19|lC-!73n7 zog-va0Y2t{-qLZ42ua>5Uy%oeB1|d3WXffzsI>K)SKc-e7 zU?NdOTU)Lcm=KcZrD^VLjS=k}yc_lDLjY?E_13waOcYx2%+`3kh+7oA0ZTk!OM&P(dWB-US3 z`M}=IP-NgM3heJgd-c%TyP(g%NM?zq-Wi36W}fR&^zR;pTomuUd-hd&jaiLRevJQQ z*|25Nz`KjVMxSO4{~kX(oVJ?A>0dM$u8jBxq(}ch{ClO?;GJ_oX#Cx&xNnqy40F2y}j7LIjmYt3-mme5a8!%p1?0!L@;7KW_uuth|yj+g9j*>y_v z(WcJi$&4DgjKu_U+T8Y^`VvDh+w8*cvet1n;s45&$Ha}(>g6O-@pqsNa(&XNPttZv z4C$0zz_N}^oLXPCqqz(?gzVTV-!ZEEAvNzyRK1m;ci5EfoRPK~IWmEmGnHF`NhCDP zXh?)4MfRZ(5*gFQ8tTuJp^^2@0~-h1PA}g60K2G5{>3HkkXpTX#Wy!_qenl>)Nk+DAjuNK&Dg2PAyovl zpp~?!boKjy)Hj2cD6GV|n#l^UxdvLir^QIL<8<50@UEb+vRND{ztqwZ$JNzy9Z0$L z0Y=^5BJGoMs-YrkWDA&6G$SHE*RSvHhxIDWX8i3VpGq;+A(M$Io6fCAx}fIFSDRR! zP@~cLDa7axt_iFf9$gw?{{$nPn)J@KH2XbE2B*>}ax1&qJBIuBwP-q~eqEN&uu5;3 z?>a4mXsi1m32}entG{JImFzzG*MlRa65To?=eORHdO9TTzoslFi2KuHG-{L)&KX$Q zll$X@67=ZJ7Mmu#Mi=*N%LwdEos4Y2iIdG&HI~jm3DzebJ}5^d>|yrOQqEYx^F-FO zC65i0e>o!NejFY~+!+4MN6a}zb3P6pL>P!?l%33*xRGXG-RG#;%#W~0dhC-@OxN#w z9^%GiM>{_sSPqSFeS5ym)uU}-e?~-+h(X8Ad*U|t-M4_rSsnQkWMu$0&#BgXUa(&B z)D49yUC=~>BSc0s*AH#ea`b|lurYj=E02!lQ z4rk86mWc0kf9Hgrrfs)d);%EBp0W1Uxx-sWV!rg~q7w>F+cwglv90cuw)dMYLv!}C z9Q}}5&PdGR#U&IT9f~uwT2~q#?}=IOKHRa2i-Ux{T01g+E#Qc1@M@csF0Sq6qIjN> zM5^6oTV7`H-iy6C`A73jaaJc7T`>2UDQwSWC&@2K#;Oq;LKM+Sy4XM4S_1vP*hYQ0 zc2q6{ODJY4ZR1qscQ(!cUYyVF**~B5HTg^Zq5J9b@LTwMR$f*9ZNgRgBV3i_8`a-- zdJm|Y_HGr)JpqBw|MuB2Cbkz=@cEseg6;VIg=vypTF<@wjatKJYi0p!^r-1CluGel z3zD;Z(a`ffC3bD^DwOLzu8~XZf*X*dxi1d;rhI#3aQR1Qk?jBhRZX%R|AuTX;6@Xx zcm^<6yN;dfJL0z4nqGEaY_2};Ld5Icsfp34sPixTDL7l^T(HSF)a8tVG`(OM(Ru~k)sgS?N`-m=ZdtTU3XKkYD-o_stpi0E{2g=kL3Kz$&c$X===hEv-u8z=V8?UmisZXU#>&th* zA?yHwiEY18{FwUMSfe;PJLp{8ufJne328rQ$1~LB1Tx-@36d5)uZrD5i=mNTr@Z;w z`%wsJxS`{)90N9l8KOB| z;UqUq6`6O3E1y94#|$35y_S?8zfc`Afmnda_2eO9Ms=sqY7SlK(chG+6LE$&voq5p ze?^X{JW`SAvqt2JwaIQ8Lk8%r?B6nSD#tb~$7Bw z#_>>1Z|Q68N{^SOgSWMde@=`H&{MqKgM9Ei;M!n`#EXu5Rc>-)QW@q<=hN}MR^YVU zO2YJ^@%aP5Dn@e?e%fnQPweFmD|D>8XQ*4NIX$D1|1`Iqh5NgM`;R`2hq+RdN@jF> zp|ucaMa>+qa9%)hmh-<4E1GY2=DQw9FLFeThany^Gd!0ZZ`35sMk2LnN_)NTMfMpZ z7H@w&xo5Afs`o^0pPpe{WVtzNl!IEX(QBxyCY+wCaqQlrgkpP&+HNi*RQ&Q%N4qay zzp+ePhCE|mvG_7yD$x=VIf8A~GJ<~L4p!~}#;5UpQD}X|8=-rMZB89?qHMNxW9;FfuS9Ak zY*uFmM^A}z%h8A?L@Z6s2oaI>1jLM2O zZruJV*?f#qBT-$+f-bQ&I7e;S*MBws&*IzOzq4s#W^E+QzOeeTVzth?6LIEsi8(WQ z`dds+q;kL3Mye%Z&N1U$jvr+w*F2x2`2I2d_4(IFcd2Oa*dm(e`nF3Zbu^{U$YyS3 z$F;s@|K=~6DZMQQMI+*WOA?|2Q2Af7Zu7fuv1p6GI7lp#v?FR{$H`&!Wtv>k-ISWR z8>o`N<$V$_EN60}hIj94;bHO%UBu~bJ~OL}k@fZ}vd&owN2#(WGQzxnr#jww7-Aom zGB2>SqCfdN;tO8MRs)ULbbBbC);4#upu%w|z&>iEW~}ldFS+uBalz=x)+18IW z291Ur=)O}faJ73GLHx(p&3|lNxp}y8Z~xHZu(O7%tSG8sNjBFt+@x#5z(6#{oU~w4 zW7MDVbQ`Y2H(*EX(0KUl@5>J_zB0+dZns7*t#=^iR>w9g`VRb~{CqoDhS0ETQ%Lr) z=TaTgcR5mEZOr6~4{pD_O%JSA_l@=6zs5e^Er%(|9>3@R)Tj2zQzdw!alNC5@&+8?2^=dtCxz|aT`QMYoD=+yJgnw-) z^s9Ec^l>#?N^a%UdVlUzHahUK>JAYd;}m4~ja?zbpKI8L$ss6G(9I2AV+a1e%D$wk z&%Vr%Wd+-e;&M78{|OzvtL}w+-;q4HBt;%tx|N=7Cs!OGg_$=f)> z*q_Ye32>zVnmW5T+9Kb6@$v>-t~=wCw8eT(VA`j>w#(ldmbEnlCs|STt_HY!5qWnC zo#h57*rK2mch_*m#`Lc5dMIE`XZtfF=Ic7$P}7Ri?;*~bRioK2y0T9n93L-u@Qq2S zO+as4jwC#t3Y$5r*0`La_U?q<6XhOJx6BYxKT? zW%-s!$&{Pq63F_mko`r=tW)gZXL8t0jBtmZ5_}U*iGRRu=Km@)4SQ1C=i(rza?VFG zg<7OM@@M#~P3c<`*v)@xHroyzku^fuZJ-*%3J%`iJPOuZ@lKazTfmTCt0VrxhWwwQ z+D~8o`7hQlO!NY*5qB3C@s;Cm`>?wm#9t6Z@}O){Z(Zg75wL`u)Uh*qdS@gQWph5> zpKyn{JKLQ;N?bk}J8O?Uosm@4Q5ZPh2xBf3v`A7(2OM;h5Hc!D&xNb9&h|9`=Li>1KVu zn*4i~zbV(4||8UKdVcA)YbKA&qO1)-*WF4L*GtS(|Rh*BzxCxC}|dU z{sM-Pbh#teX;dTS@lxN^MB{S%J5T8@&z~Wi=SpIFUT%h6?1uORg+1tzRSg)WM#Buv z+a9Gavu697(+W=%wvK`U^uEBXk3IW*7}!AfjfY9I4JeiOBI{F;m7;I@(-b~wzP98? zZQS29xaE%OB$zs zN)50|dgh0;2kBQlKy>`>@OS7phvE7RJP;<*q(jQ=V6=p&Zq3Cke0}ad$B>S!HvbtF0YBx!^b^u6@eQK|o z8Hm@5y0qTCTXI}azNX*!3Eosxveu-w-}z_zk;|c~66pzia=a=l$jjBgf@d zBbxf~nmAYpYBTvxgFL_jg`9EMT260=_wx3EtqAM~vxc-MbO%CGA*6#eqYi04+rfUN zNh_00R)tibI3uxbH7EJO8PAoZeG&$6Qcb+qJIsM2vF^lom?!v5D_EDEE+EfRavjNK zHC5-*uHHWtKb1LAF%KmYYQuLg2K+8-*x#@yFj;n!d;C&kaZg`$N`GQ=3DhH8RdlR$ z@nCt+s%S3)vzcequ}qdccS_;8yj$Y$x8)jI%?k#&cj4ELvpXLqJpP=uH@m?Y8M=i!M&M`mjOVZM|nE}`6PHN`2z4J^4u<&~I zQo5d+e#d3D7ZIhEv96Vo?<%HWO5eV*znfFKTLN$7ks-TC>zL}pAdIv&jz)1-?SgID z(f!^*(%HoB&}|R{?473*uNDsfSWI`k{1g4_Uj28DPJUacbXV>W^6}JY7%nj?%cafW z$K>JW$z#bV7YkdtnIp%$xexAGdNq|d1iSlXE?9hT>|#PUtojw$E}0B-n!1JFKxHwA z2}!H$$??duwDxYDV3^3O!nf?JVz)m(_#1EzXHSd4O8d=X{CXiBnw zmg1vN_cHZs=TFY+(@U%!?^q+*J(1r$^T+qa3a&7YWUB0VuAjTvJA)~C;IO>>Ua;mq z6Ky868ZxrLIb3^XF2NRu!@V1;U1rr+~t8jRudXDBhsYOhU+~Ok-j!nw&GK98j>>b-fA;HC3oU-a%@a%q3;J_F!>acZ{u-1!Fy zfPJ>R%!LDRhCdjhuutxm7Jm^C^_mKoMF6fpRuzv^zsXfQ&O}1 z4gEvPotiSwLM$`39`8j~4kkfB{0lYn%q-SFHiP<6TEGSF<*4%V>FS=(`->LNX`D^U z>zLb?u`ziq-)_@%Wy5^=(D>}wbA=KODWuR@67~KBJ*^W4e|v&a4RuZVHdDMiK;ddp z4|uF!`9FS(8|r*oRZ!OX)P47!6aAsg!0n4Ch6c*lkI9uW5b&Y_L&A%JDo^I(KHKb4 zAGka56!FhmSY#H@VLpdfGkrL=_*sa{5rCkwd``$;GWpc!gIV zv3*`juqz@}>thWYQr+j{`;4AHHS4@~mAyHSvc=(R9Ok3HX5{?akK6s3m96@?-C(`j zru2Ca{zt}cHTOh zS7j$8KPf2u!CIg0mj-K`p(pxdq}inNprL${;?Jd*VfRObByE56qv#s#lXFl#*Z8(wEb**XvrB$J7I4z4h||F;EmsmV_j~Z$BIWiCywgJ+ zs6xlp42hmVz>KvN_tFtrxuDG#Bx_grLVf$-NR_27VWPiZ z7!Jp@_T_!dr^ueThU$y9)F^v$8~(fq-8Hkv#9+;t%dwBpfq%y-*STE9 zi(fqjp&5k5$8EWu%EKRX>i6EqO&;G2bzOJLYjKv_`d$bBt-Z-vB|nTF_3?1pmmgB8 zyB_a~vJt>6Bol81WR4D}*;FUOBKnqMFQ^J$)&ZP(yJao`Kl4+W=d)5!(wQ*^aOF77 z)AZ#OrzQ2OTCAPeG3~PIy_Tw{<}dG>lWwXn($5d8{#*gRE0A>m&=f{>-jpZ#rh4SZ zQI>miKgcG(J}xV~UcSBx+@u}Es{OEesOY_Kd#;4vA(GU_#jo!K9HarY4Uc4CT%ayJ+FzFH_OdSF3oJ=yd*!l%LhXBcL$MB z>vrV-sZpuxN4tLAml^1$+~0-^f=m%#^R*NsUnoi5AdfQEwwe|w-$!ZO^-R8A#7^8{ zJ*geWEWNAyi=RRJy)v781wS0K%3V>AR`y*ua=!YCVTfWvr^um-PzB62zhlEY+G#KQ zPJbJyfj{`Y&~O2v5LP*||GD+Eix*}tB=*%62e^J3OVQt<@Z`na{5$jhAacw+iqrXC z`k?wq%ax8L-cSLYY?Ke6{K2-4f`r}j0;BACF^GHftVeg+-L@AT`09xzw#WrX51(u# z$XWw#Dwn^gJa6cMXjxVILUMr)PLH#fG}4R>dU9xa{O(dVblFNjosc!};>!2GTsrLkVb7V}F=Y<2z4xfU1V z+vd9wGrG6&$K!41tKLAA8W-{1aqN8U_2mWsT48Y8sk)5_f4?^zJN+%T>7#i&vN18)@x^z!%mKzvy`00{F?)7# z=l%t7c{hJ)$2GUnkMd?e&jg5-T(kulyS)##1zFN{yJosL%KUC##|o1EV1 zp}*u-j25&_qpCi%ht!{!*Q-oy@~=zWap_5>vFYe?_JdyWUvh;HJR>t-dw7#OKdx8? zK?hejp=mHhQnb6O%VJsIoSkgfTKeGFSjR=uaf~F6YvB>Mn*KZ>e}rkIQ>_<0}6<$o1A%bw8t}w%6Jy&oU);G1dfQR-Xl;!z;zV*V3L=iPY=}{iM5UATR2f zf5oJ+NvH+^4=wJh2LWOXi%~9F{!bXPms%?fxV}H!5&_NoxIrOzwBBFpKWnI6akOjq z#h!HNYU-#|tf_hI&-DvdH34Ypt#dI;yJt_`7}lFQ&~ra4BBtiw{8^Ks!(SJi-?57v zOXwwme1-76BlGCOV;ugQTZRcYRMrSWrv~g+w;Z`BW8_LU7Q7AhlpbreZJhKNn4SAL zBY3x9daP)K)7zsj8!i+jV(%V)N>{x1^}!)AN6EF>D~Y>L6gDqZB^GNOdHp)rDA@o7 zK<@JAWm#ok^E_f=WXJI!-Tfe12*bFqTE8sV4du@V0hFBp;N|57lkCN3GqOu!#IGfj z_wkL+8t$*Dj3F$#n2?q){vfdVF-M3&J)YN}>KoyS9p^7PKn6WTWv9WmDW~he%x*w; z=|e+)hj>lBTRpz6PFRT>t4;HCy&26nS?&YDNhgI6wE#$k;0r3GT!g-yS}4_pz^o*_ zxBRXmgiH4Cdu~LT`1*H!Kfjk!-p#5PrHtOTcCHc9tmB%CypN^ct~Tr3n|W|@)i93v=`A0;ht$= z*i-Q_r6nLiXxcItd(qRyZ_n6RZ9sRyi^4|KnATM7T((U z0r*8au;2;vTz{Y1&{~UW#bgS-e(PLXn_UF(0f%^|2R3;kB3LRio5aaHzHy^@mnJGP4IQ-Rg|>gW{(i(Wg4BZ+=HQ#pQ0xQJH$<^e!c8+HP5w|zx3 zLb)c>CElhG(77TEzaqR?Z%+4KZ?05pg}a5?m{1it{A%h)+Ro5DYiaZIL0&o2Qj94v zAa%a)n*CiBxXKRo=}pY~I;YU4>#yEosuvZo=zWpWo93z@w_+e0KRn@Ux5iQnVP?9b zWoAVs)e7{rVbxhiwX4P@MQ@~y_8t3#PS35v&Nn_`ja#lp2UlTRPvf_i);Bzv_)9W4 z&dJK}vx(t;J++7FYLDDMpe;9}K|u8-sU0i~I;-m+%;k2Jk@ns1S95Px_y7Edx40oM zItexSynW@!O2S^t4zur#e~DPhYl^i4rS@US=c+fib?QompeZVHZCJN5h$38*kLScd zeD{5N>lk}2XVU5VxGY{I_DHRSYmT;`4&?6jXSAzeUnZAk&hdiz7qlI~=Wl6;%HJRI zj8~qgOG$ff_aR3e|d#mrgtyf`kj@Z%3bM8-F&c$*;#F%<*>U;Z&8X` zt^qrG+Gw^Pm!2B5QS>~!Lpa~!d0A)p}lbnA34e*2#pEP!aeE(u6_Rhd9sVDbr zDfOq8BP{c|Gok1}>&029UU17>gF)R}kZ+BJoNEl90SguGbD;WxuyN+%uLks`l&jCh z5M{HCe+g*j2mi*!tEKN6e*g8ANjcuYKQ@q6LDhIzdvkZr^oh3tpAw|tT?aOM0i+)i zZ4Clh`JKnli+@ zN_lRy&+v-HpZKraot#-9tFaqs;&(B98qK%N8L$SU8Cj~2{CAdT=}SnRu9aaV9;W(h zlx2{Q(WhrVVZ^JN6RB2EzX65#$c*iR&e2tJ%#v{ZjpzkrML0683okKHa+W! z(wtFguH(~!=uw|$xgKjeJOox3>337CE#IQaQ-p!4h(OqR-iFvo^~Z7SI=*@~Ex3rlH#x*?W4idraCpVq~) zJTGe7wZBe?_sR~pS8D0BQFrK7wxDEB9{m;VkhgWAPeuO=@62|m#@C^2%${50_YRa5 zU~aSIEGM6+!M0~vjD{M;4q3c4h_qBO*b&jt_fenmm`e|y zKCfkcpCxqns7e?GXMX+u`&jaNOS0S8iA-m;H}Q3Mk1GdJv9uA_jM(nODKha@#ZQeV zf)Sg^B~o6fKfX1AArY*8ej0YG@US#e|gpAWW7$%R1#5#OwbwyiC=F zgCzfRPv~9{*7IKT;hR2-E&-9Fm7U@yqeJrjC%-+Qx60`2v}U;JeQSlPBl5Q@&X9sW z6b_%UyChV@oe6vTekl0ayD^?9_rxcvq}H~YgQ)TN@Bh#I)mIIUr}jh3q9SiVuOuvV z6wYZW-z6j&u*P3DEz#59-L+|cmt_$PbL5}MV!{R2jc)`S{lK`5(k%Dsg^H4}I|Jh@ zkQ~{e`J7zom4GQJeC}AduVGpBs}7pu9-02VJsz3X&*l$}ztP+O{mWmmoOgGJ`=uJk zwkuspP`lQ;h%708eB{W%zNx-ZUmt}AJO62=i-*6qTXrXWz!ZtBBUjk1#-|ik1a6B4 z4}w{oKwV?c&n}}Icb}*Z_d=zQd|aHBTk!ioA9&wpGY^V08y7P=KH9W55!3=s*KoO* zsh?twIhq^d(>l)M-)Z<`Et9fUAOBQPNwF%^gVb0xc*3Sln}$R3qU$^9C&I+>fZ_Yd zh2sUm`YJym+QV-%9(WgD6+`JxY)t0>V+bE|yw)0aVzcG??gIlL8XUlX>;L?29%H(* z>ZXc^f=5!~&!d&Go4!nz^#03tGrzt<-`lfbvAeyl>!XiKUB>TJna&T=H;Z^#&&9vl zH03ECe)@xl-A(psWnCir^?!2L1O9r(at8Bz`NPkZlbU6+=chD5h!curfMA$Wb%&!6 z6xH13)_=OAhggDJBQ~Px1Gj!3j6(gkd=$8Or|FX1`_DKXB6{ackHl7U4GrxdjIa8^vE=tEj0pPvmb=t zit@)hZcM$IN&nZto2`HT^Tqt>s@$4LAW4b$8xG`(6|G-@bs{h|ZR7VgXM@AO%D5l3 zi62Z>Vm$bEv<*O5`(a5+&u3AU*e;#M#BKmDoTUXT>G{(ulu&!O-jCNC*bd6ke~XCt zHi*HP-$LRf>ZKCHAcgPtMUxsUz-Vb~$oPBv&^y)PUQ2%+)+Ago|Q1jOCYwb5LhoZ6-_lcP1t8YO@LGul`MVCq6chisC zG|@_t>e&rD!YHY)kdaArCC|H6p2INVudHoNrbM&uY=7Tm5|#bvfr;lg5BC;r$NvjnBG~v^Tpc?>6O*v z&cfuC@s1B*UUi$^KBZLP(WOOK@*$|p@AkMxWsm+Wp~Q z&noA|pty0-wO?++WHjl)nXH?_NSm-HtJ(bc@taR(x`Y&S+lXm|`TiW*;yeX9m|9)P zmrR+m7z`)0eN>B;O9iP*vua%4Y88nQb})88RKDL}BkTzSop<}aIa>t1>=^Ay z<_4Zkw~K#ec4N4r`d^TNbn7P`nkw*L%gb5)e2yh}@+p&ISCn-6+SJc2HHxazdv{*F zlMJetXOyF_*b1v)L*i>`W6;+?Jmnl&OZ$L9ka9)yDsS%P8zHU@pM_iA^+~O~4s=qG zS?$>_8X#wicD0vdhh9DM)_(9CXPaUOE8b|)@nG?AiPS=5(b`QvIMss^bonnq77=vY zNfqkfdp4DWI^T(YYx;>0m|eMi=$+Xh?5EGyep(D?h7$05 z#+KOW+?9Y&!k(zygEY4>e7{Ye3dqr9Dt`>pYu>yh(KJ2B%X-^vxD)*!kO@kor{%-X zNRwZzs+MKdRt>_wC{MDU>Gv!w_)WicFjKAx2>A4Sr@W{)6rLdUPgb~3Z)RGv*?f3? zcRyYQ{-Ox2p%$lg<8y~Z1$Fbt>QV$y^+06k0L&L?2X3Gf!_GoBE~V4tfDh5 zesgsD?$3$Tr_d0;r3Dm)rG4h&&d`Nu0Gr1|KSKxJO-gD0sycN>aWb}}UG%dMIzK)m zLRHH_J$5BCwpOI-siX_wKzUU)pCaN=f+bMn9=Ym_0^|`xz+foCU0CN;KzRQB7-q`N7uz1k9?c+1um$!F`|5^q4lMSdt zCaXl}j%Amt3`UFk|}H z`Df>+K>go~KeEV+4ZdWXH@tCd^P844+FXXXEW1(mf~T~7Wc=~VLILmW-R(=#64zJ9 zgLgf;Qy-i%Hmsn1?e^)B*7-x&cMfT*Wn#3H57t$E<@EF%GwMAO%rH73mbjkarfN`ShmQo&J zJ6_F`ZbRx2X{Tt5yLuezL3c?mp8GB0=ftlSEAK(~F) zAfu&E*lq~gYSci>;R0Z3pmj;9>uty07TK>e-{!Z>xm<8H_xmz0;TA?J>q)JSRo0F* z)4E)Z)muuR*l1k5zIEF*UbfS)OUbpYZnw(y#^AId)2MrcywbHuv47(2Ir-P~$Ecsv zQ>tBnG7*EZqF@v3RpB}1AY;HXh469BTw)}YgP02oK-mj=H3!JhOIOh-0knkeY zYi+7I>`B6PKOh&&<6Yd3(9RMG3t(;6}vmnin_r z%<8ozYNPg13HrHEl7Xmkk%Ai>QS*fFetk34tY;|!q``Yx*#SUxEh(KymuIgfkCtIJ zRACjv0mTX~!Z4@e{^EQ#dwCBCwb{L4Pmb4}N0GFeS?9^c+%WUN>ZwufC$ZJV1zI!B zopr9kT7Fp;Ldb8|th#<+%HS8*Yc;lN{Ti)n2O4A&g_vhcYXP-4!vcyI`XAklL{oM> zV{c=6@c)^IT3qSU0_p{b#fFBDWy-R4?CyB}xWJ>LR#;v)c#*#y7T<<&eUE`g~fD7pcx_P`>MqM zbn1Hp)cUulBM18P`cUGt!^Ot2La_zd0Fc~wZ5Le_`(-jf-Txonh5d(jKv?501UX#@ zWyfh{tCb|whbXnD{k6G8aYH&gq)!O&4>Fq+Upl<_J|^s-5~K2jI=xRf@++r@ ze$lxrJZFvL3~E<%bz19}Sl5A0gh=5si=W7TAAeSPF?%x9s=q!slX+F7Dg!ZT^T^vV z8zI?&b8hSM@SESVKbiJNv(>^_vICx%I*_;*CfnZIm`wP&dWtu4{`P@>vm51%<>NJ_ zmvhEnSrzf4VG}v!D{tG!eFA1b3^q=Ox^SOa>;4$q_LwEFlvP;RRc`c=wNZZ9-Rk$S zP<=amQR*qx>jVoXs&J}A*ti9n0T=5toPkh2{aP>7jO|8}@j#IbCx3Sao%0i`?&~wu zea%*#ku~i*ttNV7x*`hdvE9_4di*z_Yh78I|6ljUYqJvmuAY=|DXERX!s`bYoZsJ{ z2dSx{KuU4Fpt!5NpZFr625!63=QZ!<-`T?TbqDI*vAcxBgGR@cFEm_YFT~`iI#%i> zGD`jg9Z*=N0F~-XH!MiKXYESA$hAG?MFLM*7p(Tr{2@9PPBs%*b@p>>L@OF*Wyg{I z_=j)$(uFHhLA6GLFY$a4-j}&__H3u#W8|{#d0EvGDNA8#6A0bEVB*Eeil9iJ9AfYl zd7$dXflV>ed`(=u@5SG%$PWgx67vZi>Y;Zu3a$$aCh012U;YK8c?gJXOP~M>TI+J8 z;)~c;Ou2ioo2_^nD9jY&{|1w8-0!vvNGrCI@5tq^#^asL=wJ2wx1Z6KaNBj8nspP@ zsgBV$CG5&KuN|05C5gG3HODr7F!*-?R8fE*)s(6ZFM>l|K!I}^BmR-e`1j+M0N!e~ z)QQP;UwwAueq5_HQD%9N!*6+Y+6O>QXAj!QmkPTVZ_S$u<>(Rr8R?|LqQWM@(kf77 z5P;453M*nDMYXjppR$V|8K2ld<$Dw}5Hb?fRw2)6p8U`&q+U&5c{xHh_eoLdOGwKz zVr^=B&<79GsG43NEjdq55{h!Z5Pz=zTD=0g5zxsD_7F4Y@n&Ru_qVnFviFYB{!ogzHHq;XED%66%l$g2f}LvK#{2W#94L>& zxX5~cOwTjkj9iX(;rT^3{qeDKL84Qr8)+g7rl0zZ_q(iZvS#qGS^7=o42A}=8 zQ^zs5w^I)K2ZwQaIiLoRwKXo8r28zkI%TAeFF3e*gpug>QYSG@2%vo#~TdwU+h-hAP5ELPc5~E6mTF9MCEDJWT#o0wgY~z-2usLQr&dj zOnBb!0SOT9@waa8$(@l^U-;{n>Fzf5A4Xrp@8`VpIuDh)E76`EIc1YOOh7(f`Y~*( zaSgH6!e9TL;QV|$$fU1>B%D6#A}mt|Upd2X=&GwZyp&}V5CpX8j}#;IkfBs1=&9>F z-9P1q3Ofh0l`5-&TP%QpAOy!Bprc1PiFPsTA>=Nj>-RyPBy9E4g7>10o-15!*L|wv z{UBJiV9(DANf~I;>n7Q8P@Vt5MfrboS})YXqaMFsJPalPrkbVejkMsVe zkFkcybe?l>{uq{{`T6*YU+{T%7uyrF6Qud=fVn<9{5PCCcO>ICW#g@|SV}q35q#>* z>bVa$bcr_ohSM!5H;T%K#TcPl{wM6d?A2Bmc_e=Io$sA(JT6jwQDWct#rD6FrP})3 zM<~WF1FdB#rktm9vSE%*I^^x=tEBdNmhs{L4|{JO4R!zik7qASD$16vM1|~ojFK&h zl09S{vSqKqP>K*j*%@WeZmiiuc4J?%@B2Pu8HVq3bl>;8`?H+${qy%b=R1dE8Lye^ zc`c9Yx*pH#`O@`3I||IeC4kYIU@+hy0stj1jIY2q^y@c&#*2YcRYYuMJ|>Fh z!p&WIFrkY#A6$XTdVb~AzInJuY$i0X<+8nywDIHCiO=mi-=L=gFwmW-3}`CH$~WOep@d#8`nQ znFY}$%&%a}!#)v?W%QYXl$Aqdz)v0_GtjRPfGnB?AO!dn07_B=^T7yv-1K%AzHEnL1 z9tBia_3@H-x59zrp+rTKR(POH%+N%{&?$1h6JmMxVscP%yl!)2ljoj0fU+zh^LVZU z@qWR#dk-6V_?trdV;Kxg)Yan}OaSQ4&Pc&6Y>0m!!g#kzo7-;LA>Lpxy-!VMf9`$! zjq%V}&&%x}UPX6tWKwVH4mxFNwIwJE|6HyV(q$_-fAwdZ?4@yY!1H#sw9bpp1XV|C zTM5F>c0HwOZKj`kGrn>a#ne_r`MB$~Ko3nS_liCKfC%Wj@gvRfo!dgGLl538IjUJC z=S34&82bB$+gJ&5yWxlPb+pFz&SSK)C^iiWtnqv9(H-wsUstLf8MHVnXj$wS78OT~ z-g1+TGH#@LUN5NcQ~Rb8KXF-2#+fy`g}Kqe5iS8UF}ikNnwgI57NeC^V>j0ZtYzHD zx#cBM0hsR20Ji*qEZ`LTynMj(=ER1*r$BfDD=yFOKH2$bB9sO-Ov!#|Y%W4-sjF)a z_Xjl&)TND-LZ>Y1y^Mc!i0Mv8eesVs4c2|R_`$v_t=jL$7ka)`1sb{hAjDWKe#2Ko zg8D{OVq(xCobiOm!%o1Celba3ZLRKLtzT9uCBLb<+MBd zR8y~Syk}K#U0n;6FWGtBAr8)=r5Z^}z>8DUrlD3>=)J)`Z~(BX_cC|sWI#U(N}t)3 zyb=xu2Eh>>Y|umbtLBF`4qD=1$}#%RkAcAGeAMyFxhJE` zL_2(Dobx5oeJzVFt4X4xFP)A~YN$6pKyFY!v;}?NM7sDe{yy9Wj$kq_RUL{|A*P0( z^UgoK>^y-{KaW0fo1vE;Ss zM5j(|IN}i(Z_=gRJZ0s!)R|Vj;@NA+^ae@YAhjQ3H}p5J7^*y#xYENbAX}j|N)8A~ z|2WobUyVM^Mkc>CoX2PP?LeVP4b6Op#Z_hAT!{U$5H50`sR$dX2WweQu^}zj)z_)a zDYz38v^~32WKcz&+o$c1`)tEj7=fU|Cr&+aInBMO?zUI>gL@ge??5SF0M$E78g`EM z6T_>o0`(zJ0`Dk9PhO`W7ZsFxdmOd=Aqb2CgFn09e)Cg@p39E}%|q``tv|?a$POlT7D{ z?QK_r3JDGRAv${Rbq;NPxFAXqa9k6J$%-ZU5sApvj}Cr+q1`BvB&K1aqAhXm4Z2O; zf2P=fUDlaAMMzsG+I)8AR~CI%WTl4TIyhixm_g)f;40nWZ8Bh>8Lj)OM>kVuy+X&R z+$e*hyQl3{FF2=!$&FJ3r>HJY9{+67y>&HHgR{bzk8#i9dlS0;L1Q#~^MHMJRfmmM zm@a~?ifPoH?pJu0mzifM7^yVV$*=iEWAPN!JmTiw;D8p`{QkZ#Z9`&nP3dK7e=&zS zFZK2J+?so636FTo#?vxqX#7KZI-PPO4Fx7W2;k<0eA%nU?8Bvz2$5V6>VV7)0o8!b z1r7NU;>uYvss>p2zSH3huP)+Rhgl?5NAsA9>37)HzN6qM-EjKe2nMTP#j@{4geUEI z_~kIpg7leAs(N;o;b%2>y5=Bpclp&WzY-;8*r$egQM>Z8U%wsF0~ablV+U zTCOiqtnZz6Q9pB5+d>;TGV_`GoX}wlwq@DomBJaPJn^~=JLy1ks7q>3(8}C>cmBip zae6=Aby}+D7U`dyw(?!Cxl>nYurr1{Ku~>eYtg=C>S&_U(3SNC=?fcR7<-e46>wU) z7y7yEvEk4&->jBa3OS&&whEp z-7b3FIqQ1^I*?Be+|Ny)nQiXz{4U}gR9;()ZX3n3Emm6Gw4B*HaE?sV%rl~%E_8A6 zrpxE%Xsoj$(*hs_KG~?tcBA@M$sS-O@$risCPQOH>Lc(d&G0$t(|zo-S;YI&*r8u9g?O_jq9#C`-JK> zsvBdj(${=FZ~6T4wz>K2bgNDd*;wb~aEN8rlcjhA?TQ5D*pHuepD%0=L@;9Rdgkx8 z5MmC6wj_@BtL|Hz3b0kCZ?aTUYjf7`PHm8REk9jbUC5+V zu@bsWet>V(xpu+yvrD>1nx5Jc3EGUAhn4b^fkywMR*ToYg5veo+xt24#!{VuC?5A) zN2ZOCNcsud<)3XExtbPQYo?#4_d6KN(uOMctdYx%gr#i@Qb9dKTaRa91RJnfx>lQ1 zR>7|JQ&0j6|2yrdvvk9sd1`J^R0Wz}YGpT!H9%#pZv4!(bdD_wQfT3c(C2YV>3H53 zsSaHk8myMrRrS)$l#X5We~{ZCYLMF=MU|SBqV^HPRp1_&DV4l>1`iXOf58NHUpkFd zU)Nf7PJmmVm&ro}<(UO*STizUJ_ia&pec|0>7Y|63cJZEb8W@=C&WqGD`{*yhNQ-N z-?pIkyXNvYvyT>6JtTC;Ir_7sA$>{^w&nTfMGp8$-@aO}2iIx^9<@no<~wB%)NZjF z$dHM6u@1S^d#6O#atLw0G`=CS*UMOx;KqyUUGP)n5t16qda%rH?Fsex=pthNsgJ;m zZ}ZpZQ#oxLG*q%3&0!=p+DUE}*hd;>uPub+!>aH?OV=V_1QjTQZNa#E8p*!)EP3(H;}1 zRiqKwW8S0RTrk(BTK$iDVY^QF^3qyyMO`=TrfXJ^646gYEq%IEr2^%wrojQ4@$r0_ znYq?ijSt)Hn{gHc53nazdcT79$zGDi0Eng0ixg*GSbv1<1R_$a#e}m1yx~{3N z^{6#t3tud3gkKV&7U1oW4sd=fjFMZf_ zi3|@R!Auqe1%3A`LyWRJ_I0F|Ve!YX+ZvVbxjT}LnQcs%j{e7y1_E$02BEEzs9_e9T@WLErFvj;ym+#-nMECOeG4lJpWUQ&sXdLdZ)GXQ>gHCN6|G+af}Lt?EPglc zWFy+)e{N5D>wwjI|G7bKmt?M6Jm%$^=_|x+%}v(iahDtcRtbcDwV-tZbz5t~tE<1d z^D81TX3{MNDvcqtLm??axy56666ikO`v|B}+cmo=k&qhh|2zY}%e%ifgnF82 z7@;;I94Y>Lx=-?#ry z1ONZk03Z9q@Fs4yk5shL5C3%k8Wk3#?BU{@ID*$U-tS?SmrC)s(fIq}9iGthexXp> zea9viffyHqKoC;S3!VBx(QV`Ag3-As6bkaW-2#EKKwdjNL)^3-{a}vE$;s*XHUx6) zUshID(vCv$sj8`k1bz4*OcR9AyeNs}G1*`taR2))z}K4Q)`f)yi6I+Td(QB(>CP{5 zY6--zbx$YfmA#-?BuJtu#E7{^LbX*__UeiH(&C zYc{JY>Uynp+DzKa)pcjMrw5sP?bpd|m+nnNV~PLkWpU>o#wH}lM73KSLz9TUfr;mj zK<*?Y(ogVSx!Czx9=)0blI8R1f2zgi6cUN7^E=CoMZpLw{1E(fQyPcvNkJUWaHS}+dKRKL>ZpHT zJL)!Wpr@;w!IX|N6NG%9izyTM3htP~*;-Q7_sOhD zgzU>^Bf33zN3~%~8EK-o`6*uahSM3o{DPhdSBqFHU$5$J7aM;nr0%t#K48w1#6c|wG-QNZo#z8ymUVH0nRiJjfrszq(fZuK!PB}p;F?&Y(Kgy24f zFtp`jPWhT3Rkzy1j?t6-PFYV^ClL&lMbDo&92`1Y?p7@r+djrepH&v5dt1})ma{*dBSdixRXfUn6+FeKnk^SgKN zx_zi+N+(}yoSwPYLPULAT~9}cpLlV8ZVqwUkD63dWI9>NjwuDH#k;E7@pU#;@gvJ0 zua-y7$;nx4h8#M+&ZQL}lOM@u+#`qfH)XVy9V;ODAg|ot`oK2szpe>83JEOA|8M0vmbw zv$>&7EAMDqZRG#8jLvhwePKoP&K}1Hh3brCqGWy?-u*(f> zxWFWBGrny==;8<8E+h}6lrumA3)!Rz|Ma%F7GM?DErOp@Tt{^Ba=|Rpdcbu%Nfvik%CMgJNe2AUDljCI`MmZb zXhHz16B_b|?|N%QY6C<~+kZASjvF1fefbf;y4FU>k%NKv(M0VcWUy~BAM6FSfunj! zfX+AsCzD)ybdtaSp`eLRrWO{QTy~TP`1q?MiUTXtP4L{&t@F*7yBawJXkB| zZXAUlta2rLgg}m7E~duXeQToj+N+~+93MYE+D-9eg_k;yd+vU&I>xO>9&JTd+w7G~ zGRz7g_CMT`^_r;Z71CNCyeqD$w6VVcXxn(;!G%x6rlFAYs{;H$`+TwgH@185gfzhmySUeTWOFa!fmgyD~*fQ=$PKp>_9 zRuhmmFEZ@-1OK#8q!7rSLLM6%{rdK!8;Ntx90jL56%`d1QC>&L$-zi>@1upTq?Bj; z*Qq*$q7eu_>Epv~rlYZkoMk{qXe&G4pKF@mcQ1ko;3xP2gb=#&@8EHN}wVsUH>u{RY z{-x};*#+b&PmsD62da;X)&>;_(R>vfcNfzjIIG1#qeGL$4ykn>&!cUyzWteSs&+{) z{B1$U_dDF~Q{Uc@UBv5v&dBDr>~WhA>JtlW;Y)Yr?pS$h+$IWg?9sgFthF}nEU8kB zTeVl=tJ<47hOUoHf0bN6^y*G6A2yO2b!Hmk{9+Jw^ZeKAcRswS_h(9~zoJKHdhY7k zyXVMg4K6Oi{L5ktn%h%1FR2y@2z(X2$(DZo<~a_!&*!eacg*&uzaw246*Ip}*1&2mgd(-3qffKa9O&@qYL!n z>}lhPgKf2;+bqR~{95jpDlK?nr{A#s@o}$9Lrq$wTShz%;w966PJ5CkF_OdI%-sA2 z%dm#5(W6Iq-n@BpxhevIK=cTqhKGlh6%>5uB&L4`U1burSrARk&CYiDZsBqNKDm{> z{d%9Py59zB#!NF(c_eN=U2yp&1T<>e2E$nRKC5^mgu;d_RGgeVHY;JRjM<=ng zx~eJxJD6XO#Vmhb?hH9-M-UQm&7heMm&G&Hs(ew|D8R!t`Fs?mMmqzu37T*g|OA2Eo;CAHTDz6g3Sd?01IV-+~o$`dgZQR$uPTwMAvV)$1_-N@55C zYZfG6Hq}y7GdPZ=j*q!;DP91kTHMeRf`c9*2xWT4)c=OE0`h$eqx|^k`sho}ro$L*^85tQd^Oc|i zIZFh?EHhA&lI-j%hut&u*le&Xsr&Jc;$leki^(@ra0~+)R6@Ot!Phzxq%%)1AUqru zk_0$*qHAiR%;8^X=-w%Gu+oUa)|v=HxBbpu77@FC#iXC_GLO@?=b6?K*?)<=`&}+9 zc|9dHwSUOZH##b?y~-;KkMjNBL}q3V^&Se3i*BJ=Cl_y+C(@mo+9P2H;U!w6Zp#50|vH z!Q9-~`TwH1TJ8PrEtFG!sS;gt;N)BFHS`kx6 z+w&+5R;h?m80OoRM1+WS|2UD)bOQ30lhb-07vl0|iS@}zQ{K9}Ukh014;L<*zI?eP z-QASsh1B;uBD+BNZ%mppbW;jLdmhwuc5U5AJn%M*D6%e7`3AloRFUG+5b$p=4V9GM z&*9|dCdB`s=T*pzQF}izVc8iR`G7ZJpv7wSC$<0f8Ylg)P$VClW*}x;_*^u z?t28nsDz*r`zG7p-#@vsGQO$cK@JEJK|w*`-B!nW-GL>{l-t^fmHQ}YwJncwL=wmI zjD4nDkwZ2VlTwnBO}!Z!7kBAsX`RcN!G)=NaSBPHPpdql!3zbmLUQ)?3mb#vg$Zyq z%_#kWFV$`f;Yhiw^C7SP2B0^r9hc9Y7g(lyjLw4V__6%Up7j?9v4-6mvV5(ALxolv zB?O*MPDoD6-Q8U-0Re$}OX24U`WhOh?rTeZnIgM>Z{L3W`0<<{bWNWK3!?b~Ep2is zAOb=jL<=@dy6=N-1~zhOiJ}1uuyt@yr+s;nFhBT_*Er!_&dPK2w;eQPYu?Vnzf>zz zGe_?u&Qn7b&c2U>2n#>&z)enW!-rmd8B8u;A%y*{(HVl#N$+(6P6uJFkB>KmAT=my z3iuq^9>bfFo?Q;*?=%FSC^RP?uD@F_FdcZ0buEiu6%*6mtl-wpadxlOb`iIuS}@mj zwvx4?jd-!Wr)!5=|0&ZZA(umhIm&wrabOFr#*KY}}bbV0tlh^->^+S2`H+_=i z7k|0tr9|C+wAhEZzkSTmTSc)KgFeFsy>ayyF zP<0JTuNHMk2$B2fj!ocTPh6)OF1FPm?rdmiILqob(-dk)1k;UBmY07wm*vWRbaeF4 z=wNHMm4luA&UReBX=7Me*o6k$a<_G@+q(eX&3?Q+bM|;{FyB`yXkne8wPrikL!Z(P05_ms3e2&gwe?K@a0d}(kq|G>Kch}~XA*4!JcsN8;%)6pi;LOu5pD)FwH&PQJ$4DVwWlF;R-b1(UU;Aqq&fKbLY|hss7c2{JG<@P+?)=#~(lL7J#^Xtfi%pV_13Lv%DY}dtgE7?=HaqvKvW2#FfK; z_Cq_1Fz0lBXP0hQU89~aCZyx^8LA@U`;!^FN>`Cb?Y9z-Une8HS#XSGiAKvs6VS83 zcD#GhM9!_mtIC^@6`S{~K`Xr#KTg9h z0k4B!Is;tq_7!6EsXcnkZnjWjhE7A&Y{!7^QHzX-BR-UDg_uEC+fDdU`AGTqn_lY45n0s;qqrBkN!3c4MNc?T*jS zI(ZOM0i-3uVlI8rY{wflD<9@hU)BXIq-D|K)YNI$T7>?o6`tJhD#P@+5vHs1Fzm% zVbwSm^I3;PP&UWJ_ue2d#D|oSz_%=ily?j-5yDP+Jg_Ikeo5ltJn)tgb}shPmSsIS zzIS?|D|eAh8g0FS1ei1l4V@T6mE`JW2`qBcM+XSXp!ZQ2 ziW^sO^YQIYK%dJZ4ac2!*_@n#{D)a;6HvAbU$G2tJlIZFNsL|?KnJAb82@@|1UuA) zx~-M-)IUi{GBGL!2%3yjK^x)88Uoal~&&uTVmk>F^eR53Iv_ z8gHGSuYkxOJY`vrnkxKYrx&szyz_R|@8GJ{?+1bEuq1HOE4A^6c8hHbMaX7iCHrf8 z2QOrW=h=lM zD29eTQL&oTOg9cr1t;AQI!ATuFfK0cc5_!K$1f)8&f2cflw+95&LO_8B@0rj5y zS-I@l@mvIt?i<9bg(#2~+Zk0YoE)u>jTww_NkVs7fS+i`^kcWFMH!!;i2UT`E)ZG$ zL}b`mU_%1;m+J!)&{GhiN5;PZqb{L5`e1*F@XY7WpFtd46E%eIo2P}07MKUFI%H~P zZ#5qrpnKGs3(Q+>;y}~I<+Tek2uLkg%^HK5XQo3SoC7%q+htbP*484bt{#UPd^^;L zgJ()l`iaPUg75EH4ZNdW0Q!rThJfM{53Xk~i_4yTGD{KzA0EN(gN#43w>fRLyEY0~ zWEu8xk8LX8jW)kSEtfzVD>DD=sMZK*0yz(DgG@jl9eUufE{5|4Fslw85h7r%Y` zmVTMXxEmjVw~#{~eMhSMj?Agq`Xv{X!)RAV%LgV+r>CZDLImh3WnOu$S07hRRnE@N zzQU}Fh>3-RZ1}v`pF`QK>#O_>NDne1*)UH|gNLXkta z$B(GYEiAC;g?Cr$d_h?Rn%q%_;{5!s4BhPHs||gkm2Sg(qD{;_V}e!hg#ViN&sC*P>H7q=Z4VX%9*95 zrT1NyE5$OP_}X*!c)2Y^{OH_?U$PLv(AJ|q`(#ESM%#iB0>|D6`K|5daISJp0Z`i3 zOtXyijc5r77Z=xhzL!wd?XRnCd0`x(fZLIC^ELSHoH1n&S{pw_JVoj$sQ~(hZwSxZ zx}NJbdvD298LpDv>tJOxyU#3+k8auonA!mLK`{cphrWIT|AwtCElsS)dy1e`kfS8^ zv54Jsd^!Q$8!~vn8Ezr9cW_{B7Yhgs;4~{I2%r!GoTg~4_RuF6M5aX}D$1({^>uIm zSCKUb2gkk55N!s1rJyl?AKT>)Jq={xg z11cGM;amzp+woXKqbC#Dk^8jlE~tqg0-Gx`GBFtl+5krF#;seNz_H+n{tx&%@#`SI zIL)oX2H*?m%MxE-QdcEB6%`S&im|P(Idkl}P9$XF_g8r@zBk!MaQ-|J-i7H2X@t)# zCnawPNx>t`qwDMI#YZk_c}U#CTPYX;BJH9Vm!xD&VNp>i=xRT{c|+JG%Mh&p z)9-9ug3rkp+8~$f@F@{FeG)aHhaOn8Rl~S$-m}-MlH|FSJ26vLA3jhbg3k-VA3l7T zV0O$2z-6k3sw>|3n(`H0UENy@#b3Yvm^@FvxYNY|g(j4gl!SvT+MPg*D2mz)c4iv` zdE#>z5Mg0=e+>`27cuv;d)V3KOu-M5(nx)3C)G?U{ITSXp-EmiKe!L4+1tN+{i7}h zb|2PwQ%VZt$nbto$RcDD^XTzjG&7`Cu$)Q|1X20YJT>G{eRG6a%Juph;`Xb8sUr|p zKTzlDeNU*t3$NI`e5c%{AgL<_fQLKTd>YS;qSV&cgnI0K+>2i=bX27GdR9aP@|2Xwcd zrk7TK{R-+eHqqZ^MB>fZ_OP0>HOi~>#$Gf##Pn#gcmnEsV#xyII#T>+yLfM3-wnea zlYDgsu3zvfKuWRZd(rGD3Rp6@0+=SJc@V|_MX?U-7Fd6nUSC%;F)_Jp_@wS=;p_Je zOn=@$t`XPy^XFp@`i#Vt!<6*&^g@P>F-v^`^Bzb0^8p8dExx#`t)+!^j=OpDCatgM zl>Z{#ho74&>nw*awQ4$rdp7bTjM*RQQV#~-q(U4Bv9ex!hhW%3UhRbz?!j#Eks1y! z-U9WxWrx|`HsY8PB6p7It}L~8NS8iHb3vUwcRdpBqKl+S(?6`xY=WJ8NZ# z%*tw;pRRv{h!$RXO+?nDZ(o^GY6}YfQASZIDgDj8`ucmrYP|TCbzh@oyDQOaV+C=g6f=gS_j@Fh72IiR07U8j<6Y7zFr(49R%9?|Y!lZeiZ@C#f zpKI%euYz=z2KY~~6^W5HIU;~;uR$>`v{@C~9a1u^SG5igD?feuM6qN4wYWIqOdX&` zR5zUpL7&_WoVGV_36TSG8TXNbzJBb~`*2K|5qv*o>Xky$qwOOQuk)RGOR_kSXanYF zXJ)Ric>((C`{>!`=H^j2DdvK>bY(&ssNyu~MFkw@5JZGi5llhTk%0UPyC5ma4cpUc z{+;OPJUe|}it5A*ihhMmf{(=)sUm6zsRZ9&$yYRd) z?YKUtYo{L-wdRc&0TV_b)=g$&{5|eZ7v3P^30E7Mn27A1F=Q`$89Nm)4 zY0OKd+$4P8H;s$oF$Go&{2Okc8W^Y+6ck)$c@ZOM9&iiSR%mRl%c+t2;1T^5Nl87^ z!9Y5`n+Xt?r6+z~ia4P}HJSalcv}Yyk9lmXwr_83ZCz&6@}$k=is04nUv+)Cm=Z}F zvTj9%4UxgV|87j=!<9pRIx4RM;owp1b+6n03QNa_6$sB&p1(i9ier-IdL~YFAh?m9;PP!ZolqiCvlyHtx=fi*Y*f!;QZIb$hds zl{=y>lw;gT<24Dy3*l1N-`guL;>{I&eRZr--$5S~A9JtpT!^I(rbFgWDz4(ozooGQL0~|Dqvaq8#ii?Z_O_a$MR?5A&NVD@02>T;B|g}BE|6Oujo1rD z)2>%1CHWB4xrz+K0sq7**BL<^uf9;8Cr~>&fU`hOTTL^AI#}&*F3uZs$IxQr3`Rm<*zb1OJ-6h9{O$I$MN2Dh|4_m7Ue;owviQl zEm1TJOGqkh&?hb+5Q=9C4J*ItLh!qc#RH>AXvo~U9#uKWEe)#{$RUdsP;?3cH*LTH zfE=~+vmVH~A(Pf+$q~tC#96=-KHV5%eRL@(znk85VrE{7iV2xDKA{}Z*|cx^PfC29 zG)F3buMPXo$6}ai`GbRoH)aZ|$bYhEC$YD0-(F8ZT6SG=oT@+L=YhivYzPtOvlF8w zYr48csa_fP`n>3W(xXH196}rJ#0h%u0}VJup9A3)&sxRGQoclsG#P2 zsI-%hh>prb4fx2&7Lr=Qjc*Tp1moX;8p2gBFtH}Oy zod-u!-tcyblc>c0)%t+e(9y}s-9tL5OX-%u_j~49mIu*i>g+`WZw;>wqx z7!5*y|Z3Up)UmE>3^w zyG!O=qW4AQ&;w9grGKEIfLr{?8%xZ|30@hmMpOU~n*kGD93c3X`{c!epiK$V?n;J+ zEM;KlN5u;Xb}MOVX+H)riG@u~nRNH|&h$eqt*qF=n2{O4pX98p4^Jy$pRukkV=vU6 zGSJf>4Bs&oWMTSmI~zJtvttkIVyH@}4t%N@iIpmzjSLiQC%VZUzl9_Hy^t#I)o14dR7?3ZgpD3Q~e1HjeELVtHu^FL4uyGW>3| zFkMw&65^c*&&FA4hMm`nJkz9#9q!^L&aitg%;wl)jr~x#=$5>Ms>darB=j!<-T9ul z4ivpHccuFXmb{}~7uuPE39*NV9u17eIo{q0M{b096en-^loW6`cZN~?JTLrNsI322 zq7aaSVZu1BFxq1|+GjQ7fSNhu3wRdccd&0n=Q`cF64#d>i=AFr^x`^I&OZXW@EUVG zDOHH0tRswk(U3=^s7DF`Y*Vg?%^_q|bY)lFn?M!+7nFEMAjdo*V@uq9C=0g5)s~i* zrAr~8m_7C1Bpjo6UAP|DOi~B?HmN{BdW#4`6#84w>1mRu2r738u`Mn4sn0ykmx?NWb=+J>zb=T7GW~zJ*t9YD)eA3ttKfSevd2(Lc}RKh zO)qY&SNywhG#MXdULXPJZ_|63-Ycd3R{F&G^^}O|%t~3e0-&FrBZ)oc;)2 z^DhrewAG<~Hlh44>!2}|*t|}*E6QO(hqk#l){YOG&NkoxiQ?(?Qgbt^+K5Mc#GaqN zsU+xPCH8<9L3=&n;2<($2V3c-;S}{jD5$dnq8!%LO)0dWGs{5;Tc^N~lv37>Q~%pB z3aJ7KDQ};_dx|lr09C&vEO}RvqES5eHHe2>3&Jk$+IwxkQr*L~eHCsny}3R9XapkS zYLj~mHv&6xOFl6^6wl|{K5&)9aG&pV*Z#j>o=75OGfg!>T?%$DYU-$AQAniLtu5+$ zgX!2yyU?kl_S~!um~6Gjoq)M4PmL#0bCoQ^$nyl1Ztt*!PkkrX$qp(12>vs_eB}v9 z6NvFlZXO`kQq|R5WPyP8Ox$4ZL;dY9uX1&Z^6^ItwxAJcZF~^KoLO3(@f^aQtvO>M zM^g64?ZZgAl){mo*N>e&U{&R#;(?}i-vAjBG+3B_bv0sD;^+uIgN;miTc{Np;HgiW z8<{{G;yqUWf9=?bfVs5FzS1-G^Typ(@Cr6}N|VXwvX67PhfUq95JuWMEEmiQN9_dE zG)Le|6Rv`y->uvC?4O@?W9l4W8BdXTnP(v%!THwc;LxODo`L4#1R;|;7(K+Qp)CdC;bi&4U$Z8@(n$NTyZ{sp*WA513x zw=U-Hw5axC4zOKKMe1{MCXO+*v)njvT9EOetYPR~ZRmWN}%b)1>7U_M&Hy;w7{$9P^y&&Ptaf=WI z?QWm;q4MUdewk_=hvVovupzRW;H+)reZYT+^G5u}EXxB1Uy@sQhV!o1)P(;7VxQKQ ztd<)!llf&HYM0VYpZtWL8DksJLny0hwFJz^;_~qqj3WG%hZb@inyu%=;wAyT566hj)4Jd>4U&?SY z9c)>$2rI7);L+2+&r^2o2mbrGjPiS)?#SnV#XqW32lnX`pZ@&-D2F<=RfY0?K}s@a z`sUJbtQRX2?${WL|Aj(<^c7BR+{o=ag71k=B&F|`1WketWA||790B=h2i7r zpV73r8D|&f(>TyCcdB9lmUKI;y`Pe>5=yh|s4uBD^3*zBD><%dh3E-}*ZD~WVixFk~D>18F{?|(0 zW(_&S#EB2zdCrHEQ~K7?YStdy6OV8?P+l62m#+%|xml_PwWbjfu1Y>Vt)$su;+XOb zvOmnFdAA$BcP^tv_W}O*x(`J$s~ypJv?$F;OCThuTqeMc^F>(qdz2hW33D{IL^@b~hp#qnZc;;Tx^9_Jxw ze>R3rF`Vfwk%HoDpsCaZ7&$RIhYX!ubpL$IaJZ;bz^VX%XQw^-q%Nav7?{_xkdsFc zV&CVTbIdI3-igb$&w4y!^xfTON`x*A^<-omYsC)m-#-@nW?xteM90KWlsEGPbQdr4 ziIMe~$bU(>8yVv&3QME{eJ_4FMdS+1-1tkqb8P5HJIQu zr4~!OTe&%MrNKJbKOk=u&@v!@{i+W19y6Uvt->n}09om#&@-~VAy0mU1=%n7+>boV z>w~Y0jpacBgO~b_TLuUa z^fJqt1+U}B^O8HSn2fYW?D*+z=zMUBJ&N@xl$yg%6-d$`1tfoakvaA1hJUy_X6St9 zuI@-L9p2?Yk`aTGuAG9DQ_uCR$PS~P(re{X$;06?H~9FtczJny8@(_7<8DaKzCDq>4q*Oyh(%T!YO_=pBf3!9Wp3G7Uker9rMyv^9@HQ*R_PyX%AwOB;AOGMR4 zF9|nuY@=_Gw@EQHWV=LAsehu5Ah;?AyQu8VOB`W#(&}+Y@Yu@}}1^l~y z2>j<=A&0g4(mA}xk01I@3FKz>dzXZLAzxt5COMu*kzWka-E|C}I;y`R5O84n^WM-} z`BTr2_d-rP(RZNKl);oaSo1(VK8zS~F-revJsqXxV5}odyPsTI8if_ld!}=qod3i* zI@J?pz4c5W6}+Gf~R$aA>4UzE^r^T+P^A}Yxk$$J6y z^b`+wUBli-U^XVXs7UiP`f{f`Au-zXk{E_$+X`)cWnVW!0AWFL;=~G`#Q_xH{?*>b z@;7Tf_qF%Bp{soRQV9XD3Dc&7%?p5dX$fCT2&B0oTsy?47e1MbeJ^I5${m7-={Tx1#W1He`y-iqadW)?MnY!z&RNUr>Hyvx9KcX72_cLaeR{!@ z({71ku4iH3-j)z>>fgTox0d;{lt>08t2F1$mkhT>xjJ0Wso0&-EiC?ob@fA368*9G z*WN4ypiD>n#uTi`dEAG6VbA|A<#HbiAj;ti^S{k?8jQ{!JOE)<*#YSU-PRTF{lY=` zPmkX#%p4%3tP21n*b?;D@c+JyK*M>~1<9x)WCL} zUcHjAva`9RP?XFi76=m0jfU~1-AIpM1Yy0e6j|rdz|Bf*VeoAgb0$;>- zwAIZM#QD?L|GAj|_q9Nse_UI{wzznOZQDqivW{HOyGwmWh=v>_Bh%5^Vpf@Zp3rsM z9NLxj!-_K+kAP`d|3}lHg=}+;WvC$Tgl%b$5ZO~d@~n#y91$ZI!pf(GB^-)f%JCcL&#N4mB^3WBnKJm=t}v6Nx9_b zix>2p-ht2^`3o@qvz*_7FM#;zp~vx$KC`pziZ5QP%;Q}@MagYRpdfcbI{oZUI_tw0 zdgG-#4IuRo|JRoSlSX(y+rWc}R)kb+fXJ{~*gN~nDOJc_&en0sg}i8Q-bxEP%}Q00QbgkYCN|bQ0|2iqxo&9 zX)j)U1S$b~PzikSB_z=~k(K*TS;1&PwGRKC^MXb<6BMNP=H96^^w19HJYPOt;x5X( z8d93gSLn`#wtfJ|XU(~uD7C;Ds7RspwIv}!Smbp+zBD^Fqq-K3fRlV-dA(EpJ$ODH z1;|tZPP=6*qb^fW*f7lMPRxrkH37P@x6Z!5Ot&O5ZrDcMrrcDDu5=W+Xz5jN>v^YI zff^LFBYPlB+S25=kc_=h95z~|PE=S}UF>5nKGdp4b@{VSlh9L^n=u{J%}Dsf?SeRrdHXr75`rsjxxWNpZ~5HeLU+Wt zi8Lu1u+aShm)jd2GP%B+d;lXG$r{c)GhaQ(7Fw`-;quyg9>nox&N2m7MDlk*6s2Y= z!+Y{FS!xg`YaLlKQ0{&?rSF!9heIiMd#5QIGJLS@=dvisv8RIs$-e?U{q@FZpGGAq z_z#o%%A;goVmkLnEQrdS`Cm6d`@{8vuJzVA=m9Td--^b}bk`EM7l`%;)=5gbvk-We zbQ^dQtN~8MhY25T`-S+l2dWIfWr21Uy!T%aue282GXrYJB`Ht)M=pk(Wl|4m@VI=` ze@B@14VEE~=!7f-ja8pD7^e-6{7xV}ZDSP+__D`(dI7jlr^TGAQ@h4{2emulnebG;~_QV{pa9z%~G z-@aOFe4T@QwlyNGKO{5V-u^6}O>5AU#t?(CBJ(n##YUp#7Oh#-T}Bm%a{rq!oGe8-LdD9^!ig}WT&n8U$k64 z#PQZi_FJhrQM$f#(7rGZfaNHtuJ8)EMQM#Y)5FMo_|~d@6~UhkQY_XL+67ecOr(&n z?GD9BI-)d}7y_+h@RL~{PHpm?@M_P;*Em-Fs6)TSHQ;~$Y1T|*d@T-4?OC2wsOLIc zlfe!U2=-8-0|;BA>n%1-eZ`L`QKk&Pslt65WatxH<~zZwPdiQd@sb7v1SlyglIv27<$GG1g*ZmOV9<5wJPG_Niu|O{43VI(X5wr-AXFGdzgK#a}q5<2cJhvbT z?iwuWz>DWexW7gRM?72QSGu*H=neQ&;vMCQm;Y=RK+sY%?3r9U>kR*=rXe@A&MHgp z_HDB5)eACSc=if%UN-C8dC4(5?(?0eG4r%dtd9ir5pc~Vm+%0;`baQbBb^80d=Gt~&1Qh7{JV)cN3KRp7!i$N3p znCeFv${@1uW;i~sl=)|8Q9=)e_avxXIWyv*y@CaA?LUh7L17TjP9H8sEK?J1$^eyo zGfHP2uIYNG%p_ar!$c{sex9K+R|d~} zv$^RKgV>xQrZnF;%9}rH6iCk+g&{!J%K_@ibI@i{B!!=g>oU04;1se|oR|CCB2Ya! zLI&-9Eg(qFY7`!Nf}c5tvqR~tl^=Z2zdDp2xfBt7t+oy{f@F7)$J^UG5O(I`M-)AMY5FQe? zqdbnXkJUQu79c$0Xhjj>*rNB==c3D0)5Qr6+O*8dA6=*;p$dye#ku!C;RSFE z@PBVkt)Ui3fHOR+r_4Gx3#>Kp^+pf06=2xqv}XW&3w7OpT0OAEK!P7@9Kr@>7ND|` zBY5Lc`V%Fr)8jRSD6M<;EWi$eg`VaoNl5A(j*sEF+Ltq#l5vEk=MpZxom+ z^HFJCnV&>Hn@f^*WteU3ezE{^x<~l)4EpJ^@yUgR;G94$!VJIjEAvp(i*uqiLFx6&9^+_mwb>skS&w zf^uh|kBgS1y!GGv3VH=wXAi(JfAv64p#wP znG+5?obH=Tv8sjl#T>)xAY;y9tqJGJ4ccXt+v)6%}_0vvVhtC z3 zrVUIT3A&=k!#6({%}|;PQ$k~a(LcL0Kk#t8r$nfvfd1pb6^po8(0_n7%#J0+hSbLY zp;?S>>Dj9PhYyzl z{`vc*QZW8%1pfIjF#Xs3zP4ac508K{ z;KnPQF)|%8PP6iNrM||Brwjr3Ja9&!f*jY`SsrRZ!0;HTlOco(0%-sm6vXxyS~b@L z6(m9o7*t8sp>yd)d07% zOD>;p+?~2pdtvqObpKb8n>1tz!b=VIAAlL@iMpjlgL#U8)Rfo)Co7^Km2`&!n54cF zGj1W2M;8CeBM_D3Q{y@ZHKJ?S2LoT)FtQ%A{Ug#B=LqFwwa)6D-^2^(Bxr(CeDQz_b=uGwv$)>TdR%z+iIX(k?gq+ zkOzGYD4R0Y{*GSXZ4iFX0vONw#9hofR@EQNLM@?fN{T_%lszXuj~;FI9`mN})lI#v zE{6T6iH`iC>&gb$;LcXSaZ-g>O!}6j+E+#MYpt*=d*suwr<|03lBalhvML|9>I+!u z`wR?3crE4e^pd)d;5vi8@`ky9HYOOmegrznu~~>U-5)c(BFeyZaIvQ??4|n>)p^W1 z{+83y>o*nickA#M#}Hihs~p5kW^7%kO4jY~>5=up2QBZ0FsMR39ZykRhDQuNAvko5 z1zyK7TWyTeN-m^5fM^g`eJ`#^F&zf@f1W95b>;K8bTy`;+#>!f<3;g7&mI>HG0~H= z-h}9huceibLRW8nDs+_`e^4-hvxS5YMXi2Dk%O54tDcgWNQ41KJ~~mI$r{7xe?T6KggR)a*F zMMy`!L2_*tEa{yKk$KW#A-jIg1`}=8%{;_oK?!c9gXr-cC6tZGK1Hbe&x(-$Ac1)3 zL%p?)jc@u~)IWvWilFA?57y7n!`yC^CANG32y==)Lq+zsSDz-_%O7Vo`yB%Zj~6OR zHjxVX3aDAQu+I&^o)7=@^OZBa{B*Z#zOEP#y|i!@7in5qQ7Or^(?DKSAlTsGTfAC6 zRabO_Z20|$ri;&h?bvro5-eo3hOsT5h=zDqvc-6hWt#;0u5P^)u^$b85qhhZ56d(2 z?*`)!jr*s`JV0K23B|U5Q#sdDWUB4!EPYTE@X;If_TqL%G2`Hh_fgq#P_tkCcE5bf z>TR;rP{Q{T1a38i3m%RKX5rpSx@TNC;_&B-iF*k-_H}!kb?@8vU4FV;pSrGlwE&Y; zv^yQ5on>U$f|<_R^hfSeh1`CSEv;~eFfRg%HGxnF`{m}G8y02AGX zx2kCBMbV=i6#T}hD+&~Yl&9(bl%e`f1OJOb`^^ELF*3`iv?e%|A&skM%UVG7K z_v);dsa^UdXpl3y{qvB#C*&E)WH#P`@b9!k_E=Z zGIUm#JplY0?8Q~VY?ufA*WAGSIH=IuH_UOjv#gK`A&Qhxg*lF?!CX&gu}W6*2}7_hl_oEvgRvQhNFi+sy2--CfU zlvQas+&Xp=%tSyO_&QC8msH0=-Bg1t$Z?lI`KA(a1B6T#P>#r5Zzz`E7!O~KcKJlI z&~`&{fjtFcxkDXiY#sMwT&_`uKSft5UNGZUpz48-WfLz94iV`8i{}9{vQcoglve>0}KAK zCZ+)`cHap9&tkpaLsK{1Gp3&e?vgECPa2;u*=>3tTc`)d0}TE~wSLcloRO|$`kn*s z17`e`wRPvAuKaZBAgj1~1o}sZ4oN>i86K&`V50Nq)X47%lf&UMVCGsm;FXdK82=hW z(?}oOD*5u-GvAHg|5taSWV?M~of1F<=E(EH@t02&x*vue&skjP+0{+R_JowJbA$AcL4yMOR5qG1f*9U=(LQnBvm0P#bSJ(Kmu>!jz_|sBVWz8~2R`k{pX1W@V1q9iIpz6I z+xtE@S?8Zn>H2sHLmZ5E*ojqzgNlxli*(LWN*|gh_}yOr_0Xtn#5wSF@3plF)&Ab6 zRr6{$nC{^D?8gM?aycL9da@t{i-5@I?^lli+8tyILtnOqQt*#fp8oF5$3S5^AzPoB zkr51t5&#+hUUZ6dGpOxi0TOZ!%d=cocFN{A+HgR3`r_G$gKeRhXJzr_;PW9CR5Vf) z;0-pdd4LT4f+LDTi~-CS=D$X% z0mTG{nmM4^>N5@x2SpU%m&ALvr(mr4|8Ia0T+jap*zg|%z1~?sRKxZJ4@ezlJ5zpU z2O>}eCG70oM;|+iDD;YNff4&2BWTq3)#>c`ztINrSG)FLfD)*EJjWsQt?vg{H_PA@ zS(=F->MbzdCrG%dSqyU|#ZC?6%VK;QwDcP#wyrCBgaF%50#vPfvpSu4}mqPr!2%l zm8~CeWBmh#40YTcxt})XFVGs8zX3+UeTl`MZTJGc13qBnx8{uq4|Wa?8T%9Va;FJ3 zpJt5f&~=BVqHq}|`1o6k<_|(j{al|*?ss{DQSdrUXkfPzLd|N)d(3!ZQ_hL-Q)#6! zm2Ng=%-DrG>Xo?#{r;#P=w>J}in{f5v>k|4+z0?yc=rLLgpNVK|JvQ130df#jSV-E zq~*X56Z&RpOFA172XMt8<OhYXEhLsOO%6@fQXhF|v9=tQRKI8nrtpgg*Kiz57^NPS z4HiNAFh%t4kRr<;ffRa3DL|9x(8rl_`s-H}nLRqx=?b7rcN$OGQW^lg(fu)?>zRqg z0Ja{|ZkQ@__m}bafnCh8O1>?gKr$Tn3W6QuBfNt#xYVuK7?j^ zQih;;Tm6l#T{g~Ra2-5?dZZz)v-4&j5~<{K1@pSCz5NxSi9*mE#w5gPW-$!(->~so ztS=lOo?10)Y$*{l+tVRsB(8k9Km6SlKKB#L8;pE=Vnh2*acVW*Q>%dsmxDnG(DXt^ z;Q1TUB}u^VQ<+-(W^}F2zeVJ6#9UyvjlTWK^wmJ`$fytTf*{ihfRRt%Q}1CP^qzXB z@40#Qo=wfo>#7Ofdm3J4Y(Kfw(VQrRMKGMYx+tHJem3E z=Xbv2kU0Jw*^X^%TJs0qQx&se@Ybxt1LU+!NuJ@t*OsS%X5cSHFis@r0ezGhPz~vQ zH-zPB;9grwus~;{p7UdV9?+oLukhz*Q`?|>iVGJs55Nr$6af?m)P0ZZSyg4;r0+3z z$69FWi0VMPa-!5J>W+^2neA&*@dIT{IyNPYB`fu*=T%eRZnfQzN}wBo>nrKM;dGn+ zBgRxi^d7~Zn6#w}UIp`r2!*K(!OWWvz|cTq>ceg1B`9n>_Q-Sx6OMp8`~m^R2awZY z&OaOgmcsDGs}2y6ffsJItDam|y&pK@;8Y5?8W|bcTMP*Uk{|uX?N8=94H*3!Da>@~ z%eO*Xca=IlEmiIuTp@|Sri4Qb_f)YRNH}b6o=0nY+U_){r;+<|sV>UwxSYMxW6aZ; z*G*4I7A&U_{R;5+veHw}ODYmK_LZc+yTh{^Y=tUvcAF$o0JKGHH?89?5&`qgnn-m6pRLSbM*2LGh z`b@N7)~1ENk^0H1@a!QAH4P26hOqa#8L;?)eOHo(o~iT@So}uzcEEY)Ry`4Y^a(h$5Uork<ilG02 z*Yb<_c>1N;PygWkD~}8%rxF%_mATgnMje#z%spAS3Y0%l_Mguu?0|#>GQzK-C7ZEe z3JIE$F+I?+dVLadGnP>Wu$hluC+TPwXF+2Cm7&Yj{9w|b~RO^fqFbin= zEz;)0po_>3pP!?qdv2+lgPUij&_m{_UR#|cCtB3?I!L0nbhNr8uw<4gAg`YX&S|)5 zAUg%8 z-92POaL~=C=5w8KGy~2Ol9CKG+N80gqlR7dH^JlsE4bCJcZ_Xmwl2K#BW6H35-R{+ zCuaVme>PucxAG)a{Onr)g``8~Z-nf_1-2_)SVWNY&bbgs1`B#M{tgrM+#Vv`r$h^| zveMTat=O1#hqI}t`+TXG=LVE+_u~8=Nl=S6lFjbu6$K!+vqrH=mzeijU}&<}jqYDMrUH#n219a# zo7+6(SkQ<`dlqILPrpp$q=D>-wlR+ZxW4)t+a_lCMjl9r-1jx zo04S%SuSw+We9Pdee2OB;RU%2>Oi}T2hulvJoSc7OIjRFsLaeNBow(aTJ{nsRo$MN zPH~15t4wO#-+}-ha{qO>)=}}YO2lRsQ=;BA-y`@MlklVODJKb{NKau-NP9HkW)3za z+_`X2rM%SOUeNXf(C7S-HI#Yv)kGTvoJ6D!(v#HWEb4^f)01=11%gW4O z)!f~$kCZzyfGeSwvb2(05%RR@Ko|k&6vf}UbLS0^^$_T;twSOwcncmbH|Wl$J2HFJY4r2?AO084O-+OcfQ}ww6d8H~O z5!W}|+voFEh>2dr*Abkgs5NQU>13eR>n0`4$*rU z4!>DxH<|sscadyYMwmkwQZFn|Q!US>%Q_iuTV^;504&dwS-uW^pzrBV0XIhg5tWtt z`C14iHT5Vo4|r@^*9G(n{vQM=ws$q~Pwl@&?>g5HCy`h=3$ov*TM!qJ_ko}+IKaDa zSnPIHcVXA$s@K2AMVu_W>$S41i0_(f=$S0~T*Au1apH)IRo!1oQ!%j@(C{qp$ote# z=hgG0vC$UA| zF{SnWaym^OaDBH6M>pX_e)Z@)!}lOXn8=`GDKAaf$Fok@sBMzaCoa*5dG=;2v)3~) zpWG8!B_Orl~;ZlTLxXpp;2?}`24ahBi|zgq@B!y^FJ%O@mQjA&gr952;WqoYVP=$c=w-)GpEg30pI9CQ z+x2IfB}l^-$yfa!#^}YwfBhItk=;O|^)^;)D;Mf7_P$U++P6~2d(Ur2dW`t?Jvy{u zG)$Y64i$bl6AB3qx7l87D4j~Fym>;#DY$c=IIQ!c@WX-e6cF80R@GX|%VP7u$|vtN zBe-;V6J)Q;Pym&@&p^ zXO6*by&p`cWVMaHcY`z?AgLcnF}in0hXhMd9-$JD_{$tK>rsuFL=6W=R=Rm0e%y+;?>bCGCtIG`-^_5>47?n4u2+ z3CUdCk$RorW#mQL9BI>++u+|(@uck?9q6XL{NCw{1XNHkM6V}o>s{Go$wWJNVB#*{4U-(QApjaol92S) zy@X%cU8QIZ*F$1j$@R6*vB798%V=7l1A~VreJq2CNfFHe4#CRN^e4pQ_}y}Uy7`tL zWXy)|2j9Ku9m@-Ra1vb7&~<$}RBj%3x$>s*pmkp*2xx*GSlCu@d*Lv%B^m*z4IhQ9 z?u+D6$0|^C`=vYwyC-4~6|U3D^R%b6B7s(&a%j?^7Jbw&9o1_L5`;<3X{z-O#dm$z z44rwTq_&x0d*T9OYgGd;(kcvLb#`(Kf;N(*AxU%JhJKYSK*s$Bj7xZG+$qDiY3CCe zHk;1oogX19!MwBZk6^oUJ$uCqSe^)#U35H${y}}!gmj8onP?OdE!APBNxn-XPkty69gcY3X?OBrw^m`k zD77n-j;X^GvF5`nx5=D1Gj!4`j>eOU?%k(&bN*z)dVF}wu#16|v_i^!`U_PedwHMT zePp(G^TWZ>?C9Db9;t`?@tz<+xDsLEX&RG(|5&^A^>&sjHl$K66o%LVf>V`{w!2qg zBnNR=!tTQ_&2ev=Ka@P%WsWAJz8EL|)svM|%6pQ(Isd-fu?~r$jm^>|qV>(NjhgPE zNvY?-!23Qk1qG|)3elvwHefT;qgMw!??57<0YNqp$lLxG(G8H`JodW+Ucj|Owki-G zWHK(uGbS=FzJK}(fVUHe;`3+&y?mz5Q7H;YuEJi1LBTTFU~dQYQdf(AN_@G{|7%F> zPRUXhvsyo-nHmJtv3_D_qK>tv2SPFqTiGXImzEa0)6~PQCnrueP7sstLqzY+p-)6p zX2rDcr0_Vll&7cKy%%0=?_<@7d{=%eg-3oQ=DJ4T3Lybjv1Hho*j3)kPFDp|@|*-_ zLQ}p5o;$bjO2}zpVULFf@$2rqD56uu)fb-50pk$2>%VRnF^8@$*mYy!Rqgk#b)717 zO-8ITC+T?{KCgYzam{skt+O+^^75_Ek$SkkEGst+6Qy#DT%99l%om)<#~!Z=CDv_x zcb15}(U7Ea-iL&}@Eanz@|qBQCbV-T_Q2sg{zc{2hsCGdjyFa+ELNTBPi(nyd>mBz_YEE9 zyd7BR>HP}|qO>kOnu9+|KFpZ#=G9oPa_VwCro&gT?-6jmhGfp3r_7s|)mWa_@Z~Da zq;YoeV^-Vr3^OiOn-}=-@Rjz}cqcNwv8@;8a%Am}J&`Nb2F-*I9IoIfDc_gNmn=zX zl{6M;myv?^^^rY|T&I02`p7__iXuAvYhsa{R5DNC5UB{UXKO0Hdd|=8~ zM{yONkIKBu?!2O|?l_{I!&@;t4ZcKI@<#B3nAjhD!uP1OauA&HwE{+B){mmiU0hTA z>|t@ZB9pQ#TEB3wwI}mbAy~&KxZnXb!Zr&@(>3JH^#sknvmx_ZhVctS6yg-#&4l19 zgwAJ*+CV=q=;!Xm@FdC8)l!A3g=cn7^TjM5sYUfFP}zF-$(N}c-&!LT)O&m5@_c?q zM!5sn`x$;@dZpWo@a|=U65AW_kMrLHRDFjY4@Rqqpi!$$qLMY$Ux^+h#A|-5R)S?I zz^uI2cDIRSYyDERu;os&CiW8$!O47%>Xmz%QAdyM81#Ca5m{z%^tP$0=Q+;{b|PlW zZkrsuCi_Pb0xa~CFMa!($EorB1CNLE37e3Dv4t%Gr61?%Dzqjja`3jd*e%fho|-wH zOnk9FM7!TtvT19GPEKIRyu;e=HIg##-leH znCb-@oabOEuS}zi=k>Ty7lMmh*=TNc7h}EFlF+n{p|nG{+)RzWW6-9SQEnXkVUaqn zGfGkTr4nhN+mmU(AY6qHSCRK<{m!eVdFlecpEnXhD1BU1ANxvDJ`z2m7U&vUTvyPd zgh;mxgj=dc)#!fMd1dp!@WbwM#HWJWh(#R`A^JZgVLrzvMcG|iP>`WH-rgQNPcxMh z0rx?>53jESc@`9qHT2|JWLaoddB1PyT!K2k#a|P##b8$6=do5^&o;Dwzum3gfFS?y zdPbwDNvg-t%kBE=9=Dr$^S-f#8Crlzv^`Kyr?0hdz+EO8a8(;D`OueUP=|PYU0b72 za`-6FJ(bw^D(;eax@}y47-Qi1Pm^DbO+Jhg>T#9*gjP=tzFu2Cug<>3NKYL4+*u-KhCqkuq4Lup(i-Fp}u~HFgZoW<;OX6J4doC82UMuG<*3y z!5J;r@6mhi;5EO|;(??CuZ4?mZN;;?uRf`7QnLl#pe~^L-L}%z5u+rHr$v-N2FX=nZ!KF;;-nGg98U)uJ>MG=2AWnN97@ zd87`WvtnGMNjHj*KC4(gR@4k^2-}idFgt3Ow|BHnwRGoEFqLL%cAw5?%Iq66-zZ)r zRA|5Rcy+_F`)#|i!3}FB)yEofHx@5aSBBoJdrPXLG>x{oprVpw`8vSY2+{$5y;RB^8~5Jw zUAvDpe7~?I@$d=tR+-g^kl$}^e7je`^WsYag&5|Lq!O#kBp123bCjB6)e00%>cc8W zlTOwGBdkr@nb{la>^B*qSg?|-VvXk^?mFZA7HP%&&qga$1%In=33T;*)JJJgiK_9uzggq;bL&I?Vudg7*lq2f z1=db26!x`I*Xnc_J@xSpxvYu@L?C13LdG%y#v=G_tYhe0)0lZ<$}t-qfIP43T2IU` zjINRlfU=e6661xJ^6LdOAb#B&Tze4Loo|B_oW0}zLUx`n!gh7mQhc}B`!)33M`bYh ze#xA5FVcJqn^zy6k`!5<6jzyS8&Bh?F?E~=4tDTT9H$|Y_fqFz#(^15fk5mvEdEz_ zYeC$F+!G|U7fphpV2$Uj88yn3*aB= zo}RsQ;0J<-=9rlnkY~T}^zpXvoe$zv3$~)X9j8nl^^~=jhh7f2+W2K_r&V(ru6Zu) zK5AgHT+jOt4rX*+u8p2KnS(%=`4qX~{f9AxS8eEo_O8LIVsl@$-|5e`xq$wdBq*Kb z1Ij1DOA9OuZ%xk~^k-=)^t2`UikQ7rj9$OCf0-aHhKEeCFLWYWG;Zg5@dJ-CaQts8 z6>{~B^XqO&cMgexkL2Smvo4ewIMOJ+=DLkZ2xa;7e4EV7AtvF(SLK;Qj1I?g4WIF3 z-pOxw3OCxAm{`&{HE%P*-ruD2rx`Q-3?-=tC$~QAkg%!E7nuXEd+`N;&)UR}k*yg85@g2CtScTc=tTbpd}J(4}MO`eTq(S?TD z#Z0@2{KX7}{`U~=qNEYM7EVWg2#tGO%A_pea{EeVXEg8ml?xP)YsA+^ zof{9Zw-Z>mFPio*Ul2)LSYuRMzHt2{#lSX@S*_9Yz8T~}DjU~uJ1GTqzP?adB@P;x zeCWINEBd*lpyb=Q26#Y^q^_oZHzl_#Lm~+M?^KMRo)&Qi!j}d1ioIxYlSCgC*5nGT z6K+X=dMUh3D53`TK=Q!>>Pi(wP0g%W<;&xtJ`*UM+68x9>X#hE`ex8d^ttgxexEpx zz^_bsjkF|HHKb6h&43VwAZ`Ft0F!@;mb>=Ddg@&9WUGfaSm-Z@NkpRh*KO_iu2-{v zee47S)m6~9XzdqBX-$y#L0rvCxeEu5wU$SJ9zLg-cW)nF0s_;K{evpHvW>h z>QYr&rrwHE!tlP+{#TK@$2GVh+$@WiK71M)!)fjwYq{2eGx@69-DVRVSE63Rdm;}~ zO6r|ufo~|yGEg>9f%Za-2H2C<_)9IFtZ^HWXBFi#wBUsXJ zfdW?vL=PP(dH^4|!xB7a|0!-Oe`M%hnC?ZlvX`cyIQca1`|4s8NRD638JSL?v;_So z{)5!5iLFG!uEE3n)0mnLjPHpi>}tF6`aMG|p=RRBhap~De`d7}&-)F}duFU%GyaHij)o+r|fHz`S@Cil|XgU|%$+kPK zJWjWE1LODBibOoEr!LuNVj-3p+lyGtT`M(NxY;+vTM`#rmHcfX6$&Pm7gx8S6}Foo z8h7EVm^!X^9Aza&6sNokahJo7wXO{bWo7EgG*niD#ha{u*mbUGBH6xjnabG~Z~IZf z{@(e`yLvT3X%iXkN#;=HGHz*&H(k^Zj}%O-Q%0Ao)W{VhT~jidudF}j6i%B^EZn%; zj?xrywNJY9po5K|E&DT-_I-?knlM%m;GdEl2FrA_LQ8ah`dD%#a^bQ)S=ZVn%$B#k zozaa-Iv&Bd^=cO4u8X=Pni*ufzW_NJ3o5~(q=oa-_SWSMRr=1M8TDf1mIxU?^#;;% zzGmUmu?j;(SrR^(e1XAkz+xhw2g2jgch1U}OupGuf|uka4%u4Rm}S)BKa3{QweWQ@ zdT1K6-1chiaO2DOb2;@#@)z9UAM(;9Lq)_egn~!s3#%AjprkSRRCN$RZ~U1Z+3wa& zz?jDsPy8Nxv_Elv4KmIrXDcp{WbmCxOay{NFgU!x>e8;*EuF27R_Kjvk3!ECQWQEc zJ+83jQebtBmo&Yqj9pco>W7WHoLn&a%r1>emD9skwxAO4Y3fy!!})Py>-IqQqBJ*> zc~P}*K#1ku^irV4-QLxtbx#^}aTf0{1-l#RX18ju6-l1$O1(Bhrk#izS?@d1i-!uk zQEP)OGolox$SYj4H`>zH9?nZVXOLkS@(Fa4Gyv8J)H@EW(R$!~DScy}^yF*I>sp5{c1CGU&Uhna8l=JIin`X%H}Kby?dUu1@~opC zUH+Mcn95+*pFKZ!v|LleJ9M(1ySpe%8h;5^C9#h@i8Ldaz`0EW7=j*1K0)`80N-Jl}qO%6(f*$!)y#i%}j1DjGAU zoXjfE9U&yXlY*)^EdZIq8Wo}g^p0=$U&=K1#>4o64GOb^=WkVmgjrZ)uhcMo_eRfJ zd5N2G1oafIJQ2J3rlcY1?OVB%t!16tu#*ZKc-wx3(Yg`cHo|v5P|r3{mwIQ5On#wE zofj3JoamQ=-~JUt{;S_t`O@9E(kA3rr?yL=LdP|Fg`=o_E3BA}Ttx=&vDdD*sC%y! z8k799Ul~3%gV%LmrGiDta-;7dvYm9Kb{_Ygn^yW(j?{d9U1z7{e$Iw{c`a$(a_Etb zg?q0$xA8O32`7jXrKg2lq1L)I_m72)6m_9n;(?rt71e!yJ|3ux|tbW%jbF;%wD$+jOHjkckXJFuO!pB|6~>+_9SYx z!H`YxO9@E{kKHx(4!u(5_VZfzP>f`B(2MnIF5Y=BLwMDzt?kwK$QF|xJv%B389mMJ z=S2-hb5IzG2z#=YWOMClm8+_)B;K0Er))vesh5gfyM{%_7(U$x3a_6%8ALIT<^NK+ z9;ClR8jyvsSXTJt|h@Y?5Nx3Fm_nhY&+nGAAr1ni1QWUEH zcjOWQ;3OKLBL6yrbE8(j)D!AId*)I1cfIutys{USyW|;;NEcumLu^dYZwb; z8{<$i*)c&`L>hrQjJ@KL?x}~6L9VVTNZzDc`8?(^u=^AW}HZTtnonJTZO7Y#l%Fz~)q z;CPw?s-I9W--yrAU@u$Z(Gk=T<`8lPW1NQa@MP)e~Rm) zq=cvZBa#mX4L9?=lGj%2*nab1YRXr3-C^SEzSm9*)jl!ocgha-TLyEJjkIeVlxNd_B4RTSlJuoI%JLu8JHE{CEw9S?**!7m_`_4kiHWkTFSgxee&Wk5N;E1q*b^X8*t)d z{X|w64eKxmv#&SSXV+dNPL{TIVctb;{Ki9rPMx z!nY=QVnjG!;<@eg{Y9DHt$Qu|?z<|l z>%jOx!BFCCwe&-PbU=84nl>2|;}ISJrTZA2fcDYf3ShC*UJyvVmL@P3)xL^uCaNluXV_kAFx&VcC%0>`GS|`+Ht|$?5C&>R!lDKm|jUq%#JYD+muLG6G zipvLX{07(aBnP~As{`k}kW2y-9z)U{f!xUzPKKY_`l=mbet!DtYA*yLEn*p8T?qdwfPvgG*Z+#*28Hcg`QJ3I`Nj87uA z@l7|5^U$_m;(e=s`kmNy)Msc_NX^5G>3q+jLNQq%dbO1K%L`hQX1K=Qock65`Xj8w z8WjLA)(TN0e-WfO{&2R?TOzJkitDYZ9GEt_TIhm^T=p)xBXJ`9X;QB(0m=KO#u{Lj zq_8;Ii{|W=t0JqkHwTVWZ+xd^129}Lo_#9lKS1dfPiTY>N9YaZ#z*fd*c%!y8aLf^ z>G+5)iNc$D{Ha$+6@qM&k&Lw>;SC{I9?(C=e}%yCdx9mGjJKGdVm9kM_k0xup65yC z3kx(S#~ovfh^hJVlKuIitcA-KZBE;E6hsMaL{UGbJC67~)87a={YM7%?>F|g(i(-&+%}`qRX5Zx6m1=pi zo{*9tDwajbU_zk=AW!4@J}!G>7YIN7h@yPx(PK3k(|V)qnAX`T=X-8*89xBlFxr&{ zd~H_=qu2MBJmZJ(WN;3G{Da=e`D58tCcVK`h{e=5c0{bJVVI06`9co#?bT$<&0)P| z$P`qDyN)YHi}yUU;$av%jk&4|LU@+)d+G83gadpSR58=}S$k$ld`NLNU9omg##yee z%ZwXSfN;M!)nM>~#rWoHEb8F*&Jw&7=tcukx;@M?C>&*n^#?WYU1gWGKnOZH93v5f zd&~rLh@XJl%6cwLLU($SpL}n3BJ}JZKj4G!lqh=yZ|8!#tcJOcV`+#)j^yAusod1$ zOshF#_w(DSjfP?XoF3oETf{V#Yv^~mQLxp;mAiwue;EMfoTY1Hz(-0Uq5!m_a-jUa zjpqzrP>&kWDgiDbD!>XcAh0pQ7VMqr`l;A^#jvJmqhlc1uA-@?jG9U6AmuKvF5t#303r+ z2M`Q|_CnoI0%qlR&j3tX)CDBQ_(PZw2E{iCL(q4t0T5T-T0IB6k`g(}Lxlfv@ZjSp zFD9X0>nSF#{;F+&Tvh?s6{Ij;qtfO{mkn>+Bf@yj0SJUQlLVmDSV9*it#>A~DK<^- zej*8VOfX?CU#=IDJ29g%TQ5)HFj)zZ*&7J7m`VoqY=i5s4W~NabDllQzfdV=0zo4` zb8ZPwXhUuY2&Rep=EwYnDO1)@4KlX0xcj~CUj+vIhDxGS3v{mjjZmrq2qm40FA(hV zHJ5soWqxYVwL-iUr8Og?euIG9GvsAEwL8noCd_IQvEDCsU;ZeCkT09>|fv(+sUqw-SsN11bc9 zO;}n6S$il(?c=?I-vJmC^vgi3bz|-^KhA)J9lm2yz-q+9tIC}r_aKpq=hqc9&WR;RS{G_+n zslgpGus}e)&*aeLGk@X98vsNE*3q~j^a(nYo*s3; zy!?XI95f6^6meBdB~D~#MD$w#??YUw^))?e0JDF)H#bB)ugDFw(-WbJ>0#$LvD-a00VswMw ziLExk*Jn%K)m8)FBW1zI4FIwtLVFOe2525b;7@;{b^s#Y{U|d}%|0OVoceb_C6Qyk zn{?b|XexI+pkN;`we>A5 zvWTVTj%I& zWvd}1d5iS{;>k`>0G;RW$eRmcL z_&jQXg7kZ<%L^5gGz(Vn>3004QKa$ykNoKdD`rjgbE$eyZ=yn!JvQdr5Sp?*f%h(_ zcWKNb0wd~uIAEZMo~X*Q69ECh3U=$G8?E)=+ET>EBt3w%ssV^sJnwn=byj*&6DI^< zHdZASa0(Tv_@Lb$EVPiynV%RNYFM(n|ul-!-)z_?kY9GhTTJTP8 zKq(m<60Yle9MP|Kk?$TbD$QZg`;YTlvEw8(#^_{MMqT(4)NNnwLmCat;>xG@o^btG;=G&9`$>BsLQPU5bmUz*^_k zH8S4fz>v(B+81gw4Sa1U26_%fwa?Bz?%OV?Ij zVC=d}=r&HmY>KZ&U_|Iv5`i=42AJmZ^lv5|r((DmFdZao%`_s&@{s60 z6_-;6b{(6%Mg>U|#8O@h=K=7Q*|^$qr{2(Rxp1e&&|?5?5nq{)lYZKn|Ag%>^3&zF zTS#*gd=4rLiV1twwtF!-k8l=m4qHLsNu)oR$@q>1yNiPcy_2$2Ob4sQlu%kh=oXBE z=x1&C;8|gO9NvrwYTpT;9HuC*sm}%O2EN*{PiU(lYyy?Ld!28V3Wa!|9J84C05J4W z^dYY^Yjdg~Mal=2q-K9V_`D_C9k_sGj`VqX+MsF5% zV>TDEUuV?qo#~FM0LlbX0X?O_Kz;(MFIc)7LV*IV8IK!DAr`1QvlAl6_LdF?iAO~c z`2`zm2C(*q!vawyHHcXyJ(4}3tW^~z#Q z|NG~qb?hS+W_jPotYzMuW-5HBm(Sip*0)u!@WQ9KMCZM(pSaMiTU9CGpk*2^(4gSr zCtUMh<*=gAt?gi;WihkD+`c~UiA<;S2eTnb5Kwi_>$q)HgZYhGE=jD@JcSDPPHb`{ zm^A_EQQ>U*1S0P{3pDT75A_np*|9W@h%j@_-!@xRzT<0qu*7C{<-8|2;^gE}%!a!k zelpZoCzTX$TnDTrNxt3vSG;`(FCl{UR{;T%L6T2QaU~A}BSf=cU}#eIIj7b8DEek@ zKPC>DcYxh_*J#87Y8=bg6%p5-a1U2J!O@zXLz#wF*3bqMSjs&gW*yqxcBw6WZPs%| zMQU?H*G24B;A3Uo;uYgw@bQq-eeWpEG3i2`xo?Hw4BB>2BQ2q^a8aTIh zWbF0uMY7hV|BtTk4y5vZ|5uTfT_oA0$PUR~p+r*2COfOLcN7w`vUeFNdv8VATlPxU zkr~H29Ddj1-RJZ9uHPT;N#}X)`@Y8OdR?#UKDi!UC5j*`iGhcLzUC(ANzS|y#K?;Mu=Pg(UlF;VA$U-u^3@CvRFxc>bWt(B(-8vE$9EYmISofpx z93@|%Q^V37?)9ILDuJqR$iuPLu+)^OUpBZ>F0hjImHmnPL+?gfD;X+&;#T3Q-S+zu zl`)|&AOP#2l{jmkp|8H!%ZK0lH1F*v=uK#bH<0fCj8w<@_C|HqtZ^%bfg^^B+a?^{ znxWWA@!o-y_-gP4yWbTdg4X3q=3+EpTfA!xxS5;$~d7tjVrjIa&P{=QZvc=7XiZoEe)8uwhCk zT(Eb|eUtV2mp3Kk|I^=iZ|pHA@$lsfCeh zt-{mtYuJky4a(z8Mo(=iYD-AOP~mm$du3nk-T=e{&M@YEjj;&;MQF?WPfgMeG|Ct3?L9gOa;OgZtP!!ld6*qd~S80e*jC?&=H;`sGyP#Uo^Q2|{IXv`mJUN6f^OML6?H-kPjlw`kgDEHyo7_A3h}>nb<%a;X#A zT&^tZur~)mRfxnKVSX0X|pJ18=`~zX&Bpg zS*KiF^SJkOkvi^uX(+KrSoOx)H<>!G8~8uT^$kt-NU9cbd`cP=KsaVHqy$A8GcUbP zK>>dHG`Z~Q8!1WrSC3I%`gwZpZ!82D|A7#yIJt@uU;V#LXQzu_=$(2Ag-9N^0m1)8 zb0{*cmw>4a_23GW;AS|a3^ejWoPxo>gTO>;p{v(ruW1za*B>>1qu1_pr?VC(HX?b~fZj_=0y0_pso`N9-vSs#);ZSF((2`E0TYq9`M75)}eAcc*9 z)EqdO9|>s^=^`O-4rNqflTK^mWakgW+*w}3-xv-vysk{S>>2576VAUUEjFaQkmCn! z-a61BO)u)u(8z>BknSM`|3?5T1DcKrB_q(o3NCT!vk~LJv;g=@&)u$~KrrP2tI?Fk z@i}b4p*)Kw8KNpOBmh-GO!R0GBBz4W{s?iq&Cx3ZbZy}%>*_H7CRC%Dr z=^S5u#gu>0_Y@znY$BGy-Y7uyk-yfpXKG$#Z=>`)3lJECK23nRMu6xBP$CEvhSn-t z5#H<;&Iw!$>xyr(F5O3EjnnG;0hE5m@8B?_Mih9%@JTDeVOtxR|~xgMZ_kr?}5_V~X&kRm$Oe002I?Y(B4;w`rMi|{-S;ExbZD?m+?#5NwJ z;`D2a?t{m-9cX>DzCsVCOa3Z5L@D5#SB2<4n|-toBt9-^V8TnChadV9XBjer_2Ed5 z{}18HrisZ!7%x_6R*smFkDM4>wZjfi@z+SnktLtl$MJk}ne4iE_$$vw{0k?7sWIWJo#v^4QC2;$j-4W&1{f>HCUk02Wd}P6g6=DIKBkTPc zHpe1Tkn_$QtajOT2c+T$!tVcP8+>gEQxP34Kp>j`FkfpXI5!J@W$S@yG~JkkZ(c%oZEE$VGGDQKn*1g zrbmejrZ@X}>dw4}FvzpVUj@Oo?BsXP!TT&R~26I5tjdC}NfeLkg=r>uTlZJx9CC zbFbRhOuD5c-Qx}&+)pn!J3l_U2dAZUI?R^94dQxsFvA=BLG`D}i{~#sTEkr;SlU1< z+tEr_T|rUJ2H>Db5eGhx{O8<_`g?zMTc88WfoMl~mP;@@r~0kKnT8JOU?$Hv8Rkm= zr??V2eS+_km9e~GCZzkdPb%tWBuuhat9IF*?tG6MT=rM;)&C1k5hWofs*;%F*rjNd zm{9wkUI$CgXhvE2&AFFFuZY)8y60XBteXFhq(%2O?t`1L9KTzftJ%8h9Z*N{tqGcz zbVtQ$H^htA_R)n;*vO<9O5s|Py*Fa=NyWc{q=+wa&_SWV?NJL5zAwCHU|>8}pEg=& zSKC|Y?HRK2^K|W8Mu>kC>&_FJ8ZWhxWf8`+o!9s7ZolgfomCYtj{A{p#VUm8R~Spf z{}4p9AcadYQo*dxb(q}?VslnKh`)OvkX(t9&9mtide&C@3hv3X=89|OLh0DQMRs`4 z(404s33URR$?#TcK_1Re9t@j*gQYE}FSNqxh^llQ9TLi>c-VTyCW85Rs$}_kb$Tt) zl?8B9kC}b0Wxd1AojY&VOuhMdhw9FzhuYQT(gWK5A7aitaW#*38UjCW^Wj28Uh}v@Xi!|h?&3VRg(SoR}kIcxKXX7_*OioXAJLd^4JVKW<9EWU3i}AC~ z^uc)$>c`DSr}?8}4*FMy@lUQH1B2@YO<{c6pru3g;dU0V}fj)tO1|PS@PD_Y63~$GIm} zk(`bF`J(zp^_PJ6S6G=6N8Tqp`%<^-ojoE)I9oe+yBe?hVZS4D6}#cYG~QcEzEuXQNF>y z_QNuW4p%13q}7jm))`SFlXr!P;87Q420M*-dMbH0x_B6pGQB!ewSGCSYiBdbuG32g z{OY_6@6cWpIU45$E_7ea_WtBg^~N|6Q60J_aV}af9KaWuqCg?)ye|O}7rsT8VbN)j z@yU$_Jx1>HeTCtH#er8wx74$^Z7Jh~_7fOcfUNnV&H2m9FCE_li8m+2P!zbz7+n{K z^(BEgt{{lTug!W1KnzKyax+)t^d)l{WybtKtslP0fYOfNY6#VDf{Vqn)=dTXo8_lb*`(Utt--y28o3ZD+cVehDw@I|puvKoXw|&6J z1&Q;t#+R9p1W8VgCh%qR>?$V#ZUCAQpk+q1m{V7Q>FvE_Ez@LaY=LNKy}LYlXBA_P>x5!QxWnF*#`^i5GFvFZ;2@rlhEho zP9SKR_Hn=5|e*ly* zB5L=+%jHu0%aY!>lvXNIAbBk=iK_4<$)cOCN`lzGrr_s|36_ZYCNfC_leOB2wRV7~m)S~9SZtXsX?ALtkmKdIy1 zoc4y7pfj%vQYm@l{5T z=p%kX*BZe#CBGGmc~1*znqVFB;{ zwqdf{ z6ndihFwgNgNc~v6Kl>M}vb)u*B6Xiz)`g{w!*dJ9+_(3>{rX)S?UEBp`8!~B+{p9Y zTIpnEdy&Mh_MB~;(#(}`SUDS|raow76U5#&nQ{}{qXB#(aE9&G86uU_|4oj8>^zVI zuftWJC*9@(JAeW+nbrfI`j+-ZTq_RZV4H2cB+lzZn9h+keq$0SR$`TpS(0~Cy0r95 zWU5G@=Y&8%zqqQ3aiBrhj)-<2ukdD%0aa|V*c}GYP#haC0WuVbiByved?MmmMT8mBcMmz7jZ|~vUhu{c z!r>7?CA6YDc=ZA^6afhAhm!L3NUaJ~$}c|7=Cu~`ta&f0^yH6Ny=lf=HuCxi`bmSj z#-b?A9x1EI)|!Vm`-q{n3oSLiidFR+0W$ATGC<;QBY!TYw?E)EXd(?23>7pa5%D_g z7z{#DxN$`QQoanN0i?p@oBuB5A||Hp{?|J0rz}tur^52!xOI&SA9w0gRzKMWB>>$Igx$| zbOX09Ua$gNp05pYP+E?!%|tx(S3s;c-D23VShCQXi9T0x5Q5VK41x`DJv)Yc|L%k! zPJqhk91($+1yBJ{fUh8293qrL8fDg#oQ;5P*V z1;_-nJSdtofSuiCzgaYHy^0l3`hJiQn0F=eZzBzvSxs-d@euf&FI6>1{kDH( z=0UnZ7>7l$0qDB>?&*}t_Bs&5dA*kJ(#g1|!vmovhb6CUp?)v*M?*P$jiKni-aI*r zQ;I@bQN}folNX|J)>^L<%8=nGor`5a2MqVEV|S+1{=K@^NDf$?S>u6}JU8m44J}mI ziRAp+&U&erRA@U?-4H$&WZK9?@!;Gd1@uv;uYz=9D=TvURrP3iYR>|_nc3>#FJOb-m*dZe17{qt+L zwHp*JMI`#9uIasHkZ7YoYh-rqg{k%6J5|c|ynJ2o?Xpgv*lC2x%8vCGgqVRhH1{ZS zXNhl#b;?PD^@!B%_CKqL9jba#zIUQ-$ZdhA-3b|8g~!7`USCDcIEe@;=^=sUs~~~M zVU019ZOU2lr4S#P{6ClO@gJFifyJhKM(j+=DG<9l_mt0HL3)! z37i@vy!4Nc*Z6RgkX^hU_B1?@&72A49^<*c{R|iTEs&%c7{u|HB=F0$(hY19#XRQn zwczy|*Od1$7K_F6el=pKow%7w$=_m9>Dy$c`b)nT61>0fXveHwli-F46`yKcUG2EZ zZ16sH#Ev4ukPQ4oZ;1c=&=E`6C>`LE$5hRRW)aT6!*ePZHx(gIf@X5LFAEOb;2$qk}9#Yh}hF9!S|>JP~iVdg<$JlG{0`hgOA}-93LOdVQQJ z%^Ml=5UEcUky3^R1cF&=9{v>y03QkE4wI?TX`^lFjtjwZ8qV}!?`u@Uv<{g<)zHGw z1cDA}^24hKs7kvteQ_5MmIYSio*#&=tmeCtJ{#SZi)mLdsC5&X7NAnDlkV9O*UZFQLf^Y-CMnH7UJok z=OOusaVwmWsAf=TpPLS7YmQjQPs zSJx2o6X!+D&uLq;;=TIeU+c>w#e_8RN*j2-bqjkxUy@yaK4~I=t@r2+-hUpHdBmx9 z4ZAwtkiCQ(p;Z^b>6{A$74YC2EV=cFoGb=_5WF$f?%>=@0%W7D2;mD0-Ymp)%-TIO zUi`eJZm^qXMVMV}yK1)PlyxJ$5P zKpQVzNZcJ(QU5eZkYo-kBR&n&uQ)(hbNXUQSP3yOTx2JQhd!6Q;Dc8^&A3*;F?3j{!!R2toGmSf>)@L`Ee55L+1GDEPXtv*3u`XZXXAp zV8wKpy!Pf|2xH=vp2Ins?;UJ*i$0%xTzDXu|8(@_oCs!g+;p$j_o=yR22)-1 zZg3G`dD_2*VPus}8IXMR`gpPuCGkI>VP`>ub{wMkb>3Pyp}RogS2VW{C68bDEFrGv zPXFj=Tli)WHTT}_WQ*UY((u2{zp>TPhml}}HvT)ogot(^OWqE+`>B?B{fM*DAAmN_ z$>Sc>E21Q)30Wwz*!ASmy8Ge8rlXa>_x?n98X~ai_fEE#w!VAW zXqpTMp<{d*zMcGEA6`;kX}L?=KX2CXiv9bg$VKmA-`<7IUhl)r6e^y;@NeK{OCH#* zp_1mA>6ShA+wtBl6snfQVi(Jhh>nIFR)`gzGo#_q*}QUL(~$ z-~Y!PsiDB#)iUgTQX38+DOSj8`cseOXl`!qzaQoG>(m>9*N^Hu^=Xxy7NCxurRf@lcZ9Zk2+QW)_5OKS7)-sTE^YQ3e$FK zOujePs=<%HuiPhfq1)gHqv_Tjjoi@CR#IucA+N#sv#iW$Cpl%6lXflF$f^5pHFg*9 z88XNE<#Ecy6Y&Xjk` zQ_Em90IkMNfT;a&?JPm$=H>$L!6h}mt)YWlo6OwK_JkVF+XDh26Yj|)5^7LdL4|d0 z*JHkGwUz2ly;!})Kf%mDXrZL#)5HKmwxE+^pv>@1>HOMg zME{btr2Ol9MjYwm&gN^tEnrmtC35EcTvN!022?t8s7ba8G4<608HL@sq zLc@3Qed}0}>#gZY6=M@qMC^J~>~S@;_b^@fvmO)iH4zxG=WX1M0!casNNqT4i`0fh z+sT?RfCrRxNjdWnWl1?yF%aPXkDx$)Th^&p>~thFn5aM$%1olbrc>*(y}Y*;a7+2~ zz^(Gq(q6cO#<782fU;h?=SbdrYe3V%%F5~yRwwtN8>VBAMs{qpV83f&UR6KA;JT@4 zbl;66OB+Pk3l+>WME|iY!RWdpK01bjBkS{jde&=p=4upO9)}6&yZ)hXUj5)wDsy|L zH-TPIxzt}ThnLZ5_94Y}o)#l@v?)c!6B$0tk%R}<;H_piRhIWAp_`dR z&mQK(>Wzq+r4x|nc#$-U^THNKhVjq?PVU7cs^^B*D5$K{&HcnO5_S9A^BFma zLTeskfqFbNwcbZwn6@Y`!k9Lt<%qYwhw$&{$y#^InD+10R>(`P-TR)tAQrRPKR7=( zFcrm~Fm@I6=|RZNIH(8wLHj?y7QAzGF!br3ln8A!I)Ot$=%o*`;{B`5=Pf(+$@rGY zvVTO66&^l4LrQ^e6`ln0+yx3M7&$xTB0w&4kI$oY9B|7&QR}r(M-5BOKnO7D$nW83@@`+=9zzK0qPGT zyx`y$?<5E-oZW=QP6^s3*z@aLe|GzBN08b05Nx?-m-(Bl>hq$A9TsP(7nH&?Uw^r1 z2of3bhSj~ZI7>g0pS!}HS*{*rx;glVt?dFO{-O9G!TFIXCVfM;)l9Z&|zU>8>2qByDwS&UW`c1Xm z!=&c&UFVgF<*7ePpS_#kOkGdWFKXR#cu>7=Nmf}Vu}sL3Qo;xv&uQP}&0}MCpr@lS}u$tD-kpK0p zMe{*n0aqJcg*jbBAdj>^Ac%jrvC%jperQEEZ!bza`{;(RBW(ob2>SCNUv>q|ta(mt zpL0m?UYHv75UZP<^_(BK8}F|1s$a^U*cnasaH+K!+Ysusu?L<)HK4+55cjLlg{i8m z>N6C$60d0xmu5QU1z$1bUTl|zh3}fD;qR5hxQ`xGds}R+f`n$LKX2f+a45<0pv8f! zp-daajFtbmR+#KXQk`^=oDPF|$7g3)!*j2gr?sc~ub=FsM3Exdh$2=hHC=0MHUv++uJiI;i6yR~8dDdzb&y-lBJ;k`X7O%vsX zxMroqnYc3H;6!s$zd3YL==kXA>V{uJAw&k424Q4Di-(j)<}>97@EQa!$DvwDQ@gf$zGTlyUloln>dR&f zM!|bDx;^{;3;xC8mmj9xE}84Js?_1_Rt{e5VpamdoiCD*aL^YH<&2eQ?p#z(kJoi1oy&tl*347PbCr)Jd#of2th`)We~cBs>+A)f7uGxbbLee>S&3z_YOhKs*BxB1BZi;@(*8TA2M-`rbWdfbGaV;f9egI zSLhE5x7^+oy+t|}lwW;WBJ{-kOlUh;aR;_lreA^VywcHme(sKbRE&tFBT|d+DQFBW z&7CVan$)U2~fh zh9v9*P$sCEr;I`S*Cnuf4PIqo&28NAho5U2Xm=b4_DHX!AYmsp>W*QvhZkJ5$oib) zYIlUtwS}mXAAH#PC?OtKAjlEHG%}*u)wP$3xSuJe>qF@s10P4)zLbsCMdKRZ!$lEq z`YcB+3%W!}3Lu4;^Z`zX@rYMgEWH+<%3IO#sK+{|lhrmyODL{+&tAq(1}&hvyS*dV z$y{**%_YTlNIW;b+oKN^-}Z%aPxZH&JRPcMc(yidnrmR71QY)9D~uS%1fm56N;JUt z$CB6op`!WOebdQGs+mDDh4k2<1HkSpYRXofnPx2FnQLBQ7?DD;5@{T zc2{)L8dm|29LD$t6P)|_4#zJ(qn{=x{gB7GCSih6;T=i zygddUuO?`e+~$LhQfi&atbVKR)M;j|IH;Bg_TSRU$TfNK{LD9UQXlEmdqL4RCOlW1 ztg9umVyWcuOS{S*i2rbV7&b`7u--$x{)*`RIdyrw`>NU6p%&+M1iL4@z4D!RTvI0E zy08Znnk}~;HZT)f$<(DRVf3R+Oijb1HmcXuy!LXV*A@?^sWHzhH)$`@nXO{F@iasZ z!__Set``fSPU*`SoWDyKlNL&!Cd}4y&oPkohySaJGoCO!3Si`m@wyU5(WH2 zmIQB^nEH~Y*~f@`_iqvO^- z&$Dj-sEpU-aafKy{klJyZh^C6`*z|L5tBYfc5`<8%4!>3sojx-q7yXcQrjnMLz@@) z#16xQ$vl@Ld6=az1~wT7VbgK;2dAc5=qR@`Z7p_Z3)R~a-#vEnB*gu-l<-=9 zABECXdfW*+dPIu-@j{c)7B_2s^gbjGueZoajbS2|m5 z)^eN8&clRjmQmRi83%{z^5$)gJDdt{)Gc}EN3=E*x8`TEB)UHeb!#=bOwKVg zEz}}g#Mi9hJ1IP|Wtwtl;{Ad_QP7Uth2-68U_)#0zzSZ*rktBy)o&MxwK+0j9AJEs z`POL~|2Zy&>8M!tp`oD7z6qn9%%i%WY6Bi5yBWSO`oV~B+lSh>l$SC2|FG(z4ZPcn_WlJzJ7y~{)xJ4>le7H zMrbz<+8a!;3M{TOtbwy3p=EImZ~v@Vnd|?s9K&9piV2bSKICrrR}TSSo5KHGB`{YFn_~GuIIAc2fvSYmOW=6 zsa=pU@IQlbk?V1e6{}upf4Ed`^EmbyuTIQbkr%;Y)Ch1AUvwad@}1ZEgLz{>(>dxm z?zKKDkpJ$9;&9*)7#m{OR7vtBs5@@R)U0)8?O^UpF zM<{#tccKX#W46)&r|d7u;OI-8E<%?sxnKY5rUem)&zd+Fo3Rl_LmA%J67+e7YChgo z@lxg-z%LT~-59Atk>AuECJ?dsCMIL8mYQ712pwc}gx>BcC~qPoc6EO&-EzJ&EKD84 z{FPSJI+^%FlvS$*6>RMX^TRnTio3?DhL_>FBN*YJ72(sE09UrU+~OW?#yV?3Kg|td z%ukK7A@-jmL%L(0g^a%|>^&MACle*)G0lk!EK_X^7qCYv3>R*A$x5TD*UkhT#E1m6 zXJ&EU3%-RoQgSp^J}m54fAEH(scpySKZ5Dg%9*dpOE0}k;_v1$Z;S(jJ^5?Pz`2vh zFL6g0)lO~@IE0J+aDA!$joC`tR;AeQCfo&o<&6H5vAdStI+9@}Nq5%xm4O=g$AUwm zs<`vdc8z;}_v#rfoh8?+chFq-p_4iNxX$W#s@`oI@K9LmtHa8m^B)JYpNeVtw|?P4+(Tzz2(wr1M!_^K^C>hVb=YYsNLb zRT8SpRT4Ppe|>*u?Ah;esY~gzHRqztF4Bi*GP2WIo~KPpzZ02#CL0NVRS)5X zv&?_K?zqahvXh}CWHPcEM87l#b|PFWA?cJ7zUV)c5!f{J7}m3W%Bx*uL*>~ssQr+4 zyPG3JO}S&_01n3~-g@?HS9IuItC!a19Xj7@9Bs_%v^K*wu_rJB-(r5*$aFW=@fPb# z58i(+xIb~gRw=$~J;-i19{z7Tk9}k}d>B9?=zKKbRe*7GJU8)V zPk&uTr=zmZCDPXTH1RJMk%aoJe9Z?=tf*5<$P1>ojiX~tYPI8^+-vQB{tF4}|HO>^ zueaQ)+3lLJeH-P=wOKX_5+8c%$mx?LRCu~wkJy=T{6`w%6r>%O^Awrn@R$Sy`uDb5LaOyuROK74I{V~rg@cZGG{`?eA`ss3J zV>09K7v0x;oYQtT3o&2;bRn|E$qXqh17XJ>mnK~Ml_3S3*t*`j6ESQgUxl@iGbE;x zL?TXn_51mbTYE?peD2!S9=%gMukblGv|qxCwtdCCQzypVaLCj@afcGsj$uHXo)W~e zqr^JK>w=4UYAtu_>85nHv{h8L*HNftBR#AB;@guI+;FF_PdQXch6U>*?{dB9o;!P+ zODWM0uaitzx-;#*m>BaIb9s4Yq>}UsuV_oP=X4wI#j?(n@^6cWbZSBx+h@76BYw=T zt*M=9ByZ$9%fORSUObtW>nD~dS%#4&Su51L?i%R4KT^pkYF(ZCFdeiy|KR_-iu{5Cf6>5r20dm&2lweC?8w zDFN9%+J?{3-b;?MnF&T(+nRu!+lrlK?+(7Uztl4y6^ z(dg#LR%S(0l>;)l_3PdqSVV!itbVYoVdKogW-Soc_5?*^lG9hlaY~KShK;${< zR5;Q&|^h4KaSh4r_otm-y zy^=3RW0|85Cv7-trApx>zY#`W58d3{{BYZR)U~{Hf``lAx;`;0n@T8g!A8RQ!TOG> zce>Md5K~-uTyl8(0_PI4a*CZW-vr({u`| zY`mJ{e^zBqTRJc5WLB%2=c*1cDGCtVWn^PK9!M_C74S`P<$Ng*x&MhwzJ_sIE|RK! zMyM+~Hj>l6pVgqf^|gvf4RUA`GG5IWOws}|mmhkGES0&JEy;IPW#`QthA=wA z3VP6YZ>D+r1g31)JZRG~Z!f+*QXn{A>}6U$SW~REX)!55j1fSaN{+;Am8Y6(lz!-v z>B|>dwI(1Ec%=v*hq!RC?7pw5p4(VZ^R{vy^la^*Uj1U2`sMyTL#fLTY{mNXPZed& z-;&s3lE&$zaGFMFu)Cusg;NFzHBahv32!_50L6m6d)Lox0R& z%(&K8ym^k`l9o|^y!b11ub8m{T^@JUcG4G5^;s)nidTUyxX}qtq^!z24d93hOwLAIFQY-+jv6(yQ}UiH(sQxj}+ta^oep%DyNfw=iK7qg)xdHqq7t2a(r!-~~Hmt?U_pjv!LR=5g-n`hS=8E=djf;|DT<^Ned(r7;UYD{N?~g8h{F~9AZU?Ig zw#iX#sUt?J-87vTlhJ2+f`*ta1+vg`{JKrqegxovWz2fn`|lehZ!Av?nwmj%=56NM zQjebBz4?)ZaQRQ+bA8SvYUFGm4Mi-pxO56e?_758)zrDtoyR}L${*@KL5#$krlxH# zgT?k&XVK21j6Fl2KVM|SzPC(S2!Mcw;Bv$vgCX4~L_kP(K`PzT0#^2AS!dx3cB)sH zN(7H^Caf8DX&9bAAh6Xo=ok^!6#J~`r{z-d&feRXC2+?oaBnmE#`2PqR_T_PxgS#_ zAL+hE%US&^xC6>h0&Hy7oF5WQT30G^kfo&ESqico@jZtg*!FbzZG4 z%ktne0T?Q4zmWG+=5z^u7ep(ALu3Aa(l5t6rojjgb+(?zH56@kt^1J_pK|hgHD(}< z%EkO64Q^s|#&1dhQ?u|nMD8+=#%R}&INmZR%G+*&NmDr}S>UB1)oLrxx3_v3bneQ{ zjB5kwZ|=>fD?z>GWSM|q4r>yZ)J2!7FK51O>qQ$AD-n$7EpSdYBNTn&PcuMr}SutFbk>zf3GMwop>r(o-5#?ow_fewScWN;9BxU$($v z>r$4LIHw8G4d?fPJLh?lZCM>7MyL5l7jMS+{Y=`%FM0guLY4yNP$B@&J7r5K57)Cd zN8)~1^`!1N)mT__4Kuzdgc?z4Tw2<;lU+#{!L4}d#Q<5Z;H=~ZK&_l6;t z!#K7?Q#_HA&AFArrn$h@{0XgGR=gl8$UhLjU~OKOVj$w+VemFIDV%p>j(#T{?uz`&0bfe!~jZnM3=Cny@&z zMYxl)Pj%D@&Aq8ORG#l0D7JkP9W(!^_>73tyg75|eD2uFnB!|_x~34>9;dQEd6u-L zPWnL`oAP2_+$^;YC+)jMf8a{Z%UxXzc5L3>N4AXZO=aXBVph84V|Lu*%)9Qnyo;(m zP+!Us7Y8{Zf$DZtg%iZkaGh zd-ttkpYYBx&+k*9O;PPVF@eWb^x{VAXSD02{@Zkeh}y!-JaUY50I#&0Pc2R9tk_wz zc{<+)h5SERReH9cR-v9N@xg{c*-O=}u3}#ryUDA#n`i7Vf06ov*^Ii`uDM6*sb#4# ze&?AsL41j8A~x4Re|BSUtwidlwVgW!+rI46xrow1Z!Qy}C8teu-h#=x`TmokXMIiC z@vlqqCYD!>$rjYirple5W}sAE{Cy@Mj*dBkylUimH)hy*D(;I8-{@46KR0f>16!TAQUKwF6(hjI!DJF5IxZAv{=kJJ+?VNWNjI-mKoVcW9+MMjDLNaCvElZLOq{5q ztcN}G-g7u9SDJv6N$M zOte~kY(r&O!k;oU=B=3<5O|e(GG=|LboxW>7%O%~khR}!thLcx{2r16n&VDPFDxj= zERE_>`wC-g}#6au38r!KLxHr3@G3IGjm=()ZX5Nx{6jk{8R z12b3l{5SY;$b9uRjc)F?g*i`_-TmBhVC=f{qMp6^x<@tiP%(W`v2dxUnC4AXD_V}* zit+jByJ~065&1%o2!pXMvAgC;#y?)khOy72>?Xe)L<*i@NHKfY4uz>{c9s0tywG(S zIx~8A2ra{o6FT+l2E$3I<~!%0()UDQzkDuXNMI{N*CXEDE}l@sxW(05J=HfPmt0!3 zXhv?F-VAh+WDdQgi-SZRWAE*)Iu#Y)L$l4Dk`deN1hq@K67Rv+Ud~ogyuBBRHM1|L z%3}Ik%%v9IE1!!(^4ut?a8`R#{)oukMoj{*D#OaHh007EnG%! zom<~fIpK7PkZwOea>Nv6XtSQEl(J9mmX#o8^>%G zM$lJZz1>rR!AVnWHjg?a6%^ew0aN~hn6iIe4%g#<@FKU31FHQVRi+vvQd0urPxSuz z`*}7Uf!j}RP(GZuum%&oJ?3M!-!+hV_%AJhFV-Pz#pkDALKf`mDD_qFb+9IFbEVzA zyEN#8!vVpmYua-+djV>ba9)@-^0)Fu!emwWuaUDOI#fPCBdM&YL8}PKf{2^ z$c^U<&uyi=*3a58TkJp3@He{5>ug(IZ{?zN9k$gn&yr!G5GrL+@h-2^DzK~fOV{YS zY?k*5L$Fxce>U^5y6?>BcB|y-Rm>GGUXnYp8_~cdTm*Z>&ptB^wM)V0^A5YW*pEp- z=^6))HHx)aa}{1(39-qiX05}nnWn78U!qdWCp3YP>eBBRsB$WF%&yAIun@QR zAe~{BuTcULvo5DGxGp$k7#9l#I;rPKv&KVVlF(T6O`tY*o*bOHvS4>(K~6KvlNMKE zof_Qb4BH@`Q;Riz?0dK=IIZZ;`uk40X)av5vhkU-)TL2dgDHv`=787wEtz-s;yV+u zohh|n<~GN=pBB{OP#Q417_Lz)b;r@hC6mtVVkLk{tM z5;Xw@d4P34D}IFZ*ZmuqQ35=m86)L!kEW-#F4%P1CHe}XmYtnV%PxWcfgN_IrY1DI zak+-PBt?RpoGFd2(>Y1sWW=Z+;{fKX5#^Ap9UA{&{I}qFJV--CaT?sITlT^JS4IZhqY$mNPy= zb8J5@%t>_m;|LcAf?O*#s5y2OSkync`=*xSOelK1_Q>EM+QV8g$pGS(Qxv{+XFO>! z0=C~S1LgGAb7Xu+Gm-Iv@L>u>!f&aMUUqyhkKM%x{L8JR#UC1uZc16>$80KULKtSo4(*6+AZr9>D20gpgR6t3IQe;T`8RE5pU9lKRD<)U}%h;$P8s zdv>f3{8DK|QSBYb(LQ5+DAPR_qY#q3cdO(Q_e0Y+#CLuuZWZtrdnFJTDEf2ykIBE*|baim9%(f=nAcW8ZajJz2En04_k9_ zKD&G}V_G0akZoAD$II-Lv4)H52l8deRpEWcJn%>CIg}es6Zfs_RYxGorp4-3ns7sq zpf}Uq-QU-+F_mzPp2EnyDQ0qvB2uX-Xa-=AyclbNsqGaC||Z8Kg; z_h^M|2MSPs7>Dx}cq11Bw0X1_!nJ^*J>COf0*pQZH5oC+w%LG*30XHks{JC%Lz%Tt zyvFqt9owc8x;b#kgbUU2a`7s~%*RMM4RUkN);!uYOFZWr(^VpN7ViEeuj0fN>qvSR z7v8FKz4DC>MZb}gKEY@{R}0@6%d86@8<%Tk3&2nIMlaEg$*wQa@6)f6k;UAr;+#lB z0W+1j&~i8QmuIcbkX8wa4grD~o~CdrcZwC8eE%hcq?3eITu%t3b=uF~X%KkrHdiVh zWZ2E&-BfKq4}twDkIHQ2zn4XhmcDk3Xi|CYUq8ZwKeyniMP}CfF9HTc5m2gaE z?uRCcFPXz+a53_B$|p`xc6pW={8`E)w-z5*=Cl9$V33FF>2N@?Y|mjVZKjo{v>ml? zNGbO}$8;y@S_AIAlt z_QoIc#IaFc4|C7)+`+ZQ;)4gODUsN=d1)0j+}gcm5>@7k$@?4S(~XMWzoa~VEKgHJ zPX0{EO>=F^8(CXPCHnWe<)PPrHnm245ElNmu7d?8HhdQTA7k$UPxbf5kNbS0L`7s& zMnbZ)vdJh}$%qgtSy|b+TnZUkrI0N`cG>f)tTL}XE^g+%_PCduYyHpr>NCFI-}nFj z{~q`8NY}%??=xQKHJ`8N0Udz%FMOK#B-ZYEZ6v|}+M1Hnl^N@H`Mkn{oMnrp?CUti ztN?~I{br*($s=pT41$|(x1*K43)3dbWB;;3; zS%AvHzm^D$`w{b;Iv(&25K=c5hE{6~Qk`2nOmJ$mp2_a_jZo$LxTn1ym6;)K#Y z)un)mJ%j+MZ1I!^s7=)_%M*>e4uQEdKB%uVC^z{AJ=uT@sX96ozfXl;!>rc}n zKa-;R3dIDMDp4!c3ST%vg>%3i94)gx$;zKDRL(RUX>1XS3u-98aOKLP@E<*^^}?XY z$&hPu9hA%L%vP`5A~>)gGa@+zfq+lwf;gzIHeGKL!j7-D4_kq|_W=Ge{n)h`xt1#( z(-lY5L3$^E#))!j5eOJanZ)cJG{h+*OC~0fSVirvHJQH*4baH8$as#>@K7^Q9UotY zzN}?$cY1>t+>UsJ2H>`zeOKZLy>KzYG(Z8r%pIqE#ckhB_h9!u$TSOqVnT2eoS8MT zfEuc^KH?n{?;d7wlDF%H`U=TT+yLW30)4S1RHTd=muD2

f4)eM&e}XT{;@5Szu7N9 z)1@bSINHfz_V}fRaX^=OPu%y{aCG1Y<7((3t{%5sLG9=wh$UO^i) zzaS*~gaWR!Bf9J?&>nq@2oM-+e*zN_%5o+E5jSY>?%`Fn_NVW&nT3@nN?i47Z?)d@j=mh;#VA&rY7Gz!!I6(<1(Qy?=^VC2NE*vr#`t>&s~?b z8}Iw$A9Zmkq_3Y}#`+A!Yp+`ly$8OwIp~TddmX5ZG@5!ew_E;D5%$zqZV!og?N?)} zf=~a!gXt?Lmvp%X474C!GZ;PjnD9RTKRT-iq>Ef1P(Hh$rD< z#*MC5-=X|#@``co9~C^!=b^QG_7!v?(4K=dph3<^&1o(Z`)bSi_hOFB`*7VYa*eHHB` zO7~i7zpDhQDgPES<)?8K^Oxg~3w>*5JEo3~)*OFofXvr+2DOlW03nDHB(FU>Oh~Pu zgxRkTnq>m`>2FgXbQCDSs_*Hv`0*_(@QwC=iS0F~)bCz*2ak$jYP+K)`B>s=tM@w> z{G@Fr=4bDrECM~n%dKNhr{*$G6WRZp{jm#A2_~5 zQIIV!5J*Y@Orc4VdUoyzPt{rL_hk%LC!j=^9!D8?`CXz-3 zd-wbr(nk%s3dY?s%2=FSx=)W!1P#tWLy>{Kks_&5s1^D-1Ohr-MDQ#gY(7o@)7fMt zq+2S~!bbKCG$8#RVUr|zz11%G{=bjTu|Jho<&`Yf@5CB)f#@4UYON!C`j2S-uLlqq zT0;;|R$G^Dqray!{G&A)ey)X?GtDwmLs#vXy*LN7>-fY#LCLic|> z7pggu)N>X1*55`8B8~hBB@7__r9cb4g8s9`$y4wTJ(WD7 zomJjkM(AfU(9^$$x>c0j*&5%28Q3;>;-FBNo2@QeWc@6BqYO&@U-k z=la0klx_bK9rbjTi0wavoY1$pZRaS$Dpi`4*Ug^crtQ%`@sM$sz{m$xu)jdrB3fyn z9WTI7;}5FPH>s`dZ2eXY?DwZpZ;~DNJqdE1lG?KlxS=ijQ#*Y3L{YPT$+ICz9%a6W zp2WFnADG~ff0eNshQ>K35M|g?S)PR__%K21#=!+vU>F!dpa&AZyXNn)mEUuEU(=#* zU~F;AB4t!7NyqnwJs$>{@zTNEc#PfHKqPR`pzFOLPh6i-x#`CeKlTBMK|M`3{&A-R z^R=N))BoDC`Nb2k1o0r|;Lmf|B9G*q&K$(tSof6^tP|C7rJeq0>DFp=cXu?HgKu=| z!G4xTm!+b`mTq+Po+3hCJ+uG#sn*sjC*Y6XDT9fV2x`E9KOG*%OHZ82ypSW=CVYzZ zfmG4J`@d{qn4tMWg1v8xIA8#Cp=m-DVynm5j2>+A82G`C4M-05c5a>eYGI^9x{vZ7$^K9r;P=4%s+vPL z*rTzAO53o&UGsPGog>+k*i&75tJYTdU)wu;*aMrL;~_LmOJbH&QpEY4qrb)T*@MHw zzla%o+@Og`>gwGW<2Eoz>6Yd9^NtOB`Ir$I}W+a3n)k2wU3Mb|qjy0g3UadrpubTvr#IHz(n{jSHV{eK-KA|JM! zZLIC}Xdd}l%>Kw|*r_R5G5N4k?XOdEU3;iO-zg8JgY}O2Q>9x2>Y^b4AAX;jlmY}! z#1-InJq2-zbS0`G2Xybv-ySy~y;~o|$e#LuC%43IFU-3PqkKB03KlAcf~SAj-oGy6 zQXa5%^-K?u9(NhNq7u>^VuNg%)|^L<2Om;-P}kAJ!{h;NEZ}AYhv6vTScP0%?<-y%gn6xL^-mGLvvY-Tg%4mg`ZRCYvnhIb1u$co$Y(8=4nzMAFHZ1~B5_XUNR145^ahTcuq zsIZjd4c3!dUGT~-_L*IU7n;^?E6%dpR>c&1QQMQr_jW|T;zn4fRFTme${axb0MUlh z0m{?*@Pr(n*e`{M;C zM#~FDS^+1h#ifC}Tcb9SRSUvX~CLcikkP=Mc#fYYyY zR?ZY+hc@sj1!O1{2I9T>BuAb4o{QDm&QBH@|KyME%qY=p?sq37|J?9N&sukqk2T}V zj&0qjLBOBf)j77fh3u@54M?^BmvE%9U*4i#CM5%N|3pbqrrcPp8}1 ztOI0=UWD8%P>g2Hnp^5KM`CWJA(xlnJKsNW;1A4Yo4vJwXs}pMp(dOa^ZotTHU1?M zkvf}JsW%-PU5HDeKlt7m-ofb3R?Y(Te5ne{_RB7l&rP~8Z-s8NJ-X||XfS<1iu3;( zTG~WI0%y4wgV5By^i5y{@zFbx$*3-t;cwm@Olk0@k(~PUN(+7jPk>+BFM*KOjscx1 z2b^=^fRGImB~&YRg?#U_@R@)^#~ZZXlNPV~jk9S{?;L%aN7ytU{hhMqtgSA?)1k{> zW5Lfn+nty=BM&%j$z~H(P49+TLz>tB8rcCM#qE?mqVymxIOJ|+Y5nb`4_>7#*5kEw z_=C*lr^OWVjaT}gxF5ra%-rNr&`ZHTzv*`GTAUw_~}DyXojIMMxn(f#5Q#Q929(%HbQ&`LOd! z^lELWOFmPwgN}@IRyZfl*W^-5-R8)HVHbo>fH?VP4&hq`&cGY)AR>+oZH;LwJnSe4*O8p(qs;aH>{_)-5L6)?}*VB?HP!N?TQ6?Pv z%+vZW(#4*eZEr^@vFcF5cCsD}JD5qJYdR-W?7b^R>{y24SHsiegF}#k?RU6dlRt4n zSIp5bT6O_bR9$@Eh0P{yUY%+Y?&s%+udI#gMx~^q^{ZaJL1Ef>2cL@T*Qjiw^{k2!zfdhyoHVg+AhOx$96d6IH1r&6p>n(4C@ z?CJBnl(1u@5aBS10SVTddo}K1jO>GYV5y+fXW1(KHB0gQBrXXf0~rnHJr(=_=0a$_yu*BaiQ)^cR}2|D-{jRns;% zT_GWI|Fr-x0{PyLUf~)uQ@;_pBQ{B!p7vi#MTU>V5amB*lk%ld4)+wI-41qEAGxk{ z1M4+G06y{ujaTb!+hgD^{Y3{xSN!d<)B}I>*&bd9oWWielEuLri8Q6_secM zW8`%D3U7SP^7hY}*_CN4Mj~z0T<33~SflF||I;i(%y!Ugb#5E@DZeiLK9SSyUu)!{ z=`yrh>$UH-JE=bm#>g0SO#!I4Pt?*@XmVFdeEW!(<09|Z7vL$g`&K624ve!#wjb-V zn=DsBYV&Vz1sJ5mXpaM1>#YEu0+CA4o=aP3UvdM%jG*j8RML+c7tEMY$iRm$ZV^F!V>zD-@!5PC;ndsUfLaoQ@h-ttkRyZ)jv40psTbB zD#8o8kL~GV=)Nm{K`S8_ab>i4HVso7emICkz_)DGzvJbzTN$2zaQV2@y&KB#jL54LWA+A1t@gxWD5nKthvSu zSt{EdgrkA?%H>thhvGHiY0?f5$b@n!iLK?}vHo8P5} zhw1{M6)V~0l<`CEQMZs<&JI2CUj~qCM0%k+aUkiirv2MgStpq;6Kr*_Uj>_o?X3^? zqsN2i&8(s&-k4u}E#**ZdEhxQvUQ^I)kSrsT0RXJH&*hkLpqr{+3G%6(~o6D`w&a1 ziM!e_xj@T17u|6|Ux)L7Mlrx8%(g#^(8XWbm`Tk zTLPJ2#BjIGPe$Z$osy|=hUC^Xr(%y+YmPJmOvUnEg{f|7S{#dpNFzzM=`jVr-rB*| zvI%F7Fx79ye!=zKhD{;u-{z;5FD=6L65f_}XPZXW1(?TO$ZRo<%L_r|@Zez2yY+;qq!dh$!|-XIMnVtB8*aep|xc9fdfY|ms& z7 zW7a3q{4_p>_P8MbZhxW-r0EPu3?Wpa=CH zsUDojE!kB~n*eup{++V|J`~Wl8z8yF;oNTAR}Xr_*+QA`K_UIIx35* zyNKNuqO%A27)KftUU{*UQpX?f=@$2Jab1f&OFOBQFkGYO=x6_0P4(_<+o7Y2mY#Rs z;(wWvb&>CF!BNSe065gr7l7E@w}J+hp(0jL-)Dp(Y}zG&H08e!KHLzw`ahD$qXHzs z24%HhP_7n=@*M9f&G1U2^_85#u)Nv3ryP<9>TC(@75YCo)I=2c?9T|^t$wzZBEXZN zJZMk)D@=o|TKM418$PpH*Xoqu^GqypnGcwQ&!|(i?QlhFoLc#iOUyNA!q1|sDw`s= zAJlH{j&A7`ZW_V%{_9{NjVSiYH*}LPtxT@32ZXOaj|pjQbxT?hq=Kdc>_#9khNJ*x zB$z?f#(4;zpYFWsz`1!@Vs&YAEIJ|?r6_o}&AVV(>sKlz5zflaz{V>bR^Z+*~(!Y_*qlY8I2odMdkg@*B&k?XCh+B5H4+| zxF$4?7)A=Q$&x3d70yW#%~qy%J&C^OVkO!h1j8C5sp~d&W@eoW zb|&z%+2+D7mWj?&V+cqeuiq-Km0AR*kJsNr0aL+XQSiyhe`7=PlwQOBOOTRZ&kqs! za}DVpl|EGD4=Bd~o0>2w3EXogVDGdA1D>^T2-{{@WVh_u$2Kq;s*wG=mblFQV<76};}MPJ6XZ%KHqRIDf4KtFyK*Jj&4uKf_Uj|w*ur~{tG*&O6u1Qk2ngBe}l7y1X;Yv;OWT}U)*@R z5QRLtz}F_>jbaF-L)Wd3Y#(#?*EPQC6TPxtm5#P3?JmT9!hcNl3QL;jCz<`geURcL zd(D}s{7G@DJphrpdCqK{b>+5$1!_)>Mn@~1ee`M%IP2u`)5^0swGwz0)Fs%{hXIWx zSBb5Qn$JN7jLH(RQ2OpL59A(#q&1p#Fu z#eU`{7hRVph|R?BD8^QxI@@$KbS?r7VzS;$)wL{3v-ES(jU zKkd05yj2l-y`*Y=A(sivK&O)+40E_tUn_kuE)6amIbR426<;^&=68kjPWRG#7m9Q5 z@4!szJWn$$k6zok71uW#rCEoPMAvFr7_MXuBpLP?-LcSXCv*lh2#sff9BJ2*w~vyd z-4Bq71T3w9F$@%QRP`yh)O^j6NwT?MBg)TclE+UR;V!c7g33V)M`loP)1Y+&ifIh^G|5;!)oi$P{s%C@ITk#_Tz-D=#r756J%j;^+^k5Tqvl;y$=WX(A>`knqtR$^E=S0oL~QSRoU_86M2O8 z`eH7CL=PPKy1~__qSPG~Qq+p>$YG8Mc{n9ju0u6kY>aI(W zoEc>jnC?8I80@__RDT-3Zg;~Q^fT^&*E`07(KAy$(5<%UOe{D{uUM2Bk=+fzXNco)ok|tn$}p zt|50mrZCOZE!yFu zBNnkGS!nS~cm_38Ey0mA-523naoke|l_KUDxg!nGKi;^bb7{aufYASStu}jP55|NY zTYk2lZLbQYjBgdhJRIw8M#f7s;FIb&7nT9MoIKEukZ<7||2Z`kcpr}A`ix#iI2hg2nn&Xs$wZ}}FO3F@>EH;oNr@p;k);wRl zlZ-&Qlu=}3qOUM*2O~?5hg#nNtUCa7=FGgA7ftZ9jbHNL8)AH~FwYcE_0B8#I1B}E z16(3qN%o-a3|=yYF7Z8e(KfX#Ib6Mg7WBC4Aqh&oUOWri04DZ87OI_qtZ2zgG{}nv zwA%(HfuDmkdWLLCR%U~wjSf8pWztTO*8||u4*D8mnE3=sW%YknmEF1QwB ztE2L|adAvv1cWZjJSxY>Rl-Vz0lTa@Xb~ZM-KPi~!M}ho-)sCB~Mv%Nvl^RXg68NCsi)BQD)47tGtQspb1 zAH}D!Q1AK3&BpH#PXg2{dj1eDR&@R}*HENXjg&~dbDB2Ab4(`qEGPT2{_DrzUuua( zgxyY>_%xm)a3}tIR43HE1)=`#CY%qXOoRRhM#vFtA2^;Tj42M26k%h)ImmHf9rblE%(tZpPvU#h$>B@uc+d9;d}N`$zFg(nLigTPWaW~m`vl6LZkba_^}xx6(5}C zFKc)}r`-TG_-kYlzd+4WV6^mrN$&?}xdl!45;E#Zm+s7Ew}PHR@b(VijA#b!egL$k z2O4J!OzAf%(8G6X^|rDg0)}GI$Vh$6zKFTixqLv7@j1pk(bhMTVW9vYpEKJ8_4LpP z8y+M$r$sIv5pK%$aE3THnpea)xG{o0>dpsB%D;&nXeS#7Z-J{!>$SWdrbgy~o72y& zq-F*d*4p9{bXbGV)VjCF)mRyL=DvGe%zoaWbSY`o;=|}->qn62cFZ-s!5=-ljyRgs z?W+NNEN{oXe`{a>Lxa#sXYBq$Cy5&VCpt;aP~fL;=%R0EKFE(j2n^gXB@x)u8$0Kr z#$x6dTQ8s{UC`tA3Tm;u7p!0i*5WM?J7~cGw$z#e4ne4Z+(ZH1k%yp9yfi=a>PDnl zj$xB;^Gp2c+?LAe{%U`%n?Jny^$LB&lpSH58|BF-az!Q0vPD0X#H4CBXY|Y21Wfp) z#nF(tJKxBfv%Zmi-nu_Nl{<_NdcG~aHv$S zOHecT)UXW7ykH|Wc5SKP7f|=0LjJzdRU+B>biY74(92L@vbP~i;vLro zRg7of*Q3%g#u{0f_bydEW$h#T!B_dx|**xl3>_TIxno$9bZ~$kG1vT zU3MD228Mj^<#LhL{f&>hH1eNt;uQYaN11W)G*S(|xDfYl6-_YX2T_s~~ zA6~1nayUoO4EA2NyL5{n0DAK;#AE7M0EZL>==b4pVZYG|`tn{H6n-=OG#WlsQ{LOX z-6AC55_qD5+lk0GH$usa-R^xtU;#crsD`l9=a)W4Vk<9B`S{kJzj}MVSdL!Em2ZwW9MbPL4B25ud4iX?#&Z`jq@*S3uC`?_ooh^K|DuJo9IU zNBT_2R%G(f%9Qo-*jSb)npD%IQ}XwEcFEwEK;x7jiFTrmX4eX^I{@;U*^jTqxFDFd zxLr#=$75|*Wx_^RyjN7a|TCv2rYMRXF5%0B!6@h~DV#OA#sM!jpOC zj0BE`bau}{zqBn^1zz0`yE zxl!mH^_#b1qC}y7?mE-~uI=u=btTu`yh84#((DBt zk;g%IRuC8*gdPogF>ekbia&s*neVshd9%Ic4Ja;Tx)dA-KSO4r^c&hQt*2+Cku(Je z(Fk=;KRkn5juytEew$OmMhdD=sv0knaS zH(qI;=Sl-w+GZ%o`3>lR2b3m{A)XZg-*&tXgXlCsTN?{RKqVvN!%QObP+YTl%l<3P z_$>a9)f2>T4{RW14>2JuD?lMc#YT(hw7DZQPwZhuK%=8wcGan{t)pw-_LAq5Q-drlbQ{OSGIuufRS}kz~EB_ z zrc3ze;9bC8XEsu<0whQOIXhVgn_k5j{89wbMRApG%i+>q27lbh7AbZ+U)xo9YJ?8> zD7>Jp0uU4a*sy4?!2v?5DvxACzn{)$h0#8V4AmzWgXe-=Haldg@c4soa@v z>HSR8qJj+nC`_#>a@1?n3Q!51S5gF)7=!@OoseX&AsMWq`^I2#XVUk?8ji#LuvkP! zmfkx25@tA9>K(unO}_uZqRinDBe=N@ia-H@6~t5qnC7G*quwgynOzI*OVfO6Inz88 z+&!g$D}CYpkNEBuu{qP_mpV@ab+7+tgfqBI)&fZ_*uBHQv`Vcce9wXraBA5JV-_I5g zi>pKnNQGy5gZAakQ`)14CG7RnX!4^0W9iovTcu&IDDEka)kGxE0l# zCbp9KmGKlR>uoAj+9Xt@3kwNnZ_r!0ub9Y_!yC4&G7lsG?UYGghju(59D&%%lJ_rz zQ;EZlUSN;cX~(?zSgM)rSP}bi=~!LzItZ}VNSzb^J;G=9eo%h~$a%=W`(qnQa!AqR zE2fLvjkgEaB|&S-^!s<8{62xtDxui}QnmeW!nSsAnTQ6{E2xK7buGo%C5<4|fZqsQ z%zwlOC^ddsJPQe}KsF@T0(vG(Ev;zC_vp1~K<7@21@a*D1`?eooU@9!m0`L$ON$S@ z>>GgCRXt0-N7j~X)>jEZu_1btWeF`n_D%CUX%Y@_ic?hN@*q40TJehnNcjnef<(@0 z;XW-Op8a;57!xSVfjxHHI^Omr^l?aI0B(2Q#Y z_=z)MuC(~?2Ue>cjfpk5y@Tb4Udc{+`DGMTH=hLs34l8~k5iaBxFnpKFlWtqEb?OM zY^#%g`IKI{N3N-vlFL^L@@6wWlPA1C1KP_f9uFAibk}#MV7lCL+*ZpPwsv+nQ(wIy zCqE(0^J8aIZZ#({h}pH*S24{9v7KCGrFXqYCSfq+jh&#(=h3*rj*7fknao~Oq#I0* z5(ZuDKo-DbOTIVAP|C~E#Lp0(?LEE~Qn9DtY=2fU#%r5RARu+ZAxl*KPs}!|bdtVn zlj)T6CM-%`k$ih-U^;n`^WsHp_mGbJJi5WFE;TP7T_7)D&xJ|!#BZj<`i*X z!wB17__BMgH?vwcOtZT5UUf*B$2>Bzcm)>m#bANDrNCLLLY1ukv$1({r2%k78=J27 zGDfq!mXGltm!)&hWu`LV=VAHxK2&5s@u};9ALsQ2McgU~Y5(OoOeu}_{GtcSOK_S4 zyZ1ycYUOUD_kP(~(XLJBvlawc6wXaN86}V^66OZq4#=$+!s;xeujLzdzGv4GM}~${ z7Po#UtYbG1>%!nTq>(>Q?`}xDFMsBo5-*l?COEJv+^Gi#Euq0(kqnXVme1fDtuHe# z+EW$odCi-&emYY>JNuTU{eIYJDrcXGyK75a77z?88U0J=!TerhSxkVe2zH8!fw?H$ zvFz6@?TASJY?UPv7a{fH`kbJyv62oE!2Ev#UUD~q#tY( zj|+d@^GCC}nRVMfGVgX#RVBw%pEG{dTD@O8IQ|xhp`w}WB<$1yp-p#a`2goDKMo|4 z*jG+3&xofGek|)-`0C0T{KFJ^v>7tIq;TDf4_kYY^6mptL(YW`Y)QI}msZOUZM(1p z>(9Z!D4yPj?5xvOCDW`(d>#voQM#}3msmEbV+%qkfMCM<&S=Tkkm zGK*VmpnY{%jUO@7}4w~^+&iD3~u6HzkZ!O z{ngxwO)S+l&(Sp! z3rnZMYt?riE+*8f^920FPkn4$=_RSzYs|giw)djSHe)4Wx`b5D`Rh96#XXknPm1DS z5Ik)5?#w1ItV{)8j@*+UR3UhvhZ3X7g}l%*jPL+65d8gKzg=!@?1*8xC@RmITb_Sm zWDy8icI`E7>j)Q(*(#eV`(p~B6p-x!$DJQL-c0lvTrWaZM_aAGU0n&-&#~vRjk+CbjQ#BU+w%{%S+cy#8Pt_av?KKSUgm&7T)_%bAasKB4+V7ZMD zB2@uTmmJh;_p>CN*nGFg&UdhkOm0H6CGebuxqtl4>UuTG#Gv?isj+{m>8YBVJBv*-5u0 z1S=n*j=?Mk%En?6^-|tjn{1u;5EDE2L{MpNVBfv6KCmSrL(CFpe8XhQ^!lq!{EoSQ zw~*hAmUeCz^x+SWsVqmgo=@D}R5EwjJf_tXb#|n9HIW9kWl`$|3>- z_U)q{7dV$YwQ7~$nTf!Z#vuboGHd*I&vQJIoR zMnzmnr%1c#X`M@U2NR&Ulpc@_L zfcBl;6yzoQ>r~Jf75`kS*~iRY(r~WmJ-sbm)Q%w5wdJHAY~BbTI<++K9E)TJ8ux%O z&gA~1qq0U2dvJHxr^ari!#qJrQ7wNmNbpi^qO3gI3YzY|O@~fJ!(@%ad_kr3x742R zz8X&n#yAA|-uD2xnvF~XwyPXB-#t`sgA`8KjvLb2VTp`!Yj{qJx94%~OPS;9cY~?- zpEisumAwWANJbWPHf1G!HP&4}?&8hpsYQ>uW4hh(>5BNMlofuK8>VP@(*DrIc%3GG zq77Q53dGLUaU-dSl`Dupl)3}cqt#hN#BPX}y&abq9yAU44wS2C;dKm))wCRwdcZ&{|WWI z{mElXvG;nJUmDjFy>eH@zrJTGQIqZLS#`*Iqy_BH&BK%1B{Ys>nz9ieoNNHHKQVfD zLze0eQpdCI>o&YW+C;_T(I}>wy_i2IKH9ByWY>3(9K%A^X#PlwFwcu!roZ96SDP9z zAqJeIy`AR9O87456>Oj9)>WHCgykC=5Xk zUwrO3zLFmxe_nBqG9Z;k$=`nWqn_n<;Y*SFm>A2m0>J*ZIpmwf+88$FFYCP?O^o@% zOh;d;b^J$!h!A({ny*IGq&GBwsY))xiz2{XY3{lBdS1GEY|G*C zeG8-Lkh1iHI%@_DVf@o084P)$qPf$Quaq+7`^fTqX~k0CuG-`s{24rWS1X8ZZ7@V; zwbCfDS0&$L@lZ{0yi>{}re|$BP>_yE{s0<%z~y1s-uciZa{wcy3&nq1Tv{wVH$g7X zKTlI{nX}UxF=LiWu=Zp|ukm7z@L~_72q%f_y)=zl$Wk08L?>GN9P#EPX8@hz5 zXa|lIO2~FMw0Uy?=(m}Cm$wkrcuPlB_n4QW?$YpU3iH|(P&7SoS-D?HANZD8)1o>V*R*NepP#qrjp`N+S;ayFg8nW;hw;uiQFs(8yGIFCvQ3rH#Nvyu9 z^SqAaw^GN)uO@L|DPr`)k6!&z#+V}B-tm+QHgtlB&&ZJ@huiQ72S9}kT8C>$&i4sX z)=0wbj?#M0Lu03jL?6@GFZw8rzxGM|PjT8qvjn`N?c)OpH@Aduyh{{+YCHT%z|I zv&xY>pOF|xmZPiN8vv9V^MM?O!z&ci@5TwgJ9k^D!O4}5he^_AEpeLqjg<86gNDTr z8#QxTIoS=9#^mq$JWS2zQq`BY#(p%vw$TxK=^$JG0X;7kzLO$TT7~{m!F{^WCSJ9y zszf}aTC>_?hx7|Zf~@!A{w;}4`a;M90q(Xr@FLHcl->YF<5*VwO*(MgU8|QzJe4fg zUL}D>i3g+xU0MezwJ`y1|Ee|K?cH71-ltAyrD2|DrTskZn!-voITM`=({gSJ&wCN^ zF@8BZf-7FZdu}22_?XZ*w{}j%RyCUtr@vzL<&x!@sqdvl0${Y@O=P_@W>4W;J-Vo9 z=jQzqu8ii=qUp?v-N%-_0CvC*m1PfdKbjp=LZbrq2X_IX`R)&1SbKYn)*>~~R5G`m zG^W4nY`(jRS$XOi=lfNqD`nl1-Ii(Hfoc`^YGb8#jzjzT?;|IS1&+rGcMwp)S(2u* zXYdUZw(E(UijP$lmVHmw;|LfCn%?AzEHpnhz3MZ?>FE`!C%tkugN(7bQAIf+M>5MR=h#HYlnsuO99s&X*E~=#flrgMFCe+95Vi9 z25AX#UXH;)I3_0$*<^t6G)E_Y(pj6xVlN6%xFPQDv{TpkKQG7&=#A)+LKcz(Yq%1a7}6-uz^x+rb1at=S@T_QR#dzVfIC z&}u_+{Uc(U8;DB$-db`!dZhrIi5Rd+52ND4~VC2X1B=ybES3H+fZFU%njW-ak@oHq)6Nj-NHO!*6!s~5!Z6V~wQ44z4 ze^TLC4YqjqxmCI--)3V(9T#I>-X8Ga7e^Hzc?t}iyrTw z^!+qGVxFPBJPNe;U;hzmqM6W!V< z$VryP9N_>^%5e(#*R*CC9|W)*-#07pO7q-wE0Pf-Ztk;aojM&n)p4qoGlz>sNp$4L zJmIppTz)*{p{si@!4GI=$QcHv*bikkn539;9anZ%{Nlv`I%OQ&dCr+idEnH~?t94quq6L8wpNgTnD?*U(+HCNb zNOObxMYUiWB$lop4VVZ?_d;j2u4I6Jhm3v0U{{ym)xrzF3+s~bPc4;g;%+CwLW8KB z0u&DXa-IOw4&3;+H(zGxxt{251=0wK!X0mG3Vv((Aw5EQg0>O4d{Tr1>NY_{5}d`H zoJED+od=6~_Zjiy0H6`X1yu|Ex2#eF6cSwOW_YpOF&URLSckeT$+62YF0vZX3l6!s z=J$+Y8J8O`T+omr=(N4wy4+FnEO4}@=Q2lvn5<-r16z^cg7 z;#sMQk6M@kgz-;u(sTtV_Jq-reZZ)nV_;Cv%(!T$>>vO(wPBIy+%P;ZG1cNW2)poN ztwq9gzJu58UM{Mp*z`1Uofa0Ljogt@cS!7#xd>Ef3{XAaSJK^92JM*^e#c^up;!Zm zU<~R-y(9R)!Illl{(>{W70D?LB92RdgZ8f|&8y;~&AzWq35m1o+LbBKjA2ttr|-jR zVV?IC6FTIs0pBGa(pVY1XV2u6b8%ZIHZP4XAlQ70%4>6MMt!M>*F{=^@cX@-*eHP< zevn*Re~Y6kG;S<cLmLPIO?Io=s{-BV)5}$UuGN!vQ2JKS&cz)HOn?UmeG_uz-1)QzXO@!^ zoHokYQtzE{6X0$Yl>$=D1q>N<{A-hK6yHVzp*aft9o5J7FL!FtJ+NrJ-&nLW7~Po@ z0TUKbbs26;f|yf6$rtil!&F4vNz*#jy5bo-F|2>GEOWoI6mwL_EBd7X^W4w7SD}4| zq9|ZqL{oUh%$#FsG=yI_*$JxCY5zzEa>dgn7#4f8Lj)SI4kt))y=iys*5JCX3&IC7 zbz#`cE5SS|vKTc&y+p{+-pr>%+kV;*i5|V4=HH-S7ZH8o(NboRh@5(Fcxp`hNgYk_ zVJHrn0S5vK#4Hx`RUby31{P_sTU+uMBmPLVM*kKIFe#GI8q89drh>_zUj_m;lD4QY z)o_z*?%i9KMw~JuexLELQvfh(v=Bz^$u~pD=t9}8$6Hdyyyp|Ro_9U1ZNmJ}4AiDP z*1k^rI6&ujq;!i#=1%i)NI39yfV+~3Oq2kb>W8;sVNG!INZ_v|{0@`IUV{)x!}5TQ zyB^C6`7mYE#M80!$8+9E?zZr@QnR@V&5iTPkklJFzf&!CoV$MgwDJuCw{a#LCtn{u zGT0@(yo0Q<;KGR|xK% zYP<2KtwzAE(`BTu3W@-nR_(jKg=uCqX`_fJGp){v0bK@J}90WX)VdF`N zYE>Wn*jmFNy4AB4v9frAe~y~HPGT^d#!eXAQwXIqM7>U@#rQ29S;iqu+DwnPdIJH-u?fp&HL$ac`HmwXUBLs+)+}%)fy2=X(#zmnl|cFv$kg=z z)7yK;Q~kgH<0=(NMkLuH$ttq4sgR)_xR z$6nv-dDQ3q`Fy^=+wJ?`Pq%Iy&Urr8^|;3Ux?hinM{Rb7mF(yU(mKr7RjHh$P|(_t zfd?I6l$|k^MR|)5qIN=adHZE(EArx3Ul%n&ng9uFd3NHuvriocO1cU6>BF8kM_ z1@Iq`sbqed4OXxGfvtycsTSVm7=G*}fn!c;63;OhyKH3CurHZX^I^kAKPnsD*4FhO zBvj5DFi`Q&s$jtQq}?=c(gFwq!jpc_H4g}wH80f5D1pc(*d-dR^-+<#70jRArPxwv z{pIh+K6Qu%kgyiYpdZpT&)WvvqF8rk@B|#&E=nbRWhumAxVw<}mq84fLc0M9F3)p8 zsx_Yf9mT%_Q~p&5j->+oJhm-@F}FmG`&}3ujXo_2o|W@!T!&}E_hhRKDZZcXVeZm) z;=ku(@bd@vGZb12bSqg|&Kk-%D>lVrc&Xx;su@j;Q1F#_fZ-?H3@TtS4z@XCB7vMX z_H)(qBLF9_i?-Qn%VUlT&gLw2H_I&vcDPF^*KA6Ow%aCq8`_=rNi0QJU<;6kB{>M> zj%Scm*6}gGFl_5(6=E|U1#Vuh3wsMNl_)4{c0_n)F1>9VS{8gdS3R&ZFV$cq+o`#4 z`8lh*o7%d&-N`uC67>|XJ8(^nji+A*K;s5DyZ-^Z|7JdN{NpD#)q6a?d6!3re%|Ih#f>M%YSuHQ2|rQ;FI#$s>+!o##_!(h}5qL zL*zn{IU9=1R$iFuTfunlO5ZuNB~?Fgnr-pknqR@Bzlbvp)}PxGn(7Px=A|hm%$|a< z17x)z2TTgbtp29~$WgeT@~J+I{s&%?LB*vfvI^cQ_#_CHV4@Wu}H(Ed=gRDYPdg z=jP=4BrBtid_6?8k$AGr*XPw~*2XsItKPs9La@3|5-T+`@8b>OIb<^~$<(vRsa^0z z>M22e*=KkJ29ziVNd#pGur`IMS&0hBEseVB-VP^_@_zraB0h=gC~8dFnMa2YUl#N4 zB6atoJV!5bkQO|NZ1V~G6e+p}$@6+6Qv=fH$w)>;#GlZ2$&4-UHSqQeMk!;G1b;3H z8u%r`_l`FcH1SxQy)(-!3?dS&U$3y9C#@vLcqG4^6+*(l9Xqap!ICt1SvZA+MX zaKCRE)Y9&6y~+$~r zQ$`kTJn-qM<;d0C`*ZzHj7~ON4I0riRi+EknPk@YjHa|32u1rdLuUGylEXeVU6QN| zX*0z4A`j%a13f_SNmiT*n_`2EFW>XfFW7kl;%+nClwc_z;UZfCeW(hkK$GM z#JqMK!V50&J=$y~T+woczqioG$lLOAlibO8tMD&aUgAKB;fLcw+gE=;Ul)|{o{>wl z*EWCFapfw`-FJmR$ct`gaqz}w^U9H7Gk(;@gxewFYo(p@xtA`Wa_kZVfArYQvHqF7 z1N_8;0AP7ZWt?dg=jXLB&9^MGl94EP9`_!>pCR;qBG!|-RWa8&s}zqLl_AS5?{Vd? zlck1qUFM%{*y)NA8}|ei&jm^449F(C6dPRt$1@B(FWR*HnnDyKn}W*k7p(Ypmyql8 zmrJ{2wzpuj)9d?RfF1cAoHt&{mk>K1nDTUU?tUNAaiL+ZtK)oRQ^AD_`()|W(Gp~m<4vR#M!;8&K*f+XjZP8ga`k@7%pH{m}Aivb`2$Rup)cMN= z$Q%Eb5~HEdaIH-WJ=<7r9u-D4Aj>V%p3N@Omc!N6p39y_RwG#%c2f@dU+UA+yZoDM zCQ)tc+!5)K+0)aNg2vN}BR1)0VNbFFHK{ITO`@-(hC8?+r-Ue443AYwD-Pf zVF4#{jA|#}`rG)U<#$7aDT3uvQA2Pmd z-?W;-sA)AEBU?NfG|e`9{=&6TDe^7-cLivRx}qz!-dnVU+HCwyF9(XoJbEE!{&#n_}CfFRMeNdHhARr@VE&QW@ zG8L7AoYQ1;{Cb!#na@Ia2VTR4J1F1ud|~z@0IVqCitb<2o(kT-bBl`K2Z&_}e!%M9 zUvpBHtP%EsSF>72bfdI8qRv`NuyQy_vrkDOrV^ruWM$vbVBc-Xoas;_-0!nwUp0)~ zy|-_J*J%|2*yRW-7|3eJNfn-2JS zqry9AqlB}pAnUbF=EwK&=GsNF{ILTHBfLb__Qm@8AiIjZofOT;p*wrp*CN!Y7rCbl zg!URLrSWsjVBZmouFn9Q?JU`7r0$6D@o13}egrTPDaiukNW&wZ#tqpgL4{mlncwH8 z{H9KjkUcucIhAX)&S!lE<6-c-Zy+}H=lb!HkjzZWE11Q`NS{R)Qul3FWY7pj0@zO{ z+^&*M|5kTJZG7f@ZESN-CD-Ve?YGvyuExl|tAV7+Zr~=^rnj@rKe>7x@-**bC37QH zs1jaEAkRf$Lm+0r#FNO8-o&76wu0vetD<$^pK0;&!cL;ShOz~G$7os>5#9|R_!-bJ z#EUeF+w729muAwTKCF+yKa!c~zs4r0SM&*PI8va_4fJsTs`7`31EV%O>xIJ+X<)5DnAEP)K#cDuZQ)Q{p^toM5{eqOV(5LMNkBhg)u4q6`5LEgIE`*mF-MCcW2vgxaD8kxpZsT1|5>bxtkwKP9uqE~Mncd72X)QR^Q zj*h_^U<$*$CSP8kxz|!7jc=_gd>o=7$)=(p_n^Z)e%7L&&ThkE@us?Ql1?wdf7&l7 zH~N&g{(HqL;CS%_mztel`>lQiP{q8Fo-p!EfW>w3?N8FFulqY39 z-BV3s&4>FgoACM&xaM=}4Rq z=pv3YS&b^-*%0+9LTo@;vF{KTkxe>ZY%b5rk3XB>#2v%A7uJ)a@2eh1iYd3Js;v1U@N)2fM03^gY zg>WbeHqIaB5EAWWGa?8(csip$oh%PNP>HzlCN-e5w+5ZjR+E&RapVi0)DVNJ{1x%_ z*3ox|s@$sMD)Nj|<xdSCHDlY*hQMKX_t6lo}_h- zS_GtAh|sx$N;+zw4$eog%&K8Q3FVT6F~=#d!2?G$cLXZD6~)74LBD~s4#)r zr4BV3RAeAXiR06bK4P+Hek*2t^+|GwKL#jsQUx5I(ZbJN3U^Mo&6;Y#lv=MzJ^^urL)a-@!hgk+mE*GCr9rz_E7 zUXqnP>5H#t%!@~>Bhuou_I;hXe|sBU@%Jk_Q4>*swEBMCoOw2M-l zSnx|#rOs_f72A|BR0${JF3;A+e4`P0aJ4q0dHJ#5g*sX3YqqYE2wnv~TQj7=tgr=; zA%noy6K%e{W9&!jJ&4knwF>l12mlB|BAc7{KVYdoAUmAE9FugP?(jT3|7Sr|VL@IQ zEz79t`6hEC^d|&U%=>Z+!!V5Pzov1#h69iiScXpsN`;`1$g}~PDjR#LE&xW2kfN^k zzO}eqyMD)SF^`+EV| zX6?T{QxHyJP&wka-3mqdm%)?XggA@f|01{JjjZf9>LvBR1XPn3{vU)qFx=A!h6G3{ zB4B`$u>;*F8FGVZTEXtuL1nxllS1Se#(LJbx+OXpPq`0=ogDV5_T2S&+=-#;bm*o* z5>vcFy-T2XoM0y)5(c2@4N$LiW(H&5!BzmESp<}rVEh_4mvjp9bR=)))nM!L?gN9& zHGspoK+Fv#h3=V?$g7Yju$}YS?W_(k_FFA}#JDqkmj!QUXt4p`(K-UlZqYV01dyMPUHf<$t$;;T$~F?OA&Ru2o-}Gs_hj< z;`T9Ip25Rt``zt@>5c%UjC3oO+Azj?H6-s1fKLW4+YY3b6WMhDnl??8m>Y#@0eb^c zrXWR&yR3e`UI;{4x4_ltAU#jF3>eoI*2iCZTpHLM7&Ta<7s+dv`X}{^1D$CE2`;K` z&x0%eDkGisP(+a6kHHl6C42#U;*LTf%RT}e;BcLDXUeWXDs|X{9o7O%MmJt~(pnGg!w4dC#d!S03 zgM)c(^I0}yo${0K9=kiC%Os>n$Too!AB|H(*Y$%S%`JDOUCDhaKH?#kv^F-)_;-f= z{Xg({_FvlpG>+FJ?G*S25&364!u)rwQiIVGO-7;h>fPAYdly-=t*DgHl5ARi** z;H-j~P$ve8XSR?*Wk|3L^=u?wTsPSrNeW#cGY~0La}~NIbXgZ9;D7?#5ue@rD{q)T zZGc?6M34L%#&)8*&UIeDNn~oG1n(l3KL2+Pqw>NM=>esx;f6asL&)T2kMbR+@>4@i z!mTP8^M@7l$LE8E)P@##W0F$@KF8xDdWCx~7M*Lc_JEOuuni8icKWP*18>=Kj zD)*3&>~Yc^UHUuF3nE~F{|i`zXWqB`xv8(9`d=HW*~q4R;OrW^Lbw6j=Ri+(!%g8e zSb8VGeX1WfY~4pYs{F~pB57dA!Eo)PqTc2;!m>W0Uyml-A6ARtRAC?0Q`Uqx`5<~Q_X+rkxEk@V% z_05y5#nqPu!)f-?=jtqx*Imw0@|b!$BE|;fX@stG8d77bEyCRABgYas0Auq7CX!_5 z!`VaMc(F2Oyd!vRmG#3U)Ih4My^UEGK)(9?Dcz$-E?+P=$WONG~0GP{o{;LpKq`~feP@H~UnH&xJ#plFSO z5AVY|{XUtY>Eqwi2uMgpvy!Re;E8Ih8fnNA9X3RU{e(wHE*s}$AWt;k;v(Dpg>MEA zmB49&oIC}Td`--s2(OB5gj(oN`6%@XUv1Q6qV|ftg=NBXFQ=#C#IDT#J86oj?^ zjzj|@F?H?h{+WVnrOCMi1tDu=Orw7a@kqWt&zor;?E2vm6V=T)mEoVUC>TMr_i*|* z71FgF<2_zd&xJKJ7?a!Ey#2!tcLb*VZX5pZei&YHj*RN2@9M0$-|sYb8Gn~?QKQYn zyVKRs`BYbsS)AlfSGS{RRn0sG=@K>Vp9xaJ9Q|9jyg#7!tc}?Rh=5eKSx*}ApWSV2 zE4fuv)FzMBMq_`apE2C!C zoRKrM%+y@6PDik=-~GCUqx^64e{T4*JTjfO_SyV@&7ZErV7(_K<-YkJnXa=SOT zcs3>Q`sB(w8HYm?8xO|Y@R4saRq*~<6eJ|S$i%v^~*r}9fP_a^fza(;FtDs-$l+n%@IOfEbck% z8GLLJ5ViU#`r8XW;iydtg##{)KQ(}h~!@ez%9zh5#R(5J^M*{g8f z56Uw#ePkhW`v|NDz#WMflaV}Q=Y*~gOmvoKIyhz9i?5EhpX(}N%5aUwyuud^N!*hA z`T^xF1v!9D+Y*z2dj@dubs{&KU(?6X#ytwci7-e}6u+_3 zz0`2Sjbofuu>tLPvN4>6d=tW?bjZ18CF{-v!g=5@*CeEjCO@Mcxhao^U}B3;A=^F4 zW0kdTi0oo^PtB9Y4ZUKWifWGs{&v*>swJ+^a^Z9Nc99#r;Ik|z5w!$DBm;v%F|Bec zKT7H?jzY>VfDno{72m?D)ju+lV`J>AwwD2dm}9x?wo2OsByTTk#QmURO#mUemcX4iz0!>Tc#K(+{m%4*hOr=K#RY2)3kO2yUjJR=+& z&jnh*;U-8XOyli#pa8$V)+M3nZZdrp;T}QhQkh^l7dK{`IS7Xz&}n-Q6jZcBR@8DK zKBj(?jzB|0!~9mm=Ij)dDxrGaZRO|kk2FOhNS$$>sw)Vt@7DrEwsqnsL2+5UswUA5NH*|L zw;M1|jbmH||Fb|uD>cGJCOs)P zsca6v1ch6vZRU*le0B^H5Bz_+iYa09A3KQRSE;`0pbaz%KpdvND1 zvw^9v*`*14C(0bHJ$KRm3tp6&5=V~HOf`4%Jz>ro+wx%RXPv=_5tW4b& z?mRUd9|!wS6U_g(LB!G22MVulrEY?iZhiB9A zEAg-_*_H+x=Cc=`%C1SZIQ*>RE~Q_NG}Cr){&~P{W&74JO8NOdwH8j_YtT>P>go^ zvbUz|kAB#GQqf{L(GgZ6ks`A}4&gp#m6*K&2pKBZW(+ad5sm$EuuFqefwRep-%0G& znNLSLRoj2dVYj4luCfd_l6JH@Yr^Qh|HHP99aMLx-wM7g5r0@)bE>X5A*9Hp5YjGg z6SHeoHC_iHLLEY_&a z)9u(Myc#-17(?-u%{`DS_I}Y*h1kfV#}A{uy~*5uPkz0yMD+(#M3@#3$|H4JkPXU* z*aWg?L!N1{cd_@rMZ%85?Qy^AV)%HEUv1?<{$)N|8opLF`O4&PS_~~(Q=epJNZ4BMekQJ zu3AK4j}3dm$p8~RFzYi?J$W{F0E*3!Vn!$2ntw8bjbIk%Ug*QvSgoKtLV(xKo^W$I zf`;Tc9N&z17wjI$v|CoodPdIoR@HMq7mu_}R}*Vb?{bbPySh}r#nU;{H9;76MlA?d zyb$22+49jz5=>>oCr!D@X5i1iS1WBvV`?sS&}+l@R{49`PoBZ&M#}?73d$@r$Z?8K zn)_$+8WN5HYxF1f)oDdC0k0kH9_#d57gk;4z_90TEz=R?jFx2^t#nDzccZR)BOAb+ zp*3dQaUgp6G6Bkfhjxc#lH}SCM&Nt$bve$Txxyo6+3Vk-T5JCO)k_VHX2(FSNBlL~ zz)(h@7S;5X!vKnETC~0cp!Gcf(m=~>h zc}^#HOmWBcD++tLqY%f(Y|OuCH}i_y@7!MP5tCiFVAgFr9=H#Izs+9U)BKWAk=c=< zypn?E`-o|q&(5LB%ol^$c3+WHfFg-M8lEf^C-*&r9VKF0fD@~{aB8zEsP>l(3huNp z4vUbtO&rFG7NkS%U0XZLP<|cml85y7dChVEl;jW^$^<69GnWQ0Q*+@qHOQb-Nrvhl3&^hq_F3dn$+ef;)z4eix>JjR0Wjs;_0B_>~unpO(faN00 zOoSTurxZXh%$Pt2PR`1m1>|}-5;DI$g!rvgc?LbRL}|r0UC_$bMP)U|+$Zgv^uoqc zPt_oYZ4C8_f=i$$q(dN^Z)(u9aipog1&V9)v=U5~*|gqiR7( zA-rWE1*<^SF_!#(IPm5XQXC6i)!Lt&7TFtQ%9&uxre)uz_E6Lk=({4f zofB@K7p3vJN2?*D1&3#3_&UPH;}TTo6!2;uCl#k^a#*p0=Y)kjabne6>);81V|P2~ z?$`1*Sre`3`q47%la*s+o9xttYA!TXN~=1Y5ZGFWtcV*(x<prGMU|aw0(vKenb!zG6>zVo z3u@DfHVOLu4inTeY>GTFGJCa>{VwH+pc^upPiUmVd(d8clAy|9RCt%aRtPOCw=ttJzir&aCg`mY!!N-TIT7gG_u#$NyU9)?Y$;~aO4&7f zcIeh}aca@LA-B@s9Ygjenc*RX3Bv0><9A@&&I8%40nl2 zl@O69NV;(f`|r6aDr*C;f&Xu!x3^4Thd87* zxYX(NmkS`N!1jnIVC9NQmNMk9rV(r%Y7zMV8$b#9tdVkeF9#G|C9)gA4h=8f?Dx;i z&VG!`dIv7DAANMzx}sA)#IRq3$`c*Ox?T`9kIoC~PRRTqYJOp=Bs|k{qr=sq@qMGi zDl5*KttX2`=Ss~ZVI$-6XF1vDYtAy_69ueFyUx0Yn8e@W`J?)8lf6^`;7toNa^Xni znPyvUq#_Nd@{krsGKi4n2~vj zJyx0zOTl*aJq02W&tFt$G1yJFIJ#w3`+?ua#AiFm%3p+VS_x*7^Zt| zj2BW9K;D?*Jx^K-8VD!~eH+WQ8TIEVcjOTgFx2}X6-~#qx5cv!l^V5y+nDE$MC>GAk9g_{VP_X} zEq2=4r@I{!LTaWTUFg=vh%&)h%aIgQVXcMJDFPzME}!!G%1VfB^CD z0vvRL656sv?XOR_fLM!IwGw~vDOLq*SRG?2f!#@S_$@XtGfamtX8 zkm6p0gl1rfMaUw}Z}WxPHah9?!AtEl%WJd^uE-*gu@TIq$03>{U~=jP61Bu43W28~ zHfxi@@UG_f4zN)66(b#;j|nwoALs}E1RFm=T_*b6dzf?g7_vzVa8m#1geP`gA3K zyG0r$WFXiz`LYW!bQPYp>cijNGzvW*uz)S~6%&t7N0k^(?zmF^93&p#265x1HbqSS z?(41lOB0KHRYFf*!AbCJ6+XMoW4aQ4zm&N%_a^nopjdDl{P!gASBPm`sER_!{d1LI z+{zkV;TWEOqbj9|ei-%fWTBHTf~B6yLYKXjG(;0K>Sqk5-WvruXsjsc7g{n585+07 zJsUAJzGWpgm*Sp@n}$CU?s-zk{B%h)w%9#VIDROmZL8ot+x~VAH1PZN^1=L@#_1#R z*)kFPy=yT?!5!v$C!@TaD-+HD8Yf*!&Agpwe>1SB%8AYwnGTZY2R!vcv7?QAMRUEV zDF%P_j=CKb1{7WgIEj`!ZD$Gc)`x}+j5M5eS&|V7LUCl}^eUy4WFfv(`v+!@$*Hry zX~+SD8Hr9SN-GjsKUycRNQGeF^V4t9^%i5hLKgP5mi2 zAJ`7ig*S!NMYbOuQ8%l5?W5KXpIAgeMUO5fsn;_~lT4kp=2vF$F(^=Z57+)7J!V&d zOT<;EvB8_dbql6jy>}4knmTx}|I?lPub!7Ssy4as6wIS=B)fl3vfSODO}wN$LGK(p zAtZtFj`O65dqF}<0T-x~k0uMj27SUac#Evu`EB<7imm&1YZfK`+sAJ|jnE`}huZmN zPv?Vtjzr<;|MkK5W(j?sm-^6~WdHf#NO8e)O|oiP(@x_x#fxH#k!>=W#J6S-NInlg%avOsNWDhXZ9^A3|fE^iV`5VYzmS>se(3HMH2fd zqzw|%6L1$c?KcI~5nwN?m&H-r;g7!5NQ6m`x;1#UM$$EC?b?Y&jq0pdqhyn8?YX~< zJdUb)N!R!+4<_i-^PN`k&Y!rjvb`s!F75Hup>sj+Qp)eH)RX?|LE+nNiAd=TR5gHV zx`uc>f9PPh$@XK8BS_O9{xn?<+%GB)zcqnkTZy^qT*cdFZU_C? zkwHFVmxtTZ72w)$crj0>%Td-)5@oNLVokBST#ez_ZH#>B$K2?VFYUAI@G01N19T63 zH|7udCGEV`6La|NRB78tJuhj{*>q-Lk^iX_VP}z403r-R@B@lBD->v{a!sBtA6zIZ zq-JJ&zQ2gioDrc6g)6+F8&co3$=c48at?Xk*>JeajqY5P!M|Y{^#?jkjbSZ)YfnkD z3&StP_To9=HhU*8GtwcQ5_M|Gt54&n3RDMk@1!^Hdfs~6nMd`2R=Fkc)`gf;EIUgP z2R`k|szXn-A!LTyU;)|M^I-@{YH;B)lHcYht(TQ2n^yWW?F)wyLH$aU@LyE(oNeN} zvcPsJ`;q_KV9jshjgh%4YJpmg{>0Wfw}e>&!b|Sl=OvGC|KTNbV>f6C^|*eCYrmwI z-|Yp{Qzq0VzB(`V=^y=@H+RP~(rVZQ0Cxr0(GkEdwSSHWK-6*>QNI)lV`u;DG?$}r ztvRU)F_*J&)W+{u@6l6zd{JdWf|2%4%hkU)yqfl@EXJK1|C?<*A@M)i#`$8ijV7LR ziC8s^?YqJ^pdGeLr&|XVpG+OSbm_1+YmAtQ>3^%IMUKBfI2uS0j9>%D6c8yPQFvB; zT*2q!6q24?KU6|p6&;#S)gqeX)wn!6Ca~4MTs_6%^*ZE^I6@b;h})+N4?kn8pG0Fn zV)ru2-#ulLiXFaPN6r#pw&4nN!pWFgFK#8nA-y-KoM<+SE|GhyU*RPD+D5z3_cAx27A zHv$MTCr+o9_n^7Py$vNGq#3rKr1`FTY?c=R4d7G+@Y^_xMY)0hq6#%;;F6)-p)R6k zS%#$YGSUkY7gA(NRQH!eJm(}W&D{meY#6&+^H%5SM&+$$H7pfq^riXx-gOPiKowmy8h)Aij6($&3ymjbQvoKHSIs1>?D zjn}Do{RIe7eYtn)J_N37-Uv3!+lp7}33vs353a672@W`!qLuBB93CLbAv;kR2&YPq zDPVhe$r-n%6xFVkZ>1ED&b1N@$Hc3IUcT}MDV9Pw>%-j6c@M$X9vG7G&RJxtfcL~J zKIxp&<3T%HO4MAJlwa*S6V&j1!F7dC(_H`sl4RU+^0oattFu&n& z5ift^o(W-4>XxKeWw(8Is}429kh~(|gyFLuS+16V4mfo>3q2o}^iG>|ukflnF|T`m zj>(%P(ZKL$hhPv7_u*CK91tkiAK0;qK)mzS+&fM#oSc6w_4aevYow=%#^9h|eGe~X z4F@oe9;)A~!Yt2@Yg4Qd-Og$i&fP}ftypwDe+ck97m&x{$H`|+8+#+eXt`4DNoQtn zC;gWNbc(OWxeA^-(5XgwNJeos7nn8c1mf$WeNT(2-@IQhSz;5#!t*bAsA;+H5D5h- z8FjCG|4k>8K1IEkd31^hjgGB8buPEa@KPv~MBKbxdd&|1I|HMeFp zxy;7gy1dapf{oYE@P04*w5R>&j}g?zt(P@D(O+&FoGI}B&}JYqTk6J{nf3gh$CvvCU+XYbJt+NaEEha1r4HXIMiDwxf2p)YDn7 zx65qKWU)nz&rXgx%ju4IY#6O}!X+w#Crd!|WY zk}u^SD%C#?bT6z;l6B0l@BVsGqbd`jF1|?pzC6|w7UVro2$0W07z&Un-l0N|keGNP)cK4ei z(T;Q%qiO7Hz8W4Oi4kBd74=otv!N}XbN(K@Z7}nj(96Q}BSa%%Ywo_{JKM{nI@c3xh&wvClJ8dTbUE~?+F6V_-Th^5 z!&GFB-89?GTvIarV2rk6EoLOo`XXEni0_ztEnIFkp3|FJqapJlcz8Wx^S4z}%O=N- z2E(?ax!x(P=*3S(v@XuuZ}<40W=qBIdet`aMLfjPOSMM6Xw8|ic%pq#$3);wc{Z%B zo!Z&Cvsd?I5a+kRevpHN5ymUee}O-J(i3$2lk{WN>tVduW_&nKCW5;ta)PU*2W zNG-GY_bb0DyG~9*5<2lPruiKoBHmJnc$=OzVtg#=J-SeKm2@+(KsuD?L&bHI&vOIi zyOY?vRJlqI>VF=sXVXNv%{s1xudUyv;fP&obl*!!l-_-}`D^`wWla+Dp8QN(HzhlV zVX>8$*HcK(+{j^d=&7Gk+g^P4gD-+rLmK50F~RxqbZ;k^&zSJ^q-Oq=l@>E!kDZIa z;VFpKAXC);n4>1d&0%+Xp!aa~Xrq90xm=5XWZ zEUhqzd9)eFC(bHyTJGKITTV|7kJSdgnOx)T+dG8I15BnS6J1@Xk2P+KhgSO7s-?{i z{E)q(WzfB)i%Nr;xT4$|sdNiu69bxOEt_>!0))g)`=fkR>fNPdX0E0I)J>cvh+{FC z#W-1DtU7+A`%*NGuxAYTSN7UXkxi?%6#2jJ(`X$B9c1`w{-DV*59OFgB`SBjdV>=U zj3p$NdiF$qDmYurmTh9@lRnD1FL1B4V%0N~cHiAqnKYMm#Z$m*uWZ?!FRClQhvZ_H z)}Fwd-7b0&nyrV#(Kf|SL#f`m@6+;@bsF<0Pc`;AhF6nL(v{k_FH;sh;?Aq$(AW7z z(b&I5i5{+Q_7$&s3_ahw7BFJdyrKANnbhD6bl;v5UiR?_{?0{jo{6%`KZqta!>9OKmS6?TY7BS z8wOSJpmw)(?z_=i+9`Q!sk3(uO0CAWS_e>4mV26hroCsFkwtJB-9Tm%>IN4R$}4xBO#u`o7><3T|W+e(cNcR6MfkT8Qv`OOFh2?&N7o?&v;kFO^$n>Ct6v zJ1FHgONE+{=Q4diKHTxZ`}By*FB!e+rQODZF=b+sxSvugo3t^{I2_w ztETfKdf+zS^ODzjnp^9k11zEm8(y__6pp&*+fPQwgI7`h8Z=|!X+--QhA$1*5`^#F zWh)^2RLr4wWbW3T7sZZ0?pqF(esUxPc^jRI0L<_go!5Y34KA z_9w(tX>N!xIL#&TKihOZqyE@jBDXiT@==#|n)dIXiT_mf7hah!1XIl%*+ILr(%8Zs z$`=8hTE_gakKGL+z^47^Ux)0H<@O>o%v|t0!yJynJg<5IJDl6{Z1Ig?N5@_ZD`v5K z`lS;pty28ZGu1~mZI5jR#zGBjD0Cistbg8|I;mvkyfd^Kv^COOy7J5R#BrA0p`k_P zUH1|yJH_`g{xhRTKAG{?4;FRZw;bjM>#BOU znew4>y|J3fma~B0+{i8Fzyxugv3vG!%JZyhqJAhvhCUA5jZNV9)IZmZ8dXwCaOe88 z;+9Q9+|-V_mFG~ycIAdrXYSDlYDb7)ZJwQYk>;nJdJU`>`h%>g>IkOSBY=8`q4oAb zq9iOIg3T@Bi^u@V^4djv#^V=`{JsN(-u5SB9ry2FvU3_U79boOT+Fk&q~!qS8_<5ZyG_;qyHCy}cv9=dq2N zjoCf^*2=)B$wm06at`kEt-FizN&#}lW7&+W(J+n0nc=W;y7WXI+n2v=LF|=u(-LEj z#*Ccow@aqd0AQTgD%#9nh!gLZ1b;|ZZQr%$^XcV)>?PBADsWFI?$hYl-?hD7qCqk z9TzX4t6`k~7(_YlI&E5>wR`Z7s#*)1@@@-DR&o7z^(T7-i+y6=b>qlmcTC;H2Apte zNgacB*1_eL=Ez%bcUH8T&JBlQr}QcnSwAj=n=s$6l2#IHBE-h;xat39PT@PE0n8*X zxS~Cy%ZKiSgM&z~@SXj$`zgfuD3C*k?=__NelQ2^c!{VTdD4!*UiP(+lzV!91r<@H zKTRnVKJi={{8>YI6g=@afr^2`-Fy0&tEZ8T0=7ch9euy(XzF!?vAm9M>YmrXO(k#A zeKUoLvxomdlTlXWv2zyfsX7E;1rD(n2Qu8~YxFEvuGeWolI z4AMhz^C&zx%JUu-HLz+ix4kooTUD>^?0i2s?sR=ARf$pUR}#eyb`S~0TEpwFVnKBC zo=i9YN(wM#{4uO1mtRaXB{D;S1Oaz*j?v#cl&`34R`hL>?J493H^-vb!D8MeLh}RUqTz))(a^IjzkU-H#gtri{%m z%s>Kbi>Y-!yLkAWNsZ2S)4|jU{f({?VTe248`57mf9z#8_X@`wC+p5sbl1i5g!lY6 zj_U_8obxglz?!cwP;-N4{?Bg{m~C0&Y~Ms3an$7-s7W~Q+m!M{amwMkPQ(v?{%il{ zY4iE1?>%|?T!?F6`>bFlPeD_uTd(FYwE!PQU=*@vWHQI9wg0{M{qsLrUyYbP{TPM4 zbW)h9L(adya9WRL&C;*kfN3=fv_KX(Oxf%7>xuF z(0PyPO)_*&utau*9Ym=JF8G-uE^4dXXy<)cMp*W#B-QL$OFs=hPh@+;dV(S{8fd^o z%fX{)hZhRQA70l#@S{YebJ$7+5yodP%LDm)5w#h*+Y}d&9rQ!-BR}m=nK##NUn29i z4c0k!a_@aGmM7=!wY0F!(6lmjJw3$6iPnl`b2UNS<$n9=hdLb^CtACluxZ90i5k$; z9D3PoLZh9Q9q1Nyb5lQvl=>Ja+QsuOZwS@~9==_ZaB0GwlJ_}`oZ1K4PbRjnH`9In zrJ}^5;8kn)vmZ}iO>RFAK}GTXdx+lvs>axM#0+tP`2vtDl?UFJmZns0PABJWtM0a;d8+j`Nk)HLGAX#K?DY1Gden%2%3gTffN8X;Wjz?6kH15cPHW9F>gq3njjnn|2@RuV!=6hRt;(>dtNajF!z_pB8U$zl|p$ zIq*RBs)CkqXNmCnXE4taaqyIb$C0PFjnGb~-IkgvdLg8A>E`a*rca}VKQBig^NvWS zcT{m_bjjQILeDn;4LAsSj%S4sU-IV>7pPHdfNArt#GT_S6SvvsY6g5w9|+47_Akfh zuct-H9b<=Psx3j36A>ThS*&UQaD&zfO$ts9doWK|xCQuNymYS8v1 z)dxga@_bE|;J42<9+zy1aiF;2=l`F+o*f?DJ8&cZ^-h36Ob^ng>NWMNIg0=I{2$_H B&LRK+ literal 0 HcmV?d00001 diff --git a/docs/images/combined_ThS.xml b/docs/images/combined_ThS.xml new file mode 100644 index 000000000..edf1aec9a --- /dev/null +++ b/docs/images/combined_ThS.xml @@ -0,0 +1,240 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index aa0286299..7cadd7dfb 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -13,37 +13,43 @@ Configuring Python Begin by loading the Python 3 Miniconda_ module:: - $ module load miniconda-3.6/conda-4.5.12 + $ module load miniconda-3/latest -Create a Conda_ virtual environment in which to install libEnsemble and all -dependencies:: +Create a Conda virtual environment, cloning the base-environment. This +environment will contain mpi4py_ and many other packages you may find useful:: - $ conda config --add channels intel - $ conda create --name my_env intelpython3_core python=3 - $ source activate my_env + $ conda create --name my_env --clone $MINICONDA_INSTALL_PATH + +.. note:: + The "Executing transaction" step of creating your new environment may take a while! + +Following a successful environment-creation, the prompt will suggest activating +your new environment immediately. A Conda error may result; follow the on-screen +instructions to configure your shell with Conda. + +Activate your virtual environment with:: + + $ conda activate my_env More information_ on using Conda on Theta. -Installing libEnsemble and Dependencies ---------------------------------------- +Installing libEnsemble and Balsam +--------------------------------- -libEnsemble and mpi4py -^^^^^^^^^^^^^^^^^^^^^^ +libEnsemble +^^^^^^^^^^^ -There should be an indication that the virtual environment is activated. -Install mpi4py_ and libEnsemble in this environment, making sure to reference -the pre-installed Intel C Compiler (which should support MPI). Your prompt may -be similar to the following block: +There should be an indication that your virtual environment is activated. +Obtaining libEnsemble is now as simple as ``pip install libensemble``. +Your prompt should be similar to the following line: .. code-block:: console - (my_env) user@thetalogin6:~$ CC=mpiicc MPICC=mpiicc pip install mpi4py --no-binary mpi4py (my_env) user@thetalogin6:~$ pip install libensemble .. note:: If you encounter pip errors, run ``python -m pip install --upgrade pip`` first - Balsam (Optional) ^^^^^^^^^^^^^^^^^ @@ -52,19 +58,12 @@ computations on systems like Theta. Balsam can stage in tasks to a database host on a MOM node and submit these tasks dynamically to the compute nodes. libEnsemble can also be submitted to Balsam for centralized execution on a compute-node. At this point, libEnsemble can then submit tasks to Balsam through libEnsemble's -Balsam job-controller for execution on additional allocated nodes: - -.. image:: ../images/centralized_Balsam_ThS.png - :alt: central_Balsam - :scale: 75 - :align: center - +Balsam job-controller for execution on additional allocated nodes. Load the Balsam module with:: $ module load balsam/0.3.5.1 - Initialize a new database similarly to the following (from the Balsam docs): .. code-block:: bash @@ -76,8 +75,7 @@ Initialize a new database similarly to the following (from the Balsam docs): $ balsam submit-launch -A [project] -q default -t 5 -n 1 --job-mode=mpi $ watch balsam ls # follow status in realtime from command-line - -See **Additional Information** for Balsam's documentation. +See Additional_ for Balsam's documentation. Job Submission -------------- @@ -86,16 +84,26 @@ Theta uses Cobalt_ for job management and submission. For libEnsemble, the most important command is ``qsub``, for submitting batch scripts from the login nodes to execute on the MOM nodes. -On Theta, libEnsemble's communications are commonly configured to run in one of two ways: +On Theta, libEnsemble itself can be launched to two locations: + + 1. **A MOM Node**: All of libEnsemble's manager and worker processes + run on a front-end MOM node. libEnsemble's MPI job-controller takes + responsibility for direct submissions of jobs to all allocated compute nodes. + libEnsemble must be configured to run with *multiprocessing* communications: - 1. **Multiprocessing mode**, where libEnsemble's MPI job-controller takes - responsibility for direct submissions of jobs to compute nodes. In this mode, - libEnsemble itself, including all manager and worker processes, runs on the - MOM nodes. + 2. **The Compute Nodes**: libEnsemble is submitted to Balsam and tasked to a + compute node. libEnsemble's Balsam job-controller interfaces with the Balsam + backend for dynamic task submission to the compute nodes. - 2. **MPI mode**, with libEnsemble's Balsam job-controller interfacing with the - previously-mentioned Balsam backend for dynamic task submission. In this mode - libEnsemble has been submitted to Balsam and tasked to a compute-node. + .. image:: ../images/combined_ThS.png + :alt: central_MOM + :scale: 40 + :align: center + + +When considering on which nodes to run libEnsemble, consider if your worker functions +will execute computationally expensive code, or code built for a specific +architectures or nodes. Recall also that only the MOM nodes can launch MPI jobs. Theta features one default production queue, ``default``, and two debug queues, ``debug-cache-quad`` and ``debug-flat-quad``. @@ -111,8 +119,8 @@ to the following:: $ qsub -A [project] -n 128 -q default -t 120 -I -This will place the user on a MOM node. If running in multiprocessing mode, launching -jobs to the compute nodes is as simple as ``python calling_script.py`` +This will place the user on a MOM node. To launch MPI jobs to the compute nodes from +the MOM nodes use ``aprun`` where you would use ``mpirun``. .. note:: You will need to re-activate your conda virtual environment, re-activate your @@ -170,8 +178,7 @@ on Theta becomes:: Balsam Runs ^^^^^^^^^^^ -Balsam runs are Batch runs, except Balsam is responsible for submitting libEnsemble -for execution. This is an example Balsam submission script: +Here is an example Balsam submission script: .. code-block:: bash @@ -244,7 +251,7 @@ for execution. This is an example Balsam submission script: . balsamdeactivate -See **Additional Information** for the Balsam docs. +See Additional_ for the Balsam docs. Debugging Strategies -------------------- @@ -257,6 +264,8 @@ queue interactively:: $ qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I +.. _Additional: + Additional Information ---------------------- From a897fb673d5d8e29e53e754fbc12434e11b33e46 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Nov 2019 14:43:59 -0600 Subject: [PATCH 467/644] adjust references and headers --- docs/platforms/theta.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 7cadd7dfb..244eade6a 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -15,7 +15,7 @@ Begin by loading the Python 3 Miniconda_ module:: $ module load miniconda-3/latest -Create a Conda virtual environment, cloning the base-environment. This +Create a Conda_ virtual environment, cloning the base-environment. This environment will contain mpi4py_ and many other packages you may find useful:: $ conda create --name my_env --clone $MINICONDA_INSTALL_PATH @@ -75,7 +75,7 @@ Initialize a new database similarly to the following (from the Balsam docs): $ balsam submit-launch -A [project] -q default -t 5 -n 1 --job-mode=mpi $ watch balsam ls # follow status in realtime from command-line -See Additional_ for Balsam's documentation. +Read Balsam's documentation here_. Job Submission -------------- @@ -251,8 +251,6 @@ Here is an example Balsam submission script: . balsamdeactivate -See Additional_ for the Balsam docs. - Debugging Strategies -------------------- @@ -264,8 +262,6 @@ queue interactively:: $ qsub -A [project] -n 4 -q debug-flat-quad -t 60 -I -.. _Additional: - Additional Information ---------------------- From 5a1405a342059bedb8571ab956701ba79e7cae50 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Nov 2019 15:18:25 -0600 Subject: [PATCH 468/644] more clarification and description, typos fixed --- docs/platforms/theta.rst | 71 ++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 244eade6a..95da5a6fd 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -2,11 +2,12 @@ Theta ===== -Theta_ is a 11.69 petaflops Cray XC40 system based on the second-generation Intel +Theta_ is a Cray XC40 system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), -and compute nodes. MOM nodes execute user batch-scripts to run on the compute nodes. +and compute nodes. Users on login nodes submit batch runs to the MOM nodes. +MOM nodes execute user batch-scripts to run on the compute nodes. Configuring Python ------------------ @@ -56,9 +57,9 @@ Balsam (Optional) Balsam_ is an ALCF Python utility for coordinating and executing workflows of computations on systems like Theta. Balsam can stage in tasks to a database hosted on a MOM node and submit these tasks dynamically to the compute nodes. libEnsemble -can also be submitted to Balsam for centralized execution on a compute-node. At -this point, libEnsemble can then submit tasks to Balsam through libEnsemble's -Balsam job-controller for execution on additional allocated nodes. +can also be submitted to Balsam for centralized execution on a compute-node. +libEnsemble can then submit tasks to Balsam through libEnsemble's Balsam +job-controller for execution on additional allocated nodes. Load the Balsam module with:: @@ -84,26 +85,27 @@ Theta uses Cobalt_ for job management and submission. For libEnsemble, the most important command is ``qsub``, for submitting batch scripts from the login nodes to execute on the MOM nodes. -On Theta, libEnsemble itself can be launched to two locations: +On Theta, libEnsemble can be launched to two locations: 1. **A MOM Node**: All of libEnsemble's manager and worker processes run on a front-end MOM node. libEnsemble's MPI job-controller takes - responsibility for direct submissions of jobs to all allocated compute nodes. - libEnsemble must be configured to run with *multiprocessing* communications: + responsibility for direct job-submission to allocated compute nodes. + libEnsemble must be configured to run with *multiprocessing* communications, + since mpi4py isn't configured for use on the MOM nodes. - 2. **The Compute Nodes**: libEnsemble is submitted to Balsam and tasked to a - compute node. libEnsemble's Balsam job-controller interfaces with the Balsam - backend for dynamic task submission to the compute nodes. + 2. **The Compute Nodes**: libEnsemble is submitted to Balsam and all manager + and worker processes are tasked to a backend compute node. libEnsemble's + Balsam job-controller interfaces with Balsam running on a MOM node for dynamic + task submission to the compute nodes. .. image:: ../images/combined_ThS.png :alt: central_MOM :scale: 40 :align: center - -When considering on which nodes to run libEnsemble, consider if your worker functions -will execute computationally expensive code, or code built for a specific -architectures or nodes. Recall also that only the MOM nodes can launch MPI jobs. +When considering on which nodes to run libEnsemble, consider if your user +functions execute computationally expensive code, or code built for specific +architectures. Recall also that only the MOM nodes can launch MPI jobs. Theta features one default production queue, ``default``, and two debug queues, ``debug-cache-quad`` and ``debug-flat-quad``. @@ -119,19 +121,21 @@ to the following:: $ qsub -A [project] -n 128 -q default -t 120 -I -This will place the user on a MOM node. To launch MPI jobs to the compute nodes from -the MOM nodes use ``aprun`` where you would use ``mpirun``. +This will place the user on a MOM node. Then, to launch MPI jobs to the compute +nodes use ``aprun`` where you would use ``mpirun``. .. note:: You will need to re-activate your conda virtual environment, re-activate your - Balsam database (if using Balsam), and reload your modules! Configuring this + Balsam database (if using Balsam), and reload your modules. Configuring this routine to occur automatically is recommended. Batch Runs ^^^^^^^^^^ -Batch scripts specify run-settings using ``#COBALT`` statements. A simple example -for a libEnsemble use-case may resemble the following: +Batch scripts specify run-settings using ``#COBALT`` statements. The following +simple example depicts configuring and launching libEnsemble to a MOM node with +multiprocessing. This script also assumes the user is using the ``parse_args()`` +convenience function within libEnsemble's ``/regression_tests/common.py``. .. code-block:: bash @@ -142,7 +146,19 @@ for a libEnsemble use-case may resemble the following: #COBALT -A [project] #COBALT -O libE-project - module load miniconda-3.6/conda-4.5.12 + # --- Prepare Python --- + + # Load conda module + module load miniconda-3/latest + + # Name of Conda environment + export CONDA_ENV_NAME=my_env + + # Activate Conda environment + export PYTHONNOUSERSITE=1 + source activate $CONDA_ENV_NAME + + # --- Prepare libEnsemble --- # Name of calling script export EXE=calling_script.py @@ -153,13 +169,6 @@ for a libEnsemble use-case may resemble the following: # Number of workers. export NWORKERS='--nworkers 128' - # Name of Conda environment - export CONDA_ENV_NAME=my_env - - # Activate Conda environment - export PYTHONNOUSERSITE=1 - source activate $CONDA_ENV_NAME - # Conda location - theta specific export PATH=/home/user/path/to/packages/:$PATH export LD_LIBRARY_PATH=/home/user/path/to/packages/:$LD_LIBRARY_PATH @@ -170,8 +179,8 @@ for a libEnsemble use-case may resemble the following: python $EXE $COMMS $NWORKERS > out.txt 2>&1 -With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble -on Theta becomes:: +With this saved as ``myscript.sh``, allocating, configuring, and queueing +libEnsemble on Theta becomes:: $ qsub --mode script myscript.sh @@ -256,7 +265,7 @@ Debugging Strategies View the status of your submitted jobs with ``qstat -fu [user]``. -Theta features two debug queues with sixteen nodes apiece. Each user can allocate +Theta features two debug queues each with sixteen nodes. Each user can allocate up to eight nodes at once for a maximum of one hour. Allocate nodes on a debug queue interactively:: From 287800cffbd491646bc3a8e27961423a62314925 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 8 Nov 2019 15:46:02 -0600 Subject: [PATCH 469/644] update tutorial examples for 'user' --- examples/tutorials/tutorial_calling.py | 8 +++++--- examples/tutorials/tutorial_calling_mpi.py | 8 +++++--- examples/tutorials/tutorial_gen.py | 9 ++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 8db2e8609..38d87866f 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -9,9 +9,11 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'lower': np.array([-3]), # lower boundary for random sampling. - 'upper': np.array([3]), # upper boundary for random sampling. - 'gen_batch_size': 5} # number of values gen_f will generate per call + 'user': { 'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } + } sim_specs = {'sim_f': sim_find_sine, # Our simulator function 'in': ['x'], # Input field names. 'x' from gen_f output diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 2ba751353..99d5e628a 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -13,9 +13,11 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'lower': np.array([-3]), # lower boundary for random sampling. - 'upper': np.array([3]), # upper boundary for random sampling. - 'gen_batch_size': 5} # number of values gen_f will generate per call + 'user': { 'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } + } sim_specs = {'sim_f': sim_find_sine, # Our simulator function 'in': ['x'], # Input field names. 'x' from gen_f output diff --git a/examples/tutorials/tutorial_gen.py b/examples/tutorials/tutorial_gen.py index 5a109059d..85bfedf12 100644 --- a/examples/tutorials/tutorial_gen.py +++ b/examples/tutorials/tutorial_gen.py @@ -4,13 +4,16 @@ def gen_random_sample(H, persis_info, gen_specs, _): # underscore parameter for internal/testing arguments + # Pull out user parameters to perform calculations + user_specs = gen_specs['user'] + # Get lower and upper bounds from gen_specs - lower = gen_specs['lower'] - upper = gen_specs['upper'] + lower = user_specs['lower'] + upper = user_specs['upper'] # Determine how many values to generate num = len(lower) - batch_size = gen_specs['gen_batch_size'] + batch_size = user_specs['gen_batch_size'] # Create array of 'batch_size' zeros out = np.zeros(batch_size, dtype=gen_specs['out']) From 679b974f0afc46a9aa3ae82e8b3f8697c4c62654 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 16:02:52 -0600 Subject: [PATCH 470/644] README/Intro edits --- README.rst | 70 +++++++++++++++++++++++++----------------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index dc00bbfcd..a0dec080f 100644 --- a/README.rst +++ b/README.rst @@ -124,13 +124,14 @@ Resources **Support:** -- Email questions to libensemble@lists.mcs.anl.gov -- Communicate (and establish a private channel, if desired) at https://libensemble.slack.com -- Join the libEnsemble mailing list for updates about new releases at https://lists.mcs.anl.gov/mailman/listinfo/libensemble +- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. +- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. +- Join the `libEnsemble mailing list`_ for updates about new releases. -**Documentation:** +**Further Information:** -- Provided by ReadtheDocs at http://libensemble.readthedocs.org/ +- Documentation is provided by ReadtheDocs_. +- A visual overview of libEnsemble is given in this poster_. **Citation:** @@ -148,9 +149,6 @@ Resources url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } -**Other:** - -- A visual overview of libEnsemble is given in this poster_. Quickstart Guide ---------------- @@ -161,14 +159,11 @@ Dependencies Required dependencies: * Python_ 3.5 or above. - * NumPy_ For libEnsemble running with the mpi4py parallelism: -* A functional MPI 1.x/2.x/3.x implementation such as `MPICH - `_ built with shared/dynamic libraries. - +* A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries. * mpi4py_ v2.0.0 or above Optional dependency: @@ -186,11 +181,11 @@ The example sim and gen functions and tests require the following dependencies: * SciPy_ * petsc4py_ * PETSc_ - This can optionally be installed by pip along with petsc4py -* NLopt_ - Installed with `shared libraries enabled `_. +* NLopt_ - Installed with `shared libraries enabled`_. PETSc and NLopt must be built with shared libraries enabled and present in ``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file nlopt.py if Python is found on the system. NLopt may also +should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also require SWIG_ to be installed on certain systems. Installation @@ -204,14 +199,12 @@ libEnsemble is also available in the Spack_ distribution. It can be installed fr spack install py-libensemble -.. _Spack: https://spack.readthedocs.io/en/latest - -The tests and examples can be accessed in the `GitHub `_ repository. +The tests and examples can be accessed in the GitHub_ repository. If necessary, you may install all optional dependencies (listed above) at once with:: pip install libensemble[extras] -A `tarball `_ of the most recent release is also available. +A tarball_ of the most recent release is also available. Testing ~~~~~~~ @@ -219,7 +212,7 @@ Testing The provided test suite includes both unit and regression tests and is run regularly on: -* `Travis CI `_ +* `Travis CI`_ The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ packages to be installed and can be run from the libensemble/tests directory of @@ -245,16 +238,9 @@ in the top-level directory containing the setup script. Coverage reports are produced separately for unit tests and regression tests under the relevant directories. For parallel tests, the union of all processors is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after running the tests via the HTML file +which can be viewed after ``run_tests.sh`` is completed at ``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are -available online at -`Coveralls `_. - -.. note:: - The job_controller tests can be run using the direct-launch or - Balsam job controllers. Although only the direct-launch versions can be run - on Travis CI, Balsam integration with libEnsemble is now tested via - ``test_balsam_hworld.py``. +available online at Coveralls_. Basic Usage ~~~~~~~~~~~ @@ -278,23 +264,33 @@ can then be run as a regular python script:: When specifying these options via command line options, one may use the ``parse_args`` function used in the regression tests, which can be found in -``libensemble/tests/regression_tests/common.py`` +`common.py`_ in the ``libensemble/tests/regression_tests`` directory. -See the -`user-guide `_ -for more information. +See the `user guide`_ for more information. .. _Balsam: https://www.alcf.anl.gov/balsam -.. _NumPy: http://www.numpy.org -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _Python: http://www.python.org -.. _SWIG: http://swig.org/ -.. _SciPy: http://www.scipy.org +.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py +.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _GitHub: https://github.com/Libensemble/libensemble +.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble +.. _libEnsemble Slack page: https://libensemble.slack.com .. _mock: https://pypi.org/project/mock .. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _MPICH: http://www.mpich.org/ .. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org .. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _PETSc: http://www.mcs.anl.gov/petsc .. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 .. _pytest-cov: https://pypi.org/project/pytest-cov/ .. _pytest-timeout: https://pypi.org/project/pytest-timeout/ .. _pytest: https://pypi.org/project/pytest/ +.. _Python: http://www.python.org +.. _ReadtheDocs: http://libensemble.readthedocs.org/ +.. _SciPy: http://www.scipy.org +.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries +.. _Spack: https://spack.readthedocs.io/en/latest +.. _SWIG: http://swig.org/ +.. _tarball: https://github.com/Libensemble/libensemble/releases/latest +.. _Travis CI: https://travis-ci.org/Libensemble/libensemble +.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html From 8de449d63f4a77297a168e2fd7f3c71a27651bbb Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 16:16:58 -0600 Subject: [PATCH 471/644] Section 2 doc edits --- docs/FAQ.rst | 2 +- docs/bibliography.rst | 2 +- docs/programming_libE.rst | 6 +++--- libensemble/libE.py | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 5512df3f9..135b41e45 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -103,7 +103,7 @@ does libEnsemble hang on certain systems when running with MPI?" For more information see: https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe -libEnsemble Help +libEnsemble Help ---------------- **How can I debug specific libEnsemble processes?** diff --git a/docs/bibliography.rst b/docs/bibliography.rst index 8c87684c0..33e326359 100644 --- a/docs/bibliography.rst +++ b/docs/bibliography.rst @@ -2,4 +2,4 @@ Bibliography ------------ .. Note that you can't reference a bib file in a different directory -.. bibliography:: references.bib +.. bibliography:: references.bib diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index ad8be08f9..012eea031 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -15,15 +15,15 @@ dumped to the respective files, * ``libE_history_at_abort_.npy`` * ``libE_history_at_abort_.pickle`` -where ``sim_count`` is the number of points evaluated. +where ```` is the number of points evaluated. Other libEnsemble files produced by default are: -* ``libE_stats.txt``: This contains a one-line summary of all user +* ``libE_stats.txt``: Contains a one-line summary of all user calculations. Each calculation summary is sent by workers to the manager and printed as the run progresses. -* ``ensemble.log``: This is the logging output from libEnsemble. The default +* ``ensemble.log``: The logging output from libEnsemble. The default logging is at INFO level. To gain additional diagnostics logging level can be set to DEBUG. If this file is not removed, multiple runs will append output. Messages at or above level MANAGER_WARNING are also copied to stderr to alert diff --git a/libensemble/libE.py b/libensemble/libE.py index 83c55783c..a4468e1b3 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -1,10 +1,10 @@ """ This is the outer libEnsemble routine. -We dispatch to different types of worker teams depending on -the contents of libE_specs. If 'comm' is a field, we use MPI; -if 'nthreads' is a field, we use threads; if 'nprocesses' is a -field, we use multiprocessing. +We dispatch to different types of worker teams depending on the contents of +``libE_specs``. If ``'comm'`` is a field, MPI is used; if ``'nthreads'`` is a +field, threads are used ; if ``'nprocesses'`` is a field, multiprocessing is +used. If an exception is encountered by the manager or workers, the history array is dumped to file and MPI abort is called. @@ -76,7 +76,7 @@ def libE(sim_specs, gen_specs, exit_criteria, Specifications for libEnsemble :doc:`(example)` - H0: :obj:`numpy structured array`, optional + H0: `NumPy structured array `_, optional A previous libEnsemble history to be prepended to the history in the current libEnsemble run @@ -85,7 +85,7 @@ def libE(sim_specs, gen_specs, exit_criteria, Returns ------- - H: :obj:`numpy structured array` + H: `NumPy structured array `_ History array storing rows for each point. :doc:`(example)` @@ -97,7 +97,7 @@ def libE(sim_specs, gen_specs, exit_criteria, exit_flag: :obj:`int` - Flag containing final job status: + Flag containing final job status:: 0 = No errors 1 = Exception occured From 60804b1f89e87c634c529d8aa3cd74939ab0c60b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 16:41:20 -0600 Subject: [PATCH 472/644] Further data structure edits --- docs/data_structures/data_structures.rst | 19 +++++++++++++++++- docs/data_structures/exit_criteria.rst | 3 ++- docs/data_structures/gen_specs.rst | 5 +++-- docs/data_structures/history_array.rst | 4 ++-- docs/data_structures/sim_specs.rst | 25 ++++++++++++------------ 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index dbbbc81ac..f86c9042c 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -1,7 +1,24 @@ Data Structures =============== -This section outlines the data structures used by libEnsemble. +This section outlines the data structures used by libEnsemble. We first discuss +the dictionaries that are input to libEnsemble to declare the +:ref:`simulation`, +:ref:`generation`, +and +:ref:`allocation` +specifications, as well as specify +:ref:`exit criteria`, +:ref:`persistent information`, and other +:ref:`libEnsemble` +options. + +We then discuss internal libEnsemble, including the +:ref:`history array`, +:ref:`worker array`, +and the +:ref:`work` dictionary produced by the allocation +function. .. note:: Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index 6e4cc0b9c..a4a762ac4 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -1,6 +1,7 @@ +.. _datastruct-exit-criteria: + exit_criteria ============= -.. _datastruct-exit-criteria: Exit criteria for libEnsemble:: diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index c3250e3ae..bad2c34f6 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -1,6 +1,7 @@ +.. _datastruct-gen-specs: + gen_specs ========= -.. _datastruct-gen-specs: Generation function specifications to be set in user calling script and passed to main ``libE()`` routine:: @@ -30,7 +31,7 @@ to main ``libE()`` routine:: .. _gen-specs-exmple1: - From `test_6-hump_camel_uniform_sampling.py`_ + From test_6-hump_camel_uniform_sampling.py_ .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py :start-at: gen_specs diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 9f2271f66..371362968 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -1,7 +1,7 @@ -history array -============= .. _datastruct-history-array: +history array +============= :: H: numpy structured array diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index ae7a81885..e3e353b8a 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -1,10 +1,7 @@ -We first describe the dictionaries given to libEnsemble to specify the -inputs/outputs of the ensemble of calculations to be performed. +.. _datastruct-sim-specs: sim_specs ========= -.. _datastruct-sim-specs: - Used to specify the simulation function, its inputs and outputs, and user data:: sim_specs: [dict]: @@ -34,21 +31,25 @@ Used to specify the simulation function, its inputs and outputs, and user data:: .. _sim-specs-exmple1: - From: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + - test_6-hump_camel_uniform_sampling.py_ has a ``sim_specs`` that declares + the name of the ``'in'`` field variable, ``'x'`` (as specified by the + corresponding generator ``'out'`` field ``'x'`` from the :ref:`gen_specs + example`). Only the field name is required in + ``sim_specs['in']``. .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py :start-at: sim_specs :end-before: end_sim_specs_rst_tag - The dimensions and type of the ``'in'`` field variable ``'x'`` is specified by the corresponding - generator ``'out'`` field ``'x'`` (see :ref:`gen_specs example`). - Only the variable name is then required in ``sim_specs['in']``. - - From: ``libensemble/tests/scaling_tests/forces/run_libe_forces.py`` + - run_libe_forces.py_ has a longer ``sim_specs`` declaration with a number of + user-specific fields. These are given to the corresponding sim_f, which + can be found at forces_simf.py_ .. literalinclude:: ../../libensemble/tests/scaling_tests/forces/run_libe_forces.py :start-at: sim_f :end-before: end_sim_specs_rst_tag - This example uses a number of user specific fields, that will be dealt with in the corresponding sim f, which - can be found at ``libensemble/tests/scaling_tests/forces/forces_simf.py`` + +.. _forces_simf.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/forces_simf.py +.. _run_libe_forces.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/run_libe_forces.py +.. _test_6-hump_camel_uniform_sampling.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py From fa5e29eb00393580093ab266b700decaf9e0ad6b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 16:55:35 -0600 Subject: [PATCH 473/644] A pass through data structures --- docs/data_structures/alloc_specs.rst | 6 ++++-- docs/data_structures/gen_specs.rst | 11 +++++------ docs/data_structures/persis_info.rst | 4 +++- docs/sim_gen_alloc_funcs.rst | 5 +++++ docs/user_funcs.rst | 5 ----- .../test_6-hump_camel_uniform_sampling.py | 12 ++++++------ ...hwirut_uniform_sampling_one_residual_at_a_time.py | 4 ++-- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 61cfd54eb..8e6222fb8 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -23,11 +23,13 @@ to main ``libE()`` routine:: * The tuples defined in the 'out' list are entered into the master :ref:`history array` .. seealso:: - The script `test_chwirut_uniform_sampling_one_residual_at_a_time.py`_ - specifies fields to be used by the allocation function. + - `test_chwirut_uniform_sampling_one_residual_at_a_time.py`_ specifies fields + to be used by the allocation function ``give_sim_work_first`` from + fast_alloc_and_pausing.py_. .. literalinclude:: ../../libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py :start-at: alloc_specs :end-before: end_alloc_specs_rst_tag .. _test_chwirut_uniform_sampling_one_residual_at_a_time.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +.. _fast_alloc_and_pausing.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/fast_alloc_and_pausing.py diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index bad2c34f6..566cb5040 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -31,15 +31,14 @@ to main ``libE()`` routine:: .. _gen-specs-exmple1: - From test_6-hump_camel_uniform_sampling.py_ + - test_6-hump_camel_uniform_sampling.py_ declares In this example, the + generation function ``uniform_random_sample`` in sampling.py_ will generate 500 random + points uniformly over the 2D domain defined by ``gen_specs['ub']`` and + ``gen_specs['lb']``. .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py :start-at: gen_specs :end-before: end_gen_specs_rst_tag - In this example, the generation function ``uniform_random_sample`` will generate 500 random points - uniformly over the 2D domain defined by ``gen_specs['ub']`` and ``gen_specs['lb']``. - The libEnsemble manager is set to dump the history array to file after every 300 generated points, - though in this case it will only happen after 500 points due to the batch size. - +.. _sampling.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/gen_funcs/sampling.py .. _test_6-hump_camel_uniform_sampling.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 29fa51f8c..73761a7f9 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -17,8 +17,10 @@ from the allocation function. .. seealso:: - From: ``libEnsemble/tests/regression_tests/support.py`` + From: support.py_ .. literalinclude:: ../../libensemble/tests/regression_tests/support.py :start-at: persis_info_1 :end-before: end_persis_info_rst_tag + +.. _support.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/support.py diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index cdabafa7e..09780d0ab 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -1,6 +1,11 @@ User Function API ----------------- +libEnsemble requires functions for generation, simulation and allocation. + +While libEnsemble provides a default allocation function, the sim and gen functions +must be provided. The required API and examples are given here. + **The libEnsemble History Array** libEnsemble uses a NumPy structured array :ref:`H` to diff --git a/docs/user_funcs.rst b/docs/user_funcs.rst index 45533e3c8..65a516a45 100644 --- a/docs/user_funcs.rst +++ b/docs/user_funcs.rst @@ -1,8 +1,3 @@ -libEnsemble requires functions for generation, simulation and allocation. - -While libEnsemble provides a default allocation function, the sim and gen functions -must be provided. The required API and examples are given here. - .. toctree:: :caption: libEnsemble User Functions: diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index d6263f0dc..f43f35567 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -18,8 +18,8 @@ # Import libEnsemble items for this test from libensemble.libE import libE -from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f -from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.sim_funcs.six_hump_camel import six_hump_camel +from libensemble.gen_funcs.sampling import uniform_random_sample from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima @@ -27,13 +27,13 @@ libE_specs['save_every_k_sims'] = 400 libE_specs['save_every_k_gens'] = 300 -sim_specs = {'sim_f': sim_f, # Function whose output is being minimized - 'in': ['x'], # Keys to be given to sim_f - 'out': [('f', float)], # Name of the outputs from sim_f +sim_specs = {'sim_f': six_hump_camel, # Function whose output is being minimized + 'in': ['x'], # Keys to be given to sim_f + 'out': [('f', float)], # Name of the outputs from sim_f } # end_sim_specs_rst_tag -gen_specs = {'gen_f': gen_f, # Function generating sim_f input +gen_specs = {'gen_f': uniform_random_sample, # Function generating sim_f input 'out': [('x', float, (2,))], # Tell libE gen_f output, type, size 'user': {'gen_batch_size': 500, # Used by this specific gen_f 'lb': np.array([-3, -2]), # Used by this specific gen_f diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 1d64a2eee..c7af67c49 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -25,7 +25,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample_obj_components as gen_f -from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f +from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first from libensemble.tests.regression_tests.support import persis_info_3 as persis_info from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream @@ -64,7 +64,7 @@ 'components': m} } -alloc_specs = {'alloc_f': alloc_f, # Allocation function +alloc_specs = {'alloc_f': give_sim_work_first, # Allocation function 'out': [('allocated', bool)], # Output fields (included in History) 'user': {'stop_on_NaNs': True, # Should alloc preempt evals 'batch_mode': True, # Wait until all sim evals are done From 339af2e5208c9b3eb61953e33ba24be3b72d3e82 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 16:56:35 -0600 Subject: [PATCH 474/644] whitespace --- README.rst | 3 +-- docs/data_structures/gen_specs.rst | 2 +- docs/data_structures/sim_specs.rst | 1 - libensemble/gen_funcs/aposmm.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index a0dec080f..8f530d2da 100644 --- a/README.rst +++ b/README.rst @@ -149,7 +149,6 @@ Resources url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } - Quickstart Guide ---------------- @@ -238,7 +237,7 @@ in the top-level directory containing the setup script. Coverage reports are produced separately for unit tests and regression tests under the relevant directories. For parallel tests, the union of all processors is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after ``run_tests.sh`` is completed at +which can be viewed after ``run_tests.sh`` is completed at ``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are available online at Coveralls_. diff --git a/docs/data_structures/gen_specs.rst b/docs/data_structures/gen_specs.rst index 566cb5040..6e5dec3ac 100644 --- a/docs/data_structures/gen_specs.rst +++ b/docs/data_structures/gen_specs.rst @@ -34,7 +34,7 @@ to main ``libE()`` routine:: - test_6-hump_camel_uniform_sampling.py_ declares In this example, the generation function ``uniform_random_sample`` in sampling.py_ will generate 500 random points uniformly over the 2D domain defined by ``gen_specs['ub']`` and - ``gen_specs['lb']``. + ``gen_specs['lb']``. .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py :start-at: gen_specs diff --git a/docs/data_structures/sim_specs.rst b/docs/data_structures/sim_specs.rst index e3e353b8a..9cc28969f 100644 --- a/docs/data_structures/sim_specs.rst +++ b/docs/data_structures/sim_specs.rst @@ -49,7 +49,6 @@ Used to specify the simulation function, its inputs and outputs, and user data:: :start-at: sim_f :end-before: end_sim_specs_rst_tag - .. _forces_simf.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/forces_simf.py .. _run_libe_forces.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/run_libe_forces.py .. _test_6-hump_camel_uniform_sampling.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index e1816a00e..009411429 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -1,7 +1,7 @@ """ This module contains methods used our implementation of the Asynchronously Parallel Optimization Solver for finding Multiple Minima (APOSMM) method -described in detail in :cite:`LW16` +described in detail in :cite:`LW16`. """ __all__ = ['aposmm_logic', 'initialize_APOSMM', 'decide_where_to_start_localopt', 'update_history_dist'] From 4a49ad58c65821d7fb2690e2a0b85208b9f9603e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 8 Nov 2019 17:03:04 -0600 Subject: [PATCH 475/644] Adding back note --- README.rst | 5 +++++ docs/platforms/bebop.rst | 1 - docs/platforms/example_scripts.rst | 3 --- docs/platforms/platforms_index.rst | 6 ------ docs/platforms/theta.rst | 1 - docs/tutorials/local_sine_tutorial.rst | 1 - 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 8f530d2da..7da1c07a3 100644 --- a/README.rst +++ b/README.rst @@ -241,6 +241,11 @@ which can be viewed after ``run_tests.sh`` is completed at ``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are available online at Coveralls_. +.. note:: + The job_controller tests can be run using the direct-launch or + Balsam job controllers. Balsam integration with libEnsemble is now tested + via ``test_balsam_hworld.py``. + Basic Usage ~~~~~~~~~~~ diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 71140c193..c72697025 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -73,7 +73,6 @@ the workers to launch calculations with the job-controller or a job-launch comma more MPI process than the number of allocated nodes. The Manager and first worker run together on a node. - If you would like to interact directly with the compute nodes via a shell, the following showcases starting a bash session on a Knights Landing node for thirty minutes:: diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 9300b8abc..3c9fe8d1b 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -11,21 +11,18 @@ Bebop - Central Mode .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh :language: bash - Bebop - Distributed Mode ------------------------ .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm.sh :language: bash - Blues ----- .. literalinclude:: ../../examples/job_submission_scripts/blues_script.pbs :language: bash - Theta - Central Mode with Balsam -------------------------------- diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 3716ebf47..d0ee1096a 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,4 +1,3 @@ - libEnsemble has been largely developed, supported, and tested on Linux distributions and macOS, from laptops to thousands of compute-nodes. Although libEnsemble and most user functions are cross-platform compatible, there are @@ -14,7 +13,6 @@ We recommend installing libEnsemble and it's dependencies in a virtual environme created either through ``conda create``, ``virtualenv``, or ``python -m venv``, depending on how Python is installed. - HPC Systems =========== @@ -29,8 +27,6 @@ job-launch command can execute calculations on the other allocated nodes: :scale: 75 :align: center - - Alternatively, in *distributed* mode, each worker process runs independently of other workers directly on one or more allocated nodes: @@ -39,7 +35,6 @@ other workers directly on one or more allocated nodes: :scale: 75 :align: center - .. note:: Certain machines (like Theta and Summit) that can only submit MPI jobs from @@ -60,5 +55,4 @@ Read more about configuring and launching libEnsemble on some HPC systems: theta example_scripts - .. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 957f9f87b..2c6cdfa33 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -24,7 +24,6 @@ Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), and compute nodes. Users on login nodes submit batch runs to the MOM nodes. MOM nodes execute user batch-scripts to run on the compute nodes. - Configuring Python ------------------ diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 6cfcdaf80..f76a4d04b 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -193,7 +193,6 @@ inputs and outputs from those functions to expect. } } - sim_specs = {'sim_f': sim_find_sine, # Our simulator function 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') From ec447c16b8a932f3cedea098d3d9584af2a54afb Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 10:01:46 -0600 Subject: [PATCH 476/644] split README into quick-content and overview_usecases --- README.rst | 127 +++++++++---------------------------- docs/latex_index.rst | 1 + docs/overview_usecases.rst | 65 +++++++++++++++++++ 3 files changed, 97 insertions(+), 96 deletions(-) create mode 100644 docs/overview_usecases.rst diff --git a/README.rst b/README.rst index 7da1c07a3..d46ba2f6e 100644 --- a/README.rst +++ b/README.rst @@ -53,102 +53,6 @@ many-node simulations. A job controller interface is provided to ensure scripts are portable, resilient and flexible; it also enables automatic detection of the nodes and cores in a system and can split up jobs automatically if resource data isn't supplied. - -Overview --------- -libEnsemble is a Python library to coordinate the concurrent evaluation of -dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to -various workers. A libEnsemble worker is the smallest indivisible unit to -perform some calculation. The work performed by libEnsemble is governed by -three routines: - -* :ref:`gen_f`: Generates inputs to ``sim_f``. -* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. -* :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. - -Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts -be found in the ``examples/`` directory. To enable portability, a -:doc:`job_controller` -interface is supplied for users to launch and monitor jobs in their -user-provided ``sim_f`` and ``gen_f`` routines. - -The default ``alloc_f`` tells each available worker to call ``sim_f`` with the -highest priority unit of work from ``gen_f``. If a worker is idle and there is -no ``gen_f`` output to give, the worker is told to call ``gen_f``. - -Example Use Cases -~~~~~~~~~~~~~~~~~ -Below are some expected libEnsemble use cases that we support (or are working -to support) and plan to have examples of: - -* A user is looking to optimize a simulation calculation. The simulation may - already be using parallel resources, but not a large fraction of some - computer. libEnsemble can coordinate the concurrent evaluation of the - simulation ``sim_f`` at various parameter values and ``gen_f`` would return - candidate parameter values (possibly after each ``sim_f`` output). - -* A user has a ``gen_f`` that produces different meshes to be used within a - ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or - produce a new mesh. libEnsemble can ensure that the calculated meshes can be - used by multiple simulations without requiring movement of data. - -* A user is attempting to sample a simulation ``sim_f`` at some parameter - values, many of which will cause the simulation to fail. libEnsemble can stop - unresponsive evaluations, and recover computational resources for future - evaluations. ``gen_f`` can possibly update the sampling after discovering - regions where evaluations of ``sim_f`` fail. - -* A user has a simulation ``sim_f`` that requires calculating multiple - expensive quantities, some of which depend on other quantities. ``sim_f`` can - observe intermediate quantities in order to stop related calculations and - preempt future calculations associated with poor parameter values. - -* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity - evaluations requiring more computational resources, and a - ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and - at what fidelity level. libEnsemble can coordinate these evaluations without - requiring the user know parallel programming. - -* A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, - sensitivity analysis is desired at each identified optimum. libEnsemble can - use the points from the APOSMM ``gen_f`` to identify optima; and after a - point is ruled to be an optimum, a different ``gen_f`` can produce a - collection of parameters necessary for sensitivity analysis of ``sim_f``. - -Naturally, combinations of these use cases are supported as well. An example of -such a combination is using libEnsemble to solve an optimization problem that -relies on simulations that fail frequently. - -Resources -~~~~~~~~~~~~~~~~~~ - -**Support:** - -- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. -- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. -- Join the `libEnsemble mailing list`_ for updates about new releases. - -**Further Information:** - -- Documentation is provided by ReadtheDocs_. -- A visual overview of libEnsemble is given in this poster_. - -**Citation:** - -- Please use the following to cite libEnsemble in a publication: - -.. code-block:: bibtex - - @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and - David Bindel and John-Luke Navarro}, - title = {{libEnsemble} Users Manual}, - institution = {Argonne National Laboratory}, - number = {Revision 0.5.2}, - year = {2019}, - url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} - } - Quickstart Guide ---------------- @@ -272,6 +176,37 @@ When specifying these options via command line options, one may use the See the `user guide`_ for more information. + +Resources +~~~~~~~~~~~~~~~~~~ + +**Support:** + +- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. +- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. +- Join the `libEnsemble mailing list`_ for updates about new releases. + +**Further Information:** + +- Documentation is provided by ReadtheDocs_. +- A visual overview of libEnsemble is given in this poster_. + +**Citation:** + +- Please use the following to cite libEnsemble in a publication: + +.. code-block:: bibtex + + @techreport{libEnsemble, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, + title = {{libEnsemble} Users Manual}, + institution = {Argonne National Laboratory}, + number = {Revision 0.5.2}, + year = {2019}, + url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + } + .. _Balsam: https://www.alcf.anl.gov/balsam .. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master diff --git a/docs/latex_index.rst b/docs/latex_index.rst index baaccf059..b5f713c5f 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -6,6 +6,7 @@ :maxdepth: 3 Introduction + overview_usecases programming_libE platforms/platforms dev_info diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst new file mode 100644 index 000000000..5c2bbbc80 --- /dev/null +++ b/docs/overview_usecases.rst @@ -0,0 +1,65 @@ + +Overview +-------- +libEnsemble is a Python library to coordinate the concurrent evaluation of +dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to +various workers. A libEnsemble worker is the smallest indivisible unit to +perform some calculation. The work performed by libEnsemble is governed by +three routines: + +* :ref:`gen_f`: Generates inputs to ``sim_f``. +* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. +* :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. + +Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts +be found in the ``examples/`` directory. To enable portability, a +:doc:`job_controller` +interface is supplied for users to launch and monitor jobs in their +user-provided ``sim_f`` and ``gen_f`` routines. + +The default ``alloc_f`` tells each available worker to call ``sim_f`` with the +highest priority unit of work from ``gen_f``. If a worker is idle and there is +no ``gen_f`` output to give, the worker is told to call ``gen_f``. + +Example Use Cases +~~~~~~~~~~~~~~~~~ +Below are some expected libEnsemble use cases that we support (or are working +to support) and plan to have examples of: + +* A user is looking to optimize a simulation calculation. The simulation may + already be using parallel resources, but not a large fraction of some + computer. libEnsemble can coordinate the concurrent evaluation of the + simulation ``sim_f`` at various parameter values and ``gen_f`` would return + candidate parameter values (possibly after each ``sim_f`` output). + +* A user has a ``gen_f`` that produces different meshes to be used within a + ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or + produce a new mesh. libEnsemble can ensure that the calculated meshes can be + used by multiple simulations without requiring movement of data. + +* A user is attempting to sample a simulation ``sim_f`` at some parameter + values, many of which will cause the simulation to fail. libEnsemble can stop + unresponsive evaluations, and recover computational resources for future + evaluations. ``gen_f`` can possibly update the sampling after discovering + regions where evaluations of ``sim_f`` fail. + +* A user has a simulation ``sim_f`` that requires calculating multiple + expensive quantities, some of which depend on other quantities. ``sim_f`` can + observe intermediate quantities in order to stop related calculations and + preempt future calculations associated with poor parameter values. + +* A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity + evaluations requiring more computational resources, and a + ``gen_f``/``alloc_f`` that decides which parameters should be evaluated and + at what fidelity level. libEnsemble can coordinate these evaluations without + requiring the user know parallel programming. + +* A user wishes to identify multiple local optima for a ``sim_f``. Furthermore, + sensitivity analysis is desired at each identified optimum. libEnsemble can + use the points from the APOSMM ``gen_f`` to identify optima; and after a + point is ruled to be an optimum, a different ``gen_f`` can produce a + collection of parameters necessary for sensitivity analysis of ``sim_f``. + +Naturally, combinations of these use cases are supported as well. An example of +such a combination is using libEnsemble to solve an optimization problem that +relies on simulations that fail frequently. From 4bd4d63842dbc441cac8acfe2b455519c2932a21 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 10:45:03 -0600 Subject: [PATCH 477/644] new HTML toctree structure, new files for issues and utilities, platforms added to introduction --- docs/index.rst | 30 +++++++++++++++++++----------- docs/introduction.rst | 2 ++ docs/known_issues.rst | 12 ++++++++++++ docs/overview_usecases.rst | 4 ++-- docs/platforms/platforms_index.rst | 15 --------------- docs/utilities.rst | 7 +++++++ 6 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 docs/known_issues.rst create mode 100644 docs/utilities.rst diff --git a/docs/index.rst b/docs/index.rst index 6d2860a58..2abb5df85 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,27 +7,35 @@ .. toctree:: :maxdepth: 2 - :caption: Getting Started: - + :caption: User Guide: Introduction + overview_usecases programming_libE + data_structures/data_structures + user_funcs + job_controller/jc_index + logging + utilities .. toctree:: - :caption: Further information: :maxdepth: 2 - - platforms/platforms - dev_info + :caption: Tutorials: + tutorials/local_sine_tutorial .. toctree:: - :caption: In Depth: :maxdepth: 2 - - tutorials/tutorials + :caption: Additional Reference: FAQ - examples/examples_index - bibliography release_notes + contributing + known_issues + +.. toctree:: + :maxdepth: 2 + :caption: Developer Guide: + dev_guide/release_management/release_index + dev_guide/dev_API/developer_API + Indices and tables ================== diff --git a/docs/introduction.rst b/docs/introduction.rst index 0152356dc..490299cb6 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -1,2 +1,4 @@ .. include:: ../README.rst :start-after: after_badges_rst_tag + +.. include:: platforms/platforms_index diff --git a/docs/known_issues.rst b/docs/known_issues.rst new file mode 100644 index 000000000..ccad162ee --- /dev/null +++ b/docs/known_issues.rst @@ -0,0 +1,12 @@ +Known Issues +============ + +The following selection describes known bugs, errors, or other difficulties that +may occur when using libEnsemble. + +Coming soon. + + + +Platform-specific issues +------------------------ diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 5c2bbbc80..e076dcb94 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,6 +1,6 @@ -Overview --------- +Architecture Overview +--------------------- libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to various workers. A libEnsemble worker is the smallest indivisible unit to diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index d0ee1096a..24a6c98c8 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,18 +1,3 @@ -libEnsemble has been largely developed, supported, and tested on Linux -distributions and macOS, from laptops to thousands of compute-nodes. Although -libEnsemble and most user functions are cross-platform compatible, there are -platform-specific differences for installing and configuring libEnsemble. - -Personal Computers -================== - -Users interested in installing and running libEnsemble on their personal machines -are encouraged to start by reading the Quickstart guide :doc:`here<../quickstart>`. - -We recommend installing libEnsemble and it's dependencies in a virtual environment, -created either through ``conda create``, ``virtualenv``, or ``python -m venv``, -depending on how Python is installed. - HPC Systems =========== diff --git a/docs/utilities.rst b/docs/utilities.rst new file mode 100644 index 000000000..2285801ff --- /dev/null +++ b/docs/utilities.rst @@ -0,0 +1,7 @@ +libEnsemble Utilities +===================== + +libEnsemble features several scripts and tools to assist in writing consistent +calling scripts and user functions. + +Coming soon. From 42a66688b3ff32aa4a8a1c565a2eb2b7a1fecad6 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 12:20:14 -0600 Subject: [PATCH 478/644] Moves HPC content, revises some headings --- README.rst | 7 ++++--- docs/index.rst | 4 ++++ docs/introduction.rst | 16 +++++++++++++++- docs/platforms/platforms_index.rst | 10 ---------- docs/utilities.rst | 4 ++-- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index d46ba2f6e..506a1bb72 100644 --- a/README.rst +++ b/README.rst @@ -53,8 +53,6 @@ many-node simulations. A job controller interface is provided to ensure scripts are portable, resilient and flexible; it also enables automatic detection of the nodes and cores in a system and can split up jobs automatically if resource data isn't supplied. -Quickstart Guide ----------------- Dependencies ~~~~~~~~~~~~ @@ -178,7 +176,7 @@ See the `user guide`_ for more information. Resources -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~ **Support:** @@ -207,6 +205,9 @@ Resources url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } + + + .. _Balsam: https://www.alcf.anl.gov/balsam .. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master diff --git a/docs/index.rst b/docs/index.rst index 2abb5df85..4887682da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,7 @@ .. toctree:: :maxdepth: 2 :caption: User Guide: + Introduction overview_usecases programming_libE @@ -20,11 +21,13 @@ .. toctree:: :maxdepth: 2 :caption: Tutorials: + tutorials/local_sine_tutorial .. toctree:: :maxdepth: 2 :caption: Additional Reference: + FAQ release_notes contributing @@ -33,6 +36,7 @@ .. toctree:: :maxdepth: 2 :caption: Developer Guide: + dev_guide/release_management/release_index dev_guide/dev_API/developer_API diff --git a/docs/introduction.rst b/docs/introduction.rst index 490299cb6..c45471a45 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -1,4 +1,18 @@ .. include:: ../README.rst :start-after: after_badges_rst_tag -.. include:: platforms/platforms_index +Support for HPC +~~~~~~~~~~~~~~~ + +libEnsemble's flexible architecture was developed with HPC System support and +scaling as a central focus, and can perform calculations on both laptops and +thousands of compute-nodes. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + platforms/platforms_index + platforms/bebop + platforms/theta + platforms/example_scripts diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 24a6c98c8..3b8ac53f3 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -30,14 +30,4 @@ differently. On these machines, libEnsemble is run centralized on either a compute-node with the support of Balsam_ or on a frontend server called a MOM (Machine-Oriented Mini-server) node. -Read more about configuring and launching libEnsemble on some HPC systems: - -.. toctree:: - :maxdepth: 2 - :titlesonly: - - bebop - theta - example_scripts - .. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/utilities.rst b/docs/utilities.rst index 2285801ff..03ff9a5cc 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -1,5 +1,5 @@ -libEnsemble Utilities -===================== +Utilities +========= libEnsemble features several scripts and tools to assist in writing consistent calling scripts and user functions. From 89d164df5ca2a8e2f2eb39d7126a99dcc7d4ae89 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 13:11:18 -0600 Subject: [PATCH 479/644] move check_inputs to utilities --- docs/data_structures/data_structures.rst | 9 --------- docs/utilities.rst | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index f86c9042c..1ab157e17 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -20,15 +20,6 @@ and the :ref:`work` dictionary produced by the allocation function. -.. note:: - Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` - dictionary with the ``check_inputs()`` function from the ``utils`` module. - Provide any combination of these data structures as keyword arguments. - For example:: - - from libensemble.libE import check_inputs - - check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) .. toctree:: :maxdepth: 3 diff --git a/docs/utilities.rst b/docs/utilities.rst index 03ff9a5cc..ce3dc5bad 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -1,7 +1,17 @@ Utilities ========= -libEnsemble features several scripts and tools to assist in writing consistent +libEnsemble features several modules and tools to assist in writing consistent calling scripts and user functions. -Coming soon. +libE input consistency +---------------------- + +Users can check the formatting and consistency of ``exit_criteria`` and each +``specs`` dictionary with the ``check_inputs()`` function from the ``utils`` +module. Provide any combination of these data structures as keyword arguments. +For example:: + + from libensemble.libE import check_inputs + + check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) From 44353f0f478c8031e37762eabb5ba45be13cd9ad Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 13:35:35 -0600 Subject: [PATCH 480/644] separate libe module, move known issues, header adjustments --- docs/index.rst | 2 +- docs/known_issues.rst | 11 +++++------ docs/libe_module.rst | 6 ++++++ docs/overview_usecases.rst | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 docs/libe_module.rst diff --git a/docs/index.rst b/docs/index.rst index 4887682da..4e6c889ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,7 @@ Introduction overview_usecases - programming_libE + libe_module data_structures/data_structures user_funcs job_controller/jc_index diff --git a/docs/known_issues.rst b/docs/known_issues.rst index ccad162ee..1d9c99087 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -4,9 +4,8 @@ Known Issues The following selection describes known bugs, errors, or other difficulties that may occur when using libEnsemble. -Coming soon. - - - -Platform-specific issues ------------------------- +* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches + (Either use local mode or Balsam job controller). +* Local comms mode (multiprocessing) may fail if MPI is initialized before forking processors. This is thought to be responsible for issues combining with PETSc. +* Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. +* TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. diff --git a/docs/libe_module.rst b/docs/libe_module.rst new file mode 100644 index 000000000..ddbacb128 --- /dev/null +++ b/docs/libe_module.rst @@ -0,0 +1,6 @@ +Main libEnsemble module +======================= + +.. automodule:: libE + :members: + :no-undoc-members: diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index e076dcb94..c7973e198 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,5 +1,5 @@ -Architecture Overview +Architecture overview --------------------- libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to From 91fbe4e5c299275a40d5ab121067c3279b330db2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 16:21:32 -0600 Subject: [PATCH 481/644] rename headings, re-add H and output info, update utilities and known issues, move HPC to own heading under user guide --- README.rst | 3 +- docs/index.rst | 3 +- docs/introduction.rst | 19 ++-------- docs/known_issues.rst | 16 ++++++--- docs/overview_usecases.rst | 56 ++++++++++++++++++++++++++++++ docs/platforms/platforms_index.rst | 15 ++++++++ docs/utilities.rst | 35 +++++++++++++++++-- docs/welcome.rst | 2 +- 8 files changed, 123 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index 506a1bb72..ac574dfeb 100644 --- a/README.rst +++ b/README.rst @@ -18,12 +18,13 @@ :alt: Documentation Status | -.. after_badges_rst_tag =========================== Introduction to libEnsemble =========================== +.. after_badges_rst_tag + libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. The library is developed to use massively parallel resources to accelerate the solution of design, decision, and diff --git a/docs/index.rst b/docs/index.rst index 4e6c889ca..ce8078017 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ :maxdepth: 2 :caption: User Guide: - Introduction + Quickstart overview_usecases libe_module data_structures/data_structures @@ -17,6 +17,7 @@ job_controller/jc_index logging utilities + platforms/platforms_index .. toctree:: :maxdepth: 2 diff --git a/docs/introduction.rst b/docs/introduction.rst index c45471a45..e62652fa2 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -1,18 +1,5 @@ +Quickstart +========== + .. include:: ../README.rst :start-after: after_badges_rst_tag - -Support for HPC -~~~~~~~~~~~~~~~ - -libEnsemble's flexible architecture was developed with HPC System support and -scaling as a central focus, and can perform calculations on both laptops and -thousands of compute-nodes. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - - platforms/platforms_index - platforms/bebop - platforms/theta - platforms/example_scripts diff --git a/docs/known_issues.rst b/docs/known_issues.rst index 1d9c99087..01dfd1afc 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -4,8 +4,14 @@ Known Issues The following selection describes known bugs, errors, or other difficulties that may occur when using libEnsemble. -* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches - (Either use local mode or Balsam job controller). -* Local comms mode (multiprocessing) may fail if MPI is initialized before forking processors. This is thought to be responsible for issues combining with PETSc. -* Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. -* TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. +* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it +does not support nested MPI launches (Either use local mode or Balsam job controller). +* Local comms mode (multiprocessing) may fail if MPI is initialized before forking +processors. This is thought to be responsible for issues combining with PETSc. +* Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number +of physical cores as SMT info not available. +* TCP mode does not support: 1) more than one libEnsemble call in a given script +or 2) the auto-resources option to the job controller. +* libEnsemble may hang on systems with matching probes not enabled on + the native fabric, like on Intel's Truescale (TMI) fabric for + instance. See the :doc:`FAQ` for more information. diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index c7973e198..42d8fb71d 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -63,3 +63,59 @@ to support) and plan to have examples of: Naturally, combinations of these use cases are supported as well. An example of such a combination is using libEnsemble to solve an optimization problem that relies on simulations that fail frequently. + +The History Array +~~~~~~~~~~~~~~~~~ +libEnsemble uses a NumPy structured array :ref:`H` to +store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, +``gen_f`` and ``sim_f`` are expected to return output in NumPy structured +arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` +must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output +from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will +include the following fields: + +* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an + associated ``sim_id``. The generator can assign this, but users must be + careful to ensure points are added in order. For example, ``if alloc_f`` + allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` + should ensure that both don’t generate points with the same ``sim_id``. + +* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble + worker to be evaluated yet? + +* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` + output given to a worker? + +* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. + +* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` + +* ``gen_time`` [float]: At what time (since the epoch) was this entry (or + collection of entries) put into ``H`` by the manager + +* ``returned`` [bool]: Has this worker completed the evaluation of this unit of + work? + +Output +~~~~~~ +The history array :ref:`H` and +:ref:`persis_info` dictionary are returned to the user +by libEnsemble. If libEnsemble aborts on an exception, these structures are +dumped to the respective files, + +* ``libE_history_at_abort_.npy`` +* ``libE_history_at_abort_.pickle`` + +where ``sim_count`` is the number of points evaluated. + +Other libEnsemble files produced by default are: + +* ``libE_stats.txt``: This contains a one-line summary of all user + calculations. Each calculation summary is sent by workers to the manager and + printed as the run progresses. + +* ``ensemble.log``: This is the logging output from libEnsemble. The default + logging is at INFO level. To gain additional diagnostics logging level can be + set to DEBUG. If this file is not removed, multiple runs will append output. + Messages at or above level MANAGER_WARNING are also copied to stderr to alert + the user promptly. For more info, see :doc:`Logging`. diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 3b8ac53f3..b0edc3968 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,6 +1,11 @@ HPC Systems =========== +libEnsemble has been developed, supported, and tested on systems of highly varying +scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly +parallel scaling capabilities are best exemplified on the resources available +within high-performance machines. + libEnsemble's flexible architecture lends it best to two general modes of worker distributions across allocated compute nodes. The first mode we refer to as *centralized* mode, where the libEnsemble manager and worker processes @@ -30,4 +35,14 @@ differently. On these machines, libEnsemble is run centralized on either a compute-node with the support of Balsam_ or on a frontend server called a MOM (Machine-Oriented Mini-server) node. +Read more about configuring and launching libEnsemble on some HPC systems: + +.. toctree:: + :maxdepth: 2 + :titlesonly: + + bebop + theta + example_scripts + .. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/utilities.rst b/docs/utilities.rst index ce3dc5bad..87d4734cc 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -4,8 +4,8 @@ Utilities libEnsemble features several modules and tools to assist in writing consistent calling scripts and user functions. -libE input consistency ----------------------- +Input consistency +----------------- Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` dictionary with the ``check_inputs()`` function from the ``utils`` @@ -15,3 +15,34 @@ For example:: from libensemble.libE import check_inputs check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) + +Parameters as command-line arguments +------------------------------------ + +The ``parse_args()`` function can be used to pass common libEnsemble parameters as +command-line arguments. + +In your calling script:: + + from libensemble.tests.regression_tests.common import parse_args + nworkers, is_master, libE_specs, misc_args = parse_args() + +From the shell, for example:: + + $ python calling_script --comms local --nworkers 4 + +Usage: + +.. code-block:: bash + + usage: test_... [-h] [--comms [{local,tcp,ssh,client,mpi}]] + [--nworkers [NWORKERS]] [--workers WORKERS [WORKERS ...]] + [--workerID [WORKERID]] [--server SERVER SERVER SERVER] + [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] + [--worker_python [WORKER_PYTHON]] + [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] + +.. note:: + Since this utility was developed to support automatic testing, it currently + only accepts calling scripts prepended with ``test_`` and isn't necessarily + recommended for production purposes. diff --git a/docs/welcome.rst b/docs/welcome.rst index 7decf8888..47d45369e 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -27,5 +27,5 @@ libEnsemble is a library to coordinate the concurrent evaluation of dynamic ense * New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`User Guide`. +* Go in-depth by reading the :doc:`Overview`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. From 1d929ee907539282f88701cfee00b03237d1b92f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 13 Nov 2019 16:41:22 -0600 Subject: [PATCH 482/644] How's this for html/pdf harmony --- README.rst | 180 +------------------------------------ docs/latex_index.rst | 3 +- docs/overview_usecases.rst | 4 +- 3 files changed, 5 insertions(+), 182 deletions(-) diff --git a/README.rst b/README.rst index 506a1bb72..a8b092d75 100644 --- a/README.rst +++ b/README.rst @@ -54,183 +54,7 @@ are portable, resilient and flexible; it also enables automatic detection of the nodes and cores in a system and can split up jobs automatically if resource data isn't supplied. -Dependencies -~~~~~~~~~~~~ +.. only:: html -Required dependencies: + .. include:: deps.rst -* Python_ 3.5 or above. -* NumPy_ - -For libEnsemble running with the mpi4py parallelism: - -* A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries. -* mpi4py_ v2.0.0 or above - -Optional dependency: - -* Balsam_ - -From v0.2.0, libEnsemble has the option of using the Balsam job manager. This -is required for running libEnsemble on the compute nodes of some supercomputing -platforms (e.g., Cray XC40); platforms that do not support launching jobs from -compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the -launch nodes using multiprocessing. - -The example sim and gen functions and tests require the following dependencies: - -* SciPy_ -* petsc4py_ -* PETSc_ - This can optionally be installed by pip along with petsc4py -* NLopt_ - Installed with `shared libraries enabled`_. - -PETSc and NLopt must be built with shared libraries enabled and present in -``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also -require SWIG_ to be installed on certain systems. - -Installation -~~~~~~~~~~~~ - -Use pip to install libEnsemble and its dependencies:: - - pip install libensemble - -libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: - - spack install py-libensemble - -The tests and examples can be accessed in the GitHub_ repository. -If necessary, you may install all optional dependencies (listed above) at once with:: - - pip install libensemble[extras] - -A tarball_ of the most recent release is also available. - -Testing -~~~~~~~ - -The provided test suite includes both unit and regression tests and is run -regularly on: - -* `Travis CI`_ - -The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ -packages to be installed and can be run from the libensemble/tests directory of -the source distribution by running:: - - ./run-tests.sh - -To clean the test repositories run:: - - ./run-tests.sh -c - -Further options are available. To see a complete list of options run:: - - ./run-tests.sh -h - -If you have the source distribution, you can download (but not install) the testing -prerequisites and run the tests with:: - - python setup.py test - -in the top-level directory containing the setup script. - -Coverage reports are produced separately for unit tests and regression tests -under the relevant directories. For parallel tests, the union of all processors -is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after ``run_tests.sh`` is completed at -``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are -available online at Coveralls_. - -.. note:: - The job_controller tests can be run using the direct-launch or - Balsam job controllers. Balsam integration with libEnsemble is now tested - via ``test_balsam_hworld.py``. - -Basic Usage -~~~~~~~~~~~ - -The examples directory contains example libEnsemble calling scripts, sim -functions, gen functions, alloc functions and job submission scripts. - -The default manager/worker communications mode is MPI. The user script is -launched as:: - - mpiexec -np N python myscript.py - -where ``N`` is the number of processors. This will launch one manager and -``N-1`` workers. - -If running in local mode, which uses Python's multiprocessing module, the -``local`` comms option and the number of workers must be specified. The script -can then be run as a regular python script:: - - python myscript.py - -When specifying these options via command line options, one may use the -``parse_args`` function used in the regression tests, which can be found in -`common.py`_ in the ``libensemble/tests/regression_tests`` directory. - -See the `user guide`_ for more information. - - -Resources -~~~~~~~~~ - -**Support:** - -- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. -- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. -- Join the `libEnsemble mailing list`_ for updates about new releases. - -**Further Information:** - -- Documentation is provided by ReadtheDocs_. -- A visual overview of libEnsemble is given in this poster_. - -**Citation:** - -- Please use the following to cite libEnsemble in a publication: - -.. code-block:: bibtex - - @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and - David Bindel and John-Luke Navarro}, - title = {{libEnsemble} Users Manual}, - institution = {Argonne National Laboratory}, - number = {Revision 0.5.2}, - year = {2019}, - url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} - } - - - - -.. _Balsam: https://www.alcf.anl.gov/balsam -.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py -.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master -.. _GitHub: https://github.com/Libensemble/libensemble -.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble -.. _libEnsemble Slack page: https://libensemble.slack.com -.. _mock: https://pypi.org/project/mock -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _MPICH: http://www.mpich.org/ -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 -.. _pytest-cov: https://pypi.org/project/pytest-cov/ -.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ -.. _pytest: https://pypi.org/project/pytest/ -.. _Python: http://www.python.org -.. _ReadtheDocs: http://libensemble.readthedocs.org/ -.. _SciPy: http://www.scipy.org -.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries -.. _Spack: https://spack.readthedocs.io/en/latest -.. _SWIG: http://swig.org/ -.. _tarball: https://github.com/Libensemble/libensemble/releases/latest -.. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html diff --git a/docs/latex_index.rst b/docs/latex_index.rst index b5f713c5f..771299c70 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -5,8 +5,7 @@ .. toctree:: :maxdepth: 3 - Introduction - overview_usecases + Introduction programming_libE platforms/platforms dev_info diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index c7973e198..7ef8683ec 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,6 +1,6 @@ +Overview +~~~~~~~~ -Architecture overview ---------------------- libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to various workers. A libEnsemble worker is the smallest indivisible unit to From 0c6a3a6c0f98f0bea1f66c8da51b7c6a3933a76e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 13 Nov 2019 16:43:54 -0600 Subject: [PATCH 483/644] removing table complaints --- README.rst | 3 +- docs/deps.rst | 182 ++++++++++++++++++++++++++++++++++++ docs/introduction_latex.rst | 22 +++++ docs/known_issues.rst | 24 +++-- 4 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 docs/deps.rst create mode 100644 docs/introduction_latex.rst diff --git a/README.rst b/README.rst index c3232d617..a8b092d75 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,12 @@ :alt: Documentation Status | +.. after_badges_rst_tag =========================== Introduction to libEnsemble =========================== -.. after_badges_rst_tag - libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. The library is developed to use massively parallel resources to accelerate the solution of design, decision, and diff --git a/docs/deps.rst b/docs/deps.rst new file mode 100644 index 000000000..cac04e759 --- /dev/null +++ b/docs/deps.rst @@ -0,0 +1,182 @@ +Dependencies +~~~~~~~~~~~~ + +Required dependencies: + +* Python_ 3.5 or above. +* NumPy_ + +For libEnsemble running with the mpi4py parallelism: + +* A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries. +* mpi4py_ v2.0.0 or above + +Optional dependency: + +* Balsam_ + +From v0.2.0, libEnsemble has the option of using the Balsam job manager. This +is required for running libEnsemble on the compute nodes of some supercomputing +platforms (e.g., Cray XC40); platforms that do not support launching jobs from +compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the +launch nodes using multiprocessing. + +The example sim and gen functions and tests require the following dependencies: + +* SciPy_ +* petsc4py_ +* PETSc_ - This can optionally be installed by pip along with petsc4py +* NLopt_ - Installed with `shared libraries enabled`_. + +PETSc and NLopt must be built with shared libraries enabled and present in +``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt +should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also +require SWIG_ to be installed on certain systems. + +Installation +~~~~~~~~~~~~ + +Use pip to install libEnsemble and its dependencies:: + + pip install libensemble + +libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: + + spack install py-libensemble + +The tests and examples can be accessed in the GitHub_ repository. +If necessary, you may install all optional dependencies (listed above) at once with:: + + pip install libensemble[extras] + +A tarball_ of the most recent release is also available. + +Testing +~~~~~~~ + +The provided test suite includes both unit and regression tests and is run +regularly on: + +* `Travis CI`_ + +The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ +packages to be installed and can be run from the libensemble/tests directory of +the source distribution by running:: + + ./run-tests.sh + +To clean the test repositories run:: + + ./run-tests.sh -c + +Further options are available. To see a complete list of options run:: + + ./run-tests.sh -h + +If you have the source distribution, you can download (but not install) the testing +prerequisites and run the tests with:: + + python setup.py test + +in the top-level directory containing the setup script. + +Coverage reports are produced separately for unit tests and regression tests +under the relevant directories. For parallel tests, the union of all processors +is taken. Furthermore, a combined coverage report is created at the top level, +which can be viewed after ``run_tests.sh`` is completed at +``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are +available online at Coveralls_. + +.. note:: + The job_controller tests can be run using the direct-launch or + Balsam job controllers. Balsam integration with libEnsemble is now tested + via ``test_balsam_hworld.py``. + +Basic Usage +~~~~~~~~~~~ + +The examples directory contains example libEnsemble calling scripts, sim +functions, gen functions, alloc functions and job submission scripts. + +The default manager/worker communications mode is MPI. The user script is +launched as:: + + mpiexec -np N python myscript.py + +where ``N`` is the number of processors. This will launch one manager and +``N-1`` workers. + +If running in local mode, which uses Python's multiprocessing module, the +``local`` comms option and the number of workers must be specified. The script +can then be run as a regular python script:: + + python myscript.py + +When specifying these options via command line options, one may use the +``parse_args`` function used in the regression tests, which can be found in +`common.py`_ in the ``libensemble/tests/regression_tests`` directory. + +See the `user guide`_ for more information. + + +Resources +~~~~~~~~~ + +**Support:** + +- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. +- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. +- Join the `libEnsemble mailing list`_ for updates about new releases. + +**Further Information:** + +- Documentation is provided by ReadtheDocs_. +- A visual overview of libEnsemble is given in this poster_. + +**Citation:** + +- Please use the following to cite libEnsemble in a publication: + +.. code-block:: bibtex + + @techreport{libEnsemble, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, + title = {{libEnsemble} Users Manual}, + institution = {Argonne National Laboratory}, + number = {Revision 0.5.2}, + year = {2019}, + url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + } + + +.. after_resources_rst_tag + + + +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py +.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _GitHub: https://github.com/Libensemble/libensemble +.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble +.. _libEnsemble Slack page: https://libensemble.slack.com +.. _mock: https://pypi.org/project/mock +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _MPICH: http://www.mpich.org/ +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 +.. _pytest-cov: https://pypi.org/project/pytest-cov/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ +.. _pytest: https://pypi.org/project/pytest/ +.. _Python: http://www.python.org +.. _ReadtheDocs: http://libensemble.readthedocs.org/ +.. _SciPy: http://www.scipy.org +.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries +.. _Spack: https://spack.readthedocs.io/en/latest +.. _SWIG: http://swig.org/ +.. _tarball: https://github.com/Libensemble/libensemble/releases/latest +.. _Travis CI: https://travis-ci.org/Libensemble/libensemble +.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst new file mode 100644 index 000000000..d8a0f0786 --- /dev/null +++ b/docs/introduction_latex.rst @@ -0,0 +1,22 @@ +.. include:: ../README.rst + :start-after: after_badges_rst_tag + +.. include:: overview_usecases.rst + +.. include:: deps.rst + +Support for HPC +~~~~~~~~~~~~~~~ + +libEnsemble's flexible architecture was developed with HPC System support and +scaling as a central focus, and can perform calculations on both laptops and +thousands of compute-nodes. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + platforms/platforms_index + platforms/bebop + platforms/theta + platforms/example_scripts diff --git a/docs/known_issues.rst b/docs/known_issues.rst index 01dfd1afc..e93a8c936 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -4,14 +4,18 @@ Known Issues The following selection describes known bugs, errors, or other difficulties that may occur when using libEnsemble. -* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it -does not support nested MPI launches (Either use local mode or Balsam job controller). -* Local comms mode (multiprocessing) may fail if MPI is initialized before forking -processors. This is thought to be responsible for issues combining with PETSc. +* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as + it does not support nested MPI launches (Either use local mode or Balsam job + controller). +* Local comms mode (multiprocessing) may fail if MPI is initialized before + forking processors. This is thought to be responsible for issues combining + with PETSc. * Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number -of physical cores as SMT info not available. -* TCP mode does not support: 1) more than one libEnsemble call in a given script -or 2) the auto-resources option to the job controller. -* libEnsemble may hang on systems with matching probes not enabled on - the native fabric, like on Intel's Truescale (TMI) fabric for - instance. See the :doc:`FAQ` for more information. + of physical cores as SMT info not available. +* TCP mode does not support: + 1) more than one libEnsemble call in a given script or + 2) the auto-resources option to the job controller. +* libEnsemble may hang on systems with matching probes not enabled on the + native fabric, like on Intel's Truescale (TMI) fabric for instance. See the + :doc:`FAQ` for more information. + From fcf38b050333791c94632f6a16f818879149081e Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 13 Nov 2019 16:51:51 -0600 Subject: [PATCH 484/644] Revert "rename headings, re-add H and output info, update utilities and known issues, move HPC to own heading under user guide" This reverts commit 91fbe4e5c299275a40d5ab121067c3279b330db2. --- docs/index.rst | 3 +- docs/introduction.rst | 19 ++++++++-- docs/known_issues.rst | 8 +++++ docs/overview_usecases.rst | 56 ------------------------------ docs/platforms/platforms_index.rst | 15 -------- docs/utilities.rst | 35 ++----------------- docs/welcome.rst | 2 +- 7 files changed, 28 insertions(+), 110 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ce8078017..4e6c889ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ :maxdepth: 2 :caption: User Guide: - Quickstart + Introduction overview_usecases libe_module data_structures/data_structures @@ -17,7 +17,6 @@ job_controller/jc_index logging utilities - platforms/platforms_index .. toctree:: :maxdepth: 2 diff --git a/docs/introduction.rst b/docs/introduction.rst index e62652fa2..c45471a45 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -1,5 +1,18 @@ -Quickstart -========== - .. include:: ../README.rst :start-after: after_badges_rst_tag + +Support for HPC +~~~~~~~~~~~~~~~ + +libEnsemble's flexible architecture was developed with HPC System support and +scaling as a central focus, and can perform calculations on both laptops and +thousands of compute-nodes. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + platforms/platforms_index + platforms/bebop + platforms/theta + platforms/example_scripts diff --git a/docs/known_issues.rst b/docs/known_issues.rst index e93a8c936..b308fce26 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -4,6 +4,7 @@ Known Issues The following selection describes known bugs, errors, or other difficulties that may occur when using libEnsemble. +<<<<<<< HEAD * OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches (Either use local mode or Balsam job controller). @@ -19,3 +20,10 @@ may occur when using libEnsemble. native fabric, like on Intel's Truescale (TMI) fabric for instance. See the :doc:`FAQ` for more information. +======= +* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches + (Either use local mode or Balsam job controller). +* Local comms mode (multiprocessing) may fail if MPI is initialized before forking processors. This is thought to be responsible for issues combining with PETSc. +* Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. +* TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. +>>>>>>> parent of 91fbe4e... rename headings, re-add H and output info, update utilities and known issues, move HPC to own heading under user guide diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 0945bd3a9..7ef8683ec 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -63,59 +63,3 @@ to support) and plan to have examples of: Naturally, combinations of these use cases are supported as well. An example of such a combination is using libEnsemble to solve an optimization problem that relies on simulations that fail frequently. - -The History Array -~~~~~~~~~~~~~~~~~ -libEnsemble uses a NumPy structured array :ref:`H` to -store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, -``gen_f`` and ``sim_f`` are expected to return output in NumPy structured -arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` -must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output -from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will -include the following fields: - -* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an - associated ``sim_id``. The generator can assign this, but users must be - careful to ensure points are added in order. For example, ``if alloc_f`` - allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` - should ensure that both don’t generate points with the same ``sim_id``. - -* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble - worker to be evaluated yet? - -* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` - output given to a worker? - -* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. - -* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` - -* ``gen_time`` [float]: At what time (since the epoch) was this entry (or - collection of entries) put into ``H`` by the manager - -* ``returned`` [bool]: Has this worker completed the evaluation of this unit of - work? - -Output -~~~~~~ -The history array :ref:`H` and -:ref:`persis_info` dictionary are returned to the user -by libEnsemble. If libEnsemble aborts on an exception, these structures are -dumped to the respective files, - -* ``libE_history_at_abort_.npy`` -* ``libE_history_at_abort_.pickle`` - -where ``sim_count`` is the number of points evaluated. - -Other libEnsemble files produced by default are: - -* ``libE_stats.txt``: This contains a one-line summary of all user - calculations. Each calculation summary is sent by workers to the manager and - printed as the run progresses. - -* ``ensemble.log``: This is the logging output from libEnsemble. The default - logging is at INFO level. To gain additional diagnostics logging level can be - set to DEBUG. If this file is not removed, multiple runs will append output. - Messages at or above level MANAGER_WARNING are also copied to stderr to alert - the user promptly. For more info, see :doc:`Logging`. diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index b0edc3968..3b8ac53f3 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,11 +1,6 @@ HPC Systems =========== -libEnsemble has been developed, supported, and tested on systems of highly varying -scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly -parallel scaling capabilities are best exemplified on the resources available -within high-performance machines. - libEnsemble's flexible architecture lends it best to two general modes of worker distributions across allocated compute nodes. The first mode we refer to as *centralized* mode, where the libEnsemble manager and worker processes @@ -35,14 +30,4 @@ differently. On these machines, libEnsemble is run centralized on either a compute-node with the support of Balsam_ or on a frontend server called a MOM (Machine-Oriented Mini-server) node. -Read more about configuring and launching libEnsemble on some HPC systems: - -.. toctree:: - :maxdepth: 2 - :titlesonly: - - bebop - theta - example_scripts - .. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/utilities.rst b/docs/utilities.rst index 87d4734cc..ce3dc5bad 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -4,8 +4,8 @@ Utilities libEnsemble features several modules and tools to assist in writing consistent calling scripts and user functions. -Input consistency ------------------ +libE input consistency +---------------------- Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` dictionary with the ``check_inputs()`` function from the ``utils`` @@ -15,34 +15,3 @@ For example:: from libensemble.libE import check_inputs check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) - -Parameters as command-line arguments ------------------------------------- - -The ``parse_args()`` function can be used to pass common libEnsemble parameters as -command-line arguments. - -In your calling script:: - - from libensemble.tests.regression_tests.common import parse_args - nworkers, is_master, libE_specs, misc_args = parse_args() - -From the shell, for example:: - - $ python calling_script --comms local --nworkers 4 - -Usage: - -.. code-block:: bash - - usage: test_... [-h] [--comms [{local,tcp,ssh,client,mpi}]] - [--nworkers [NWORKERS]] [--workers WORKERS [WORKERS ...]] - [--workerID [WORKERID]] [--server SERVER SERVER SERVER] - [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] - [--worker_python [WORKER_PYTHON]] - [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] - -.. note:: - Since this utility was developed to support automatic testing, it currently - only accepts calling scripts prepended with ``test_`` and isn't necessarily - recommended for production purposes. diff --git a/docs/welcome.rst b/docs/welcome.rst index 47d45369e..7decf8888 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -27,5 +27,5 @@ libEnsemble is a library to coordinate the concurrent evaluation of dynamic ense * New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`Overview`. +* Go in-depth by reading the :doc:`User Guide`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. From c441f3a4573252c960b92ec6c87693e45c59a30c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 13 Nov 2019 16:59:12 -0600 Subject: [PATCH 485/644] Leaving --- README.rst | 187 +++++++++++++++++++++++++++++++++++- docs/deps.rst | 182 ----------------------------------- docs/introduction_latex.rst | 41 +++++++- docs/overview_usecases.rst | 3 - 4 files changed, 224 insertions(+), 189 deletions(-) delete mode 100644 docs/deps.rst diff --git a/README.rst b/README.rst index a8b092d75..d5cdddde1 100644 --- a/README.rst +++ b/README.rst @@ -18,12 +18,13 @@ :alt: Documentation Status | -.. after_badges_rst_tag =========================== Introduction to libEnsemble =========================== +.. after_badges_rst_tag + libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. The library is developed to use massively parallel resources to accelerate the solution of design, decision, and @@ -54,7 +55,187 @@ are portable, resilient and flexible; it also enables automatic detection of the nodes and cores in a system and can split up jobs automatically if resource data isn't supplied. -.. only:: html +.. before_dependencies_rst_tag + +Dependencies +~~~~~~~~~~~~ + +Required dependencies: + +* Python_ 3.5 or above. +* NumPy_ + +For libEnsemble running with the mpi4py parallelism: + +* A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries. +* mpi4py_ v2.0.0 or above + +Optional dependency: + +* Balsam_ + +From v0.2.0, libEnsemble has the option of using the Balsam job manager. This +is required for running libEnsemble on the compute nodes of some supercomputing +platforms (e.g., Cray XC40); platforms that do not support launching jobs from +compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the +launch nodes using multiprocessing. + +The example sim and gen functions and tests require the following dependencies: + +* SciPy_ +* petsc4py_ +* PETSc_ - This can optionally be installed by pip along with petsc4py +* NLopt_ - Installed with `shared libraries enabled`_. + +PETSc and NLopt must be built with shared libraries enabled and present in +``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt +should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also +require SWIG_ to be installed on certain systems. + +Installation +~~~~~~~~~~~~ + +Use pip to install libEnsemble and its dependencies:: + + pip install libensemble + +libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: + + spack install py-libensemble + +The tests and examples can be accessed in the GitHub_ repository. +If necessary, you may install all optional dependencies (listed above) at once with:: + + pip install libensemble[extras] + +A tarball_ of the most recent release is also available. + +Testing +~~~~~~~ + +The provided test suite includes both unit and regression tests and is run +regularly on: + +* `Travis CI`_ + +The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ +packages to be installed and can be run from the libensemble/tests directory of +the source distribution by running:: + + ./run-tests.sh + +To clean the test repositories run:: + + ./run-tests.sh -c + +Further options are available. To see a complete list of options run:: + + ./run-tests.sh -h + +If you have the source distribution, you can download (but not install) the testing +prerequisites and run the tests with:: + + python setup.py test + +in the top-level directory containing the setup script. + +Coverage reports are produced separately for unit tests and regression tests +under the relevant directories. For parallel tests, the union of all processors +is taken. Furthermore, a combined coverage report is created at the top level, +which can be viewed after ``run_tests.sh`` is completed at +``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are +available online at Coveralls_. + +.. note:: + The job_controller tests can be run using the direct-launch or + Balsam job controllers. Balsam integration with libEnsemble is now tested + via ``test_balsam_hworld.py``. + +Basic Usage +~~~~~~~~~~~ + +The examples directory contains example libEnsemble calling scripts, sim +functions, gen functions, alloc functions and job submission scripts. + +The default manager/worker communications mode is MPI. The user script is +launched as:: + + mpiexec -np N python myscript.py + +where ``N`` is the number of processors. This will launch one manager and +``N-1`` workers. + +If running in local mode, which uses Python's multiprocessing module, the +``local`` comms option and the number of workers must be specified. The script +can then be run as a regular python script:: + + python myscript.py + +When specifying these options via command line options, one may use the +``parse_args`` function used in the regression tests, which can be found in +`common.py`_ in the ``libensemble/tests/regression_tests`` directory. + +See the `user guide`_ for more information. + + +Resources +~~~~~~~~~ + +**Support:** + +- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. +- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. +- Join the `libEnsemble mailing list`_ for updates about new releases. + +**Further Information:** + +- Documentation is provided by ReadtheDocs_. +- A visual overview of libEnsemble is given in this poster_. + +**Citation:** + +- Please use the following to cite libEnsemble in a publication: + +.. code-block:: bibtex + + @techreport{libEnsemble, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, + title = {{libEnsemble} Users Manual}, + institution = {Argonne National Laboratory}, + number = {Revision 0.5.2}, + year = {2019}, + url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + } + + +.. after_resources_rst_tag + - .. include:: deps.rst +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py +.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _GitHub: https://github.com/Libensemble/libensemble +.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble +.. _libEnsemble Slack page: https://libensemble.slack.com +.. _mock: https://pypi.org/project/mock +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _MPICH: http://www.mpich.org/ +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 +.. _pytest-cov: https://pypi.org/project/pytest-cov/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ +.. _pytest: https://pypi.org/project/pytest/ +.. _Python: http://www.python.org +.. _ReadtheDocs: http://libensemble.readthedocs.org/ +.. _SciPy: http://www.scipy.org +.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries +.. _Spack: https://spack.readthedocs.io/en/latest +.. _SWIG: http://swig.org/ +.. _tarball: https://github.com/Libensemble/libensemble/releases/latest +.. _Travis CI: https://travis-ci.org/Libensemble/libensemble +.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html diff --git a/docs/deps.rst b/docs/deps.rst deleted file mode 100644 index cac04e759..000000000 --- a/docs/deps.rst +++ /dev/null @@ -1,182 +0,0 @@ -Dependencies -~~~~~~~~~~~~ - -Required dependencies: - -* Python_ 3.5 or above. -* NumPy_ - -For libEnsemble running with the mpi4py parallelism: - -* A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries. -* mpi4py_ v2.0.0 or above - -Optional dependency: - -* Balsam_ - -From v0.2.0, libEnsemble has the option of using the Balsam job manager. This -is required for running libEnsemble on the compute nodes of some supercomputing -platforms (e.g., Cray XC40); platforms that do not support launching jobs from -compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the -launch nodes using multiprocessing. - -The example sim and gen functions and tests require the following dependencies: - -* SciPy_ -* petsc4py_ -* PETSc_ - This can optionally be installed by pip along with petsc4py -* NLopt_ - Installed with `shared libraries enabled`_. - -PETSc and NLopt must be built with shared libraries enabled and present in -``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also -require SWIG_ to be installed on certain systems. - -Installation -~~~~~~~~~~~~ - -Use pip to install libEnsemble and its dependencies:: - - pip install libensemble - -libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: - - spack install py-libensemble - -The tests and examples can be accessed in the GitHub_ repository. -If necessary, you may install all optional dependencies (listed above) at once with:: - - pip install libensemble[extras] - -A tarball_ of the most recent release is also available. - -Testing -~~~~~~~ - -The provided test suite includes both unit and regression tests and is run -regularly on: - -* `Travis CI`_ - -The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ -packages to be installed and can be run from the libensemble/tests directory of -the source distribution by running:: - - ./run-tests.sh - -To clean the test repositories run:: - - ./run-tests.sh -c - -Further options are available. To see a complete list of options run:: - - ./run-tests.sh -h - -If you have the source distribution, you can download (but not install) the testing -prerequisites and run the tests with:: - - python setup.py test - -in the top-level directory containing the setup script. - -Coverage reports are produced separately for unit tests and regression tests -under the relevant directories. For parallel tests, the union of all processors -is taken. Furthermore, a combined coverage report is created at the top level, -which can be viewed after ``run_tests.sh`` is completed at -``libensemble/tests/cov_merge/index.html``. The Travis CI coverage results are -available online at Coveralls_. - -.. note:: - The job_controller tests can be run using the direct-launch or - Balsam job controllers. Balsam integration with libEnsemble is now tested - via ``test_balsam_hworld.py``. - -Basic Usage -~~~~~~~~~~~ - -The examples directory contains example libEnsemble calling scripts, sim -functions, gen functions, alloc functions and job submission scripts. - -The default manager/worker communications mode is MPI. The user script is -launched as:: - - mpiexec -np N python myscript.py - -where ``N`` is the number of processors. This will launch one manager and -``N-1`` workers. - -If running in local mode, which uses Python's multiprocessing module, the -``local`` comms option and the number of workers must be specified. The script -can then be run as a regular python script:: - - python myscript.py - -When specifying these options via command line options, one may use the -``parse_args`` function used in the regression tests, which can be found in -`common.py`_ in the ``libensemble/tests/regression_tests`` directory. - -See the `user guide`_ for more information. - - -Resources -~~~~~~~~~ - -**Support:** - -- The best way to receive support is to email questions to ``libEnsemble@lists.mcs.anl.gov``. -- Communicate (and establish a private channel, if desired) at the `libEnsemble Slack page`_. -- Join the `libEnsemble mailing list`_ for updates about new releases. - -**Further Information:** - -- Documentation is provided by ReadtheDocs_. -- A visual overview of libEnsemble is given in this poster_. - -**Citation:** - -- Please use the following to cite libEnsemble in a publication: - -.. code-block:: bibtex - - @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and - David Bindel and John-Luke Navarro}, - title = {{libEnsemble} Users Manual}, - institution = {Argonne National Laboratory}, - number = {Revision 0.5.2}, - year = {2019}, - url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} - } - - -.. after_resources_rst_tag - - - -.. _Balsam: https://www.alcf.anl.gov/balsam -.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py -.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master -.. _GitHub: https://github.com/Libensemble/libensemble -.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble -.. _libEnsemble Slack page: https://libensemble.slack.com -.. _mock: https://pypi.org/project/mock -.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py -.. _MPICH: http://www.mpich.org/ -.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt -.. _NumPy: http://www.numpy.org -.. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _PETSc: http://www.mcs.anl.gov/petsc -.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 -.. _pytest-cov: https://pypi.org/project/pytest-cov/ -.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ -.. _pytest: https://pypi.org/project/pytest/ -.. _Python: http://www.python.org -.. _ReadtheDocs: http://libensemble.readthedocs.org/ -.. _SciPy: http://www.scipy.org -.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries -.. _Spack: https://spack.readthedocs.io/en/latest -.. _SWIG: http://swig.org/ -.. _tarball: https://github.com/Libensemble/libensemble/releases/latest -.. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index d8a0f0786..18145a38b 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -1,9 +1,20 @@ +========== +INTROLATEX +========== + .. include:: ../README.rst :start-after: after_badges_rst_tag + :end-before: before_dependencies_rst_tag +Overview +======== .. include:: overview_usecases.rst -.. include:: deps.rst +Further +======= +.. include:: ../README.rst + :start-after: before_dependencies_rst_tag + :end-before: after_resources_rst_tag Support for HPC ~~~~~~~~~~~~~~~ @@ -20,3 +31,31 @@ thousands of compute-nodes. platforms/bebop platforms/theta platforms/example_scripts + + +.. _Balsam: https://www.alcf.anl.gov/balsam +.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py +.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _GitHub: https://github.com/Libensemble/libensemble +.. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble +.. _libEnsemble Slack page: https://libensemble.slack.com +.. _mock: https://pypi.org/project/mock +.. _mpi4py: https://bitbucket.org/mpi4py/mpi4py +.. _MPICH: http://www.mpich.org/ +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _NumPy: http://www.numpy.org +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _PETSc: http://www.mcs.anl.gov/petsc +.. _poster: https://figshare.com/articles/LibEnsemble_PETSc_TAO-_Sustaining_a_library_for_dynamic_ensemble-based_computations/7765454 +.. _pytest-cov: https://pypi.org/project/pytest-cov/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ +.. _pytest: https://pypi.org/project/pytest/ +.. _Python: http://www.python.org +.. _ReadtheDocs: http://libensemble.readthedocs.org/ +.. _SciPy: http://www.scipy.org +.. _shared libraries enabled: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries +.. _Spack: https://spack.readthedocs.io/en/latest +.. _SWIG: http://swig.org/ +.. _tarball: https://github.com/Libensemble/libensemble/releases/latest +.. _Travis CI: https://travis-ci.org/Libensemble/libensemble +.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 0945bd3a9..0af016346 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,6 +1,3 @@ -Overview -~~~~~~~~ - libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to various workers. A libEnsemble worker is the smallest indivisible unit to From 1b2875854664409ee6bc3cb6c3e5a8a99be90e73 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 13 Nov 2019 19:31:52 -0600 Subject: [PATCH 486/644] Minor treaks to theta guide --- docs/platforms/theta.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 95da5a6fd..b1689775c 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -6,8 +6,8 @@ Theta_ is a Cray XC40 system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), -and compute nodes. Users on login nodes submit batch runs to the MOM nodes. -MOM nodes execute user batch-scripts to run on the compute nodes. +and compute nodes. Users on login nodes submit batch jobs to the MOM nodes. +MOM nodes execute user batch-scripts to run on the compute nodes via ``aprun``. Configuring Python ------------------ @@ -119,7 +119,7 @@ Interactive Runs Users can run interactively with ``qsub`` by specifying the ``-I`` flag, similarly to the following:: - $ qsub -A [project] -n 128 -q default -t 120 -I + $ qsub -A [project] -n 8 -q debug-cache-quad -t 60 -I This will place the user on a MOM node. Then, to launch MPI jobs to the compute nodes use ``aprun`` where you would use ``mpirun``. @@ -196,7 +196,7 @@ Here is an example Balsam submission script: #COBALT -O libE_test #COBALT -n 128 #COBALT -q default - ##COBALT -A [project] + #COBALT -A [project] # Name of calling script export EXE=calling_script.py From fbdbe7e2f982719b2f9ed8a762208a78cade8d6b Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 13 Nov 2019 19:37:03 -0600 Subject: [PATCH 487/644] Add Summit guide --- docs/platforms/platforms_index.rst | 1 + docs/platforms/summit.rst | 138 +++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 docs/platforms/summit.rst diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 3716ebf47..fccb66075 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -58,6 +58,7 @@ Read more about configuring and launching libEnsemble on some HPC systems: bebop theta + summit example_scripts diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst new file mode 100644 index 000000000..6108b1c73 --- /dev/null +++ b/docs/platforms/summit.rst @@ -0,0 +1,138 @@ +====== +Summit +====== + +Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing Facility. +Each of the approximately 4,600 compute nodes on Summit contains two IBM POWER9 processors and six NVIDIA Volta V100 accelerators. + +Summit features three tiers of nodes: login, launch, and compute nodes. +Users on login nodes submit batch runs to the launch nodes. +Launch nodes execute user batch-scripts to run on the compute nodes via ``jsrun``. + +Configuring Python +------------------ + +Begin by loading the Python 3 Anaconda module:: + + $ module load python + +You can now create your own custom Conda_ environment:: + + conda create --name myenv python=3.7 + +Now activate environment:: + + export PYTHONNOUSERSITE=1 # Make sure get python from conda env + . activate myenv + +If you are installing any packages with extensions, ensure the correct compiler module +is loaded. If using mpi4py_, this must be installed from source, referencing the compiler. +At time of writing, mpi4py must be built with gcc:: + + module load gcc + +With your environment activated:: + + CC=mpicc MPICC=mpicc pip install mpi4py --no-binary mpi4py + + +Installing libEnsemble +---------------------- + +Obtaining libEnsemble is now as simple as ``pip install libensemble``. +Your prompt should be similar to the following line: + +.. code-block:: console + + (my_env) user@login5:~$ pip install libensemble + +.. note:: + If you encounter pip errors, run ``python -m pip install --upgrade pip`` first + + +Job Submission +-------------- + +Summit uses LSF_ for job management and submission. For libEnsemble, the most +important command is ``bsub``, for submitting batch scripts from the login nodes +to execute on the Launch nodes. + +It is recommended to run libEnsemble on the Launch nodes (assuming workers are submitting +MPI jobs) using ``local`` comm mode (multiprocessing). In the future, Balsam may be used +to run libEnsemble on compute nodes. + +Interactive Runs +^^^^^^^^^^^^^^^^ + +Users can run interactively with ``bsub`` by specifying the ``-Is`` flag, similarly +to the following:: + + $ bsub -W 30 -P [project] -nnodes 8 -Is + +This will place the user on a launch node. Then, to launch MPI jobs to the compute +nodes use ``jsrun`` where you would use ``mpirun``. + +.. note:: + You will need to re-activate your conda virtual environment. + +Batch Runs +^^^^^^^^^^ + +Batch scripts specify run-settings using ``#BSUB`` statements. The following +simple example depicts configuring and launching libEnsemble to a launch node with +multiprocessing. This script also assumes the user is using the ``parse_args()`` +convenience function within libEnsemble's ``/regression_tests/common.py``. + +.. code-block:: bash + + #!/bin/bash -x + #BSUB -P + #BSUB -J libe_mproc + #BSUB -W 60 + #BSUB -nnodes 128 + #BSUB -alloc_flags "smt1" + + # --- Prepare Python --- + + # Load conda module and gcc. + module load python + module load gcc + + # Name of Conda environment + export CONDA_ENV_NAME=my_env + + # Activate Conda environment + export PYTHONNOUSERSITE=1 + source activate $CONDA_ENV_NAME + + # --- Prepare libEnsemble --- + + # Name of calling script + export EXE=calling_script.py + + # Communication Method + export COMMS='--comms local' + + # Number of workers. + export NWORKERS='--nworkers 128' + + hash -r # Check no commands hashed (pip/python...) + + # Launch libE + python $EXE $COMMS $NWORKERS > out.txt 2>&1 + +With this saved as ``myscript.sh``, allocating, configuring, and queueing +libEnsemble on Summit becomes:: + + $ bsub script myscript.sh + +Additional Information +---------------------- + +See the OCLF guides_ on for more information about Summit. + +.. _Summit: https://www.olcf.ornl.gov/for-users/system-user-guides/summit/ +.. _LSF: https://www.olcf.ornl.gov/wp-content/uploads/2018/12/summit_workshop_fuson.pdf +.. _guides: https://www.olcf.ornl.gov/for-users/system-user-guides/summit/ +.. _Conda: https://conda.io/en/latest/ +.. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ From 2b7750d8fdc627981d6c7bb1b16874044c3a1ea7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 10:46:39 -0600 Subject: [PATCH 488/644] trying to re-add sections and features removed by recent merge --- README.rst | 4 +-- docs/architecture_html.rst | 7 ++++ docs/history_output.rst | 55 ++++++++++++++++++++++++++++++ docs/index.rst | 8 +++-- docs/introduction.rst | 16 --------- docs/known_issues.rst | 11 +----- docs/platforms/platforms_index.rst | 16 +++++++++ docs/utilities.rst | 36 +++++++++++++++++-- 8 files changed, 119 insertions(+), 34 deletions(-) create mode 100644 docs/architecture_html.rst create mode 100644 docs/history_output.rst diff --git a/README.rst b/README.rst index d5cdddde1..3ddcf59ed 100644 --- a/README.rst +++ b/README.rst @@ -19,12 +19,12 @@ | +.. after_badges_rst_tag + =========================== Introduction to libEnsemble =========================== -.. after_badges_rst_tag - libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. The library is developed to use massively parallel resources to accelerate the solution of design, decision, and diff --git a/docs/architecture_html.rst b/docs/architecture_html.rst new file mode 100644 index 000000000..929173303 --- /dev/null +++ b/docs/architecture_html.rst @@ -0,0 +1,7 @@ + +Understanding libEnsemble +========================= + +.. include:: overview_usecases.rst + +.. include:: history_output.rst diff --git a/docs/history_output.rst b/docs/history_output.rst new file mode 100644 index 000000000..d6146fda4 --- /dev/null +++ b/docs/history_output.rst @@ -0,0 +1,55 @@ +The History Array +~~~~~~~~~~~~~~~~~ +libEnsemble uses a NumPy structured array :ref:`H` to +store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, +``gen_f`` and ``sim_f`` are expected to return output in NumPy structured +arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` +must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output +from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will +include the following fields: + +* ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an + associated ``sim_id``. The generator can assign this, but users must be + careful to ensure points are added in order. For example, ``if alloc_f`` + allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` + should ensure that both don’t generate points with the same ``sim_id``. + +* ``given`` [bool]: Has this ``gen_f`` output been given to a libEnsemble + worker to be evaluated yet? + +* ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` + output given to a worker? + +* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. + +* ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` + +* ``gen_time`` [float]: At what time (since the epoch) was this entry (or + collection of entries) put into ``H`` by the manager + +* ``returned`` [bool]: Has this worker completed the evaluation of this unit of + work? + +Output +~~~~~~ +The history array :ref:`H` and +:ref:`persis_info` dictionary are returned to the user +by libEnsemble. If libEnsemble aborts on an exception, these structures are +dumped to the respective files, + +* ``libE_history_at_abort_.npy`` +* ``libE_history_at_abort_.pickle`` + +where ``sim_count`` is the number of points evaluated. + +Other libEnsemble files produced by default are: + +* ``libE_stats.txt``: This contains a one-line summary of all user + calculations. Each calculation summary is sent by workers to the manager and + printed as the run progresses. + +* ``ensemble.log``: This is the logging output from libEnsemble. The default + logging is at INFO level. To gain additional diagnostics logging level can be + set to DEBUG. If this file is not removed, multiple runs will append output. + Messages at or above level MANAGER_WARNING are also copied to stderr to alert + the user promptly. For more info, see :doc:`Logging`. diff --git a/docs/index.rst b/docs/index.rst index 4e6c889ca..bc747e79d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,14 +9,15 @@ :maxdepth: 2 :caption: User Guide: - Introduction - overview_usecases + Quickstart + architecture_html libe_module data_structures/data_structures user_funcs job_controller/jc_index logging utilities + platforms/platforms_index .. toctree:: :maxdepth: 2 @@ -29,9 +30,10 @@ :caption: Additional Reference: FAQ + known_issues release_notes contributing - known_issues + .. toctree:: :maxdepth: 2 diff --git a/docs/introduction.rst b/docs/introduction.rst index c45471a45..0152356dc 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -1,18 +1,2 @@ .. include:: ../README.rst :start-after: after_badges_rst_tag - -Support for HPC -~~~~~~~~~~~~~~~ - -libEnsemble's flexible architecture was developed with HPC System support and -scaling as a central focus, and can perform calculations on both laptops and -thousands of compute-nodes. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - - platforms/platforms_index - platforms/bebop - platforms/theta - platforms/example_scripts diff --git a/docs/known_issues.rst b/docs/known_issues.rst index b308fce26..caf187767 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -4,7 +4,6 @@ Known Issues The following selection describes known bugs, errors, or other difficulties that may occur when using libEnsemble. -<<<<<<< HEAD * OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches (Either use local mode or Balsam job controller). @@ -13,17 +12,9 @@ may occur when using libEnsemble. with PETSc. * Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. -* TCP mode does not support: +* TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. * libEnsemble may hang on systems with matching probes not enabled on the native fabric, like on Intel's Truescale (TMI) fabric for instance. See the :doc:`FAQ` for more information. - -======= -* OpenMPI does not work with direct MPI job launches in mpi4py comms mode, as it does not support nested MPI launches - (Either use local mode or Balsam job controller). -* Local comms mode (multiprocessing) may fail if MPI is initialized before forking processors. This is thought to be responsible for issues combining with PETSc. -* Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. -* TCP mode does not support: 1) more than one libEnsemble call in a given script or 2) the auto-resources option to the job controller. ->>>>>>> parent of 91fbe4e... rename headings, re-add H and output info, update utilities and known issues, move HPC to own heading under user guide diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 3b8ac53f3..2cfd4bde6 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,6 +1,11 @@ HPC Systems =========== +libEnsemble has been developed, supported, and tested on systems of highly varying +scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly +parallel scaling capabilities are best exemplified on the resources available +within high-performance machines. + libEnsemble's flexible architecture lends it best to two general modes of worker distributions across allocated compute nodes. The first mode we refer to as *centralized* mode, where the libEnsemble manager and worker processes @@ -30,4 +35,15 @@ differently. On these machines, libEnsemble is run centralized on either a compute-node with the support of Balsam_ or on a frontend server called a MOM (Machine-Oriented Mini-server) node. +Read more about configuring and launching libEnsemble on some HPC systems: + +.. toctree:: + :maxdepth: 2 + :titlesonly: + + bebop + theta + example_scripts + + .. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/utilities.rst b/docs/utilities.rst index ce3dc5bad..bd8a44529 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -4,8 +4,8 @@ Utilities libEnsemble features several modules and tools to assist in writing consistent calling scripts and user functions. -libE input consistency ----------------------- +Input consistency +----------------- Users can check the formatting and consistency of ``exit_criteria`` and each ``specs`` dictionary with the ``check_inputs()`` function from the ``utils`` @@ -13,5 +13,35 @@ module. Provide any combination of these data structures as keyword arguments. For example:: from libensemble.libE import check_inputs - check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) + +Parameters as command-line arguments +------------------------------------ + +The ``parse_args()`` function can be used to pass common libEnsemble parameters as +command-line arguments. + +In your calling script:: + + from libensemble.tests.regression_tests.common import parse_args + nworkers, is_master, libE_specs, misc_args = parse_args() + +From the shell, for example:: + + $ python calling_script --comms local --nworkers 4 + +Usage: + +.. code-block:: bash + + usage: test_... [-h] [--comms [{local,tcp,ssh,client,mpi}]] + [--nworkers [NWORKERS]] [--workers WORKERS [WORKERS ...]] + [--workerID [WORKERID]] [--server SERVER SERVER SERVER] + [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] + [--worker_python [WORKER_PYTHON]] + [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] + +.. note:: + Since this utility was developed to support automatic testing, + it currently only accepts calling scripts prepended with ``test_`` + and isn't necessarily recommended for production purposes. From b0083ea46fc9cdb44187899ce41bb32ee9364d26 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 11:54:58 -0600 Subject: [PATCH 489/644] Harmony? --- docs/introduction_latex.rst | 32 +++++++++----------------------- docs/latex_index.rst | 2 +- docs/libe_module.rst | 6 ------ docs/overview_usecases.rst | 6 ++++++ docs/platforms/platforms.rst | 7 ------- docs/programming_libE.rst | 25 +------------------------ 6 files changed, 17 insertions(+), 61 deletions(-) delete mode 100644 docs/libe_module.rst delete mode 100644 docs/platforms/platforms.rst diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index 18145a38b..e4ad2f94b 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -1,7 +1,3 @@ -========== -INTROLATEX -========== - .. include:: ../README.rst :start-after: after_badges_rst_tag :end-before: before_dependencies_rst_tag @@ -9,30 +5,20 @@ INTROLATEX Overview ======== .. include:: overview_usecases.rst + :start-after: begin_overview_rst_tag + :end-before: Example Use Cases + +Example Use Cases +================= +.. include:: overview_usecases.rst + :start-after: begin_usecases_rst_tag -Further -======= +Further Information +=================== .. include:: ../README.rst :start-after: before_dependencies_rst_tag :end-before: after_resources_rst_tag -Support for HPC -~~~~~~~~~~~~~~~ - -libEnsemble's flexible architecture was developed with HPC System support and -scaling as a central focus, and can perform calculations on both laptops and -thousands of compute-nodes. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - - platforms/platforms_index - platforms/bebop - platforms/theta - platforms/example_scripts - - .. _Balsam: https://www.alcf.anl.gov/balsam .. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 771299c70..8ca9be61b 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -7,6 +7,6 @@ Introduction programming_libE - platforms/platforms + platforms/platforms_index dev_info appendix diff --git a/docs/libe_module.rst b/docs/libe_module.rst deleted file mode 100644 index ddbacb128..000000000 --- a/docs/libe_module.rst +++ /dev/null @@ -1,6 +0,0 @@ -Main libEnsemble module -======================= - -.. automodule:: libE - :members: - :no-undoc-members: diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 6b6a04400..24ea1dfc2 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,3 +1,7 @@ +Overview +~~~~~~~~ +.. begin_overview_rst_tag + libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to various workers. A libEnsemble worker is the smallest indivisible unit to @@ -20,6 +24,8 @@ no ``gen_f`` output to give, the worker is told to call ``gen_f``. Example Use Cases ~~~~~~~~~~~~~~~~~ +.. begin_usecases_rst_tag + Below are some expected libEnsemble use cases that we support (or are working to support) and plan to have examples of: diff --git a/docs/platforms/platforms.rst b/docs/platforms/platforms.rst deleted file mode 100644 index bb0295a50..000000000 --- a/docs/platforms/platforms.rst +++ /dev/null @@ -1,7 +0,0 @@ -Platforms ---------- - -.. toctree:: - - theta - bebop diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index 012eea031..175d9bfee 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -5,31 +5,8 @@ Programming with libEnsemble :members: :no-undoc-members: -**libEnsemble Output** - -The history array :ref:`H` and -:ref:`persis_info` dictionary are returned to the user -by libEnsemble. If libEnsemble aborts on an exception, these structures are -dumped to the respective files, - -* ``libE_history_at_abort_.npy`` -* ``libE_history_at_abort_.pickle`` - -where ```` is the number of points evaluated. - -Other libEnsemble files produced by default are: - -* ``libE_stats.txt``: Contains a one-line summary of all user - calculations. Each calculation summary is sent by workers to the manager and - printed as the run progresses. - -* ``ensemble.log``: The logging output from libEnsemble. The default - logging is at INFO level. To gain additional diagnostics logging level can be - set to DEBUG. If this file is not removed, multiple runs will append output. - Messages at or above level MANAGER_WARNING are also copied to stderr to alert - the user promptly. For more info, see :doc:`Logging`. - .. toctree:: + history_output data_structures/data_structures user_funcs job_controller/jc_index From dbaf9fa6c7efc66a1dd3a98ec969f811173beed9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 11:59:43 -0600 Subject: [PATCH 490/644] preparing for merge --- docs/conf.py | 3 ++- docs/index.rst | 6 +----- docs/libe_module.rst | 8 ++------ docs/platforms/platforms_index.rst | 4 ++-- docs/programming_libE.rst | 2 ++ 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 575bb506c..1e10253f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -138,7 +138,8 @@ def __getattr__(cls, name): # further. For a list of options available for each theme, see the # documentation. # -# html_theme_options = {} +html_theme_options = {'navigation_depth': 2, + 'collapse_navigation': False} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/index.rst b/docs/index.rst index bc747e79d..7b82d8177 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,11 +11,7 @@ Quickstart architecture_html - libe_module - data_structures/data_structures - user_funcs - job_controller/jc_index - logging + programming_libE utilities platforms/platforms_index diff --git a/docs/libe_module.rst b/docs/libe_module.rst index ddbacb128..a0c5c9e51 100644 --- a/docs/libe_module.rst +++ b/docs/libe_module.rst @@ -1,6 +1,2 @@ -Main libEnsemble module -======================= - -.. automodule:: libE - :members: - :no-undoc-members: +.. include:: programming_libE + :end-before: end-before-toctree_rst_tag diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 2cfd4bde6..e4d4bda23 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -1,5 +1,5 @@ -HPC Systems -=========== +Running on HPC Systems +====================== libEnsemble has been developed, supported, and tested on systems of highly varying scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index 012eea031..d61415d35 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -29,6 +29,8 @@ Other libEnsemble files produced by default are: Messages at or above level MANAGER_WARNING are also copied to stderr to alert the user promptly. For more info, see :doc:`Logging`. +.. end-before-toctree_rst_tag + .. toctree:: data_structures/data_structures user_funcs From a1275d41a92ffbd35523781e798e7b8a2b7ede8d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:02:12 -0600 Subject: [PATCH 491/644] Flake8 --- examples/tutorials/tutorial_calling.py | 8 ++++---- examples/tutorials/tutorial_calling_mpi.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 38d87866f..6681f20db 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -9,10 +9,10 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'user': { 'lower': np.array([-3]), # random sampling lower bound - 'upper': np.array([3]), # random sampling upper bound - 'gen_batch_size': 5 # number of values gen_f will generate per call - } + 'user': {'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } } sim_specs = {'sim_f': sim_find_sine, # Our simulator function diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 99d5e628a..82805aee6 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -13,10 +13,10 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'user': { 'lower': np.array([-3]), # random sampling lower bound - 'upper': np.array([3]), # random sampling upper bound - 'gen_batch_size': 5 # number of values gen_f will generate per call - } + 'user': {'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } } sim_specs = {'sim_f': sim_find_sine, # Our simulator function From a4b75c6d6766cdee5de5fcbfff12a237c819f993 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:03:09 -0600 Subject: [PATCH 492/644] Whitespace --- README.rst | 4 ---- docs/data_structures/data_structures.rst | 1 - docs/index.rst | 2 -- docs/platforms/platforms_index.rst | 1 - 4 files changed, 8 deletions(-) diff --git a/README.rst b/README.rst index 3ddcf59ed..9d16ae55d 100644 --- a/README.rst +++ b/README.rst @@ -177,7 +177,6 @@ When specifying these options via command line options, one may use the See the `user guide`_ for more information. - Resources ~~~~~~~~~ @@ -208,11 +207,8 @@ Resources url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } - .. after_resources_rst_tag - - .. _Balsam: https://www.alcf.anl.gov/balsam .. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index 1ab157e17..fff56d816 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -20,7 +20,6 @@ and the :ref:`work` dictionary produced by the allocation function. - .. toctree:: :maxdepth: 3 :caption: libEnsemble Data Structures: diff --git a/docs/index.rst b/docs/index.rst index 7b82d8177..755b15c98 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,6 @@ release_notes contributing - .. toctree:: :maxdepth: 2 :caption: Developer Guide: @@ -38,7 +37,6 @@ dev_guide/release_management/release_index dev_guide/dev_API/developer_API - Indices and tables ================== diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index e4d4bda23..fb1b4fbac 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -45,5 +45,4 @@ Read more about configuring and launching libEnsemble on some HPC systems: theta example_scripts - .. _Balsam: https://balsam.readthedocs.io/en/latest/ From af24e4776895ee0779866c07497e05f6bad7e42f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:03:22 -0600 Subject: [PATCH 493/644] Whitespace --- docs/architecture_html.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/architecture_html.rst b/docs/architecture_html.rst index 929173303..4809adeb7 100644 --- a/docs/architecture_html.rst +++ b/docs/architecture_html.rst @@ -1,4 +1,3 @@ - Understanding libEnsemble ========================= From 9302d21af6a052e0c71026d7ed0b5b8a2c9a4339 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:04:37 -0600 Subject: [PATCH 494/644] Whitespace --- docs/introduction_latex.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index e4ad2f94b..8a85f4686 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -6,7 +6,7 @@ Overview ======== .. include:: overview_usecases.rst :start-after: begin_overview_rst_tag - :end-before: Example Use Cases + :end-before: Example Use Cases Example Use Cases ================= From 8d285146bb7ca194071e973fec062fc82c33789c Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 12:05:44 -0600 Subject: [PATCH 495/644] remove redundant info --- docs/architecture_html.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/architecture_html.rst b/docs/architecture_html.rst index 929173303..1192f7463 100644 --- a/docs/architecture_html.rst +++ b/docs/architecture_html.rst @@ -3,5 +3,3 @@ Understanding libEnsemble ========================= .. include:: overview_usecases.rst - -.. include:: history_output.rst From 16e10974ff7a9d4af85a48b2627e0560e77e6805 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:07:22 -0600 Subject: [PATCH 496/644] Adding sphinxcontrib_bibtex as a dependency (for now) --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1e10253f7..39ec59570 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,8 +58,8 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -# extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +extensions = ['sphinxcontrib.bibtex','sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } #breathe_default_project = "libEnsemble" From cc80ac0c0080d38bff3f48351dde9e2960073d88 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 12:09:52 -0600 Subject: [PATCH 497/644] overview to toctree --- docs/architecture_html.rst | 4 ---- docs/index.rst | 2 +- docs/overview_usecases.rst | 3 +++ 3 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 docs/architecture_html.rst diff --git a/docs/architecture_html.rst b/docs/architecture_html.rst deleted file mode 100644 index 44719846d..000000000 --- a/docs/architecture_html.rst +++ /dev/null @@ -1,4 +0,0 @@ -Understanding libEnsemble -========================= - -.. include:: overview_usecases.rst diff --git a/docs/index.rst b/docs/index.rst index 755b15c98..48267e565 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,7 +10,7 @@ :caption: User Guide: Quickstart - architecture_html + overview_usecases programming_libE utilities platforms/platforms_index diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 24ea1dfc2..6fdc1e69e 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -1,3 +1,6 @@ +Understanding libEnsemble +========================= + Overview ~~~~~~~~ .. begin_overview_rst_tag From 1a025edc863695dc05c82c11ca438643a1d0f5e8 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 12:20:54 -0600 Subject: [PATCH 498/644] Adding examples to tutorials(?) --- docs/appendix.rst | 1 + docs/index.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/appendix.rst b/docs/appendix.rst index a99b081ef..c642ae13e 100644 --- a/docs/appendix.rst +++ b/docs/appendix.rst @@ -4,6 +4,7 @@ Appendices .. toctree:: tutorials/tutorials FAQ + known_issues examples/examples_index release_notes bibliography diff --git a/docs/index.rst b/docs/index.rst index 48267e565..9d784d684 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,7 @@ :caption: Tutorials: tutorials/local_sine_tutorial + examples/examples_index .. toctree:: :maxdepth: 2 From 28ee0fd926c0eb2684c29734df405640ef058ae9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 12:26:03 -0600 Subject: [PATCH 499/644] dont re-add already-gone content --- docs/platforms/bebop.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index c72697025..75e1fb583 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -5,16 +5,6 @@ Bebop Bebop_ is a Cray CS400 cluster available within LCRC at Argonne National Laboratory, featuring both Intel Broadwell and Knights Landing compute nodes. -Prerequisites -------------- - -An Argonne LCRC_ account is required to access Bebop. Interested users will need -to apply for and be granted an account before continuing. To submit jobs to Bebop, -users can charge hours to a project or their personal allocation (default). - -Bebop consists primarily of login and compute nodes. Users start on the login -nodes, and schedule work for execution on the compute nodes. - Configuring Python ------------------ From 24e0716ef8a135bc037d3d4e2841d28cdd112328 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 12:39:22 -0600 Subject: [PATCH 500/644] ditto --- docs/platforms/theta.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 2c6cdfa33..95da5a6fd 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -5,21 +5,6 @@ Theta Theta_ is a Cray XC40 system based on the second-generation Intel Xeon Phi processor, available within ALCF_ at Argonne National Laboratory. -Prerequisites -------------- - -An Argonne ALCF account is required to access Theta. Interested users will need -to apply for and be granted an account before continuing. To submit jobs to Theta, -users must charge their jobs to a project (specified while requesting an -account). - -Theta contains numerous node-types, but libEnsemble users will mostly interact -with login, Machine Oriented Mini-server (MOM) and compute nodes. MOM nodes execute -user batch-scripts to run on the compute nodes. - -libEnsemble functions need to incorporate the libEnsemble -:ref:`job controller` to perform calculations on Theta. - Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), and compute nodes. Users on login nodes submit batch runs to the MOM nodes. MOM nodes execute user batch-scripts to run on the compute nodes. From 3c83ad22e8ca56564d6cdbb484e5b4e40b30d1fc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 13:30:35 -0600 Subject: [PATCH 501/644] Debugging readthedocs --- .readthedocs.yml | 16 +++++----------- docs/requirements.txt | 1 - 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index f947fcee2..a3d25fee2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,11 @@ +version: 2 + conda: file: conda/environment.yml - environment: environment.yml + +sphinx: + configuration: docs/conf.py build: image: latest -python: - version: 3.6 - pip_install: true - setup_py_install: true - install: - - method: pip - - sphinxcontrib.bibtex - - sphinxcontrib-bibtex - - requirements: docs/requirements.txt - system_packages: true \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index ce15bceee..c3e8a7601 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1 @@ sphinxcontrib.bibtex -sphinxcontrib-bibtex \ No newline at end of file From 24245edb1e6eb0df7c1eae3a781b86f947e6b205 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 13:31:48 -0600 Subject: [PATCH 502/644] Debugging readthedocs --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index a3d25fee2..f84aa8c26 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,7 @@ version: 2 conda: - file: conda/environment.yml + environment: conda/environment.yml sphinx: configuration: docs/conf.py From 4c1e46231daaf8053c006c525159fe9f49588aed Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 13:35:58 -0600 Subject: [PATCH 503/644] Debugging readthedocs --- .readthedocs.yml | 10 ++++++---- docs/requirements.txt | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index f84aa8c26..36218d02f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,11 +1,13 @@ version: 2 -conda: - environment: conda/environment.yml - sphinx: - configuration: docs/conf.py + configuration: docs/conf.py build: image: latest +python: + version: 3.6 + install: + - requirements: docs/requirements.txt + system_packages: true diff --git a/docs/requirements.txt b/docs/requirements.txt index c3e8a7601..b6f35cbcb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ -sphinxcontrib.bibtex +sphinxcontrib-bibtex +libensemble From e347436452567d2865fe4228b41932a2cd91b372 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 14:02:56 -0600 Subject: [PATCH 504/644] modify utilities docs for parse_args move --- docs/utilities.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/utilities.rst b/docs/utilities.rst index bd8a44529..53f2ae749 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -12,7 +12,7 @@ Users can check the formatting and consistency of ``exit_criteria`` and each module. Provide any combination of these data structures as keyword arguments. For example:: - from libensemble.libE import check_inputs + from libensemble.utils import check_inputs check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) Parameters as command-line arguments @@ -23,7 +23,7 @@ command-line arguments. In your calling script:: - from libensemble.tests.regression_tests.common import parse_args + from libensemble.utils import parse_args nworkers, is_master, libE_specs, misc_args = parse_args() From the shell, for example:: @@ -40,8 +40,3 @@ Usage: [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] [--worker_python [WORKER_PYTHON]] [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] - -.. note:: - Since this utility was developed to support automatic testing, - it currently only accepts calling scripts prepended with ``test_`` - and isn't necessarily recommended for production purposes. From 067ae5701a647ac8d4a3aa140a3d6dc07fc34cfe Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 14:23:05 -0600 Subject: [PATCH 505/644] Not sure what this does, but maybe it helps --- libensemble/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libensemble/utils.py b/libensemble/utils.py index 435ef1cd1..a9bced316 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -4,6 +4,8 @@ """ +__all__ = ['check_inputs'] + import traceback import logging import numpy as np From 469be8e04418b8ea5da59840da1194386575b41c Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 14:39:14 -0600 Subject: [PATCH 506/644] Move many common utilities from regression_tests/common.py to libensemble/utils.py --- libensemble/tests/regression_tests/common.py | 131 ----------------- .../script_test_balsam_hworld.py | 2 +- .../regression_tests/test_1d_sampling.py | 2 +- .../test_1d_uniform_sampling_with_comm_dup.py | 2 +- ...mp_camel_active_persistent_worker_abort.py | 2 +- .../test_6-hump_camel_aposmm_LD_MMA.py | 2 +- .../test_6-hump_camel_elapsed_time_abort.py | 2 +- .../test_6-hump_camel_persistent_aposmm_1.py | 2 +- .../test_6-hump_camel_persistent_aposmm_2.py | 2 +- .../test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../test_6-hump_camel_persistent_aposmm_4.py | 2 +- ...-hump_camel_persistent_uniform_sampling.py | 2 +- .../test_6-hump_camel_pregenerated_sample.py | 2 +- .../test_6-hump_camel_uniform_sampling.py | 2 +- ..._sampling_with_persistent_localopt_gens.py | 2 +- ...mel_with_different_nodes_uniform_sample.py | 2 +- ...test_branin_aposmm_nlopt_and_then_scipy.py | 2 +- .../regression_tests/test_calc_exception.py | 2 +- ...t_chwirut_aposmm_one_residual_at_a_time.py | 2 +- .../regression_tests/test_chwirut_pounders.py | 2 +- .../test_chwirut_pounders_persistent.py | 2 +- .../test_chwirut_pounders_splitcomm.py | 2 +- .../test_chwirut_pounders_subcomm.py | 2 +- ...uniform_sampling_one_residual_at_a_time.py | 2 +- .../tests/regression_tests/test_comms.py | 2 +- .../tests/regression_tests/test_fast_alloc.py | 4 +- .../test_inverse_bayes_example.py | 2 +- .../test_jobcontroller_hworld.py | 2 +- .../tests/regression_tests/test_mpi_comms.py | 2 +- .../regression_tests/test_nan_func_aposmm.py | 2 +- .../test_worker_exceptions.py | 2 +- libensemble/utils.py | 136 ++++++++++++++++++ 32 files changed, 167 insertions(+), 162 deletions(-) diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index 672fd362b..c5f490af9 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -5,40 +5,9 @@ import sys import os import os.path -import argparse import numpy as np import pickle -parser = argparse.ArgumentParser(prog='test_...') -parser.add_argument('--comms', type=str, nargs='?', - choices=['local', 'tcp', 'ssh', 'client', 'mpi'], - default='mpi', help='Type of communicator') -parser.add_argument('--nworkers', type=int, nargs='?', - help='Number of local forked processes') -parser.add_argument('--workers', type=str, nargs='+', - help='List of worker nodes') -parser.add_argument('--workerID', type=int, nargs='?', help='Client worker ID') -parser.add_argument('--server', type=str, nargs=3, - help='Triple of (ip, port, authkey) used to reach manager') -parser.add_argument('--pwd', type=str, nargs='?', - help='Working directory to be used') -parser.add_argument('--worker_pwd', type=str, nargs='?', - help='Working directory on remote client') -parser.add_argument('--worker_python', type=str, nargs='?', - default=sys.executable, - help='Python version on remote client') -parser.add_argument('--tester_args', type=str, nargs='*', - help='Additional arguments for use by specific testers') - - -def mpi_parse_args(args): - "Parse arguments for MPI comms." - from mpi4py import MPI - nworkers = MPI.COMM_WORLD.Get_size()-1 - is_master = MPI.COMM_WORLD.Get_rank() == 0 - libE_specs = {'comm': MPI.COMM_WORLD, 'comms': 'mpi'} - return nworkers, is_master, libE_specs, args.tester_args - def mpi_comm_excl(exc=[0], comm=None): "Exlude ranks from a communicator for MPI comms." @@ -62,106 +31,6 @@ def mpi_comm_split(num_parts, comm=None): return sub_comm, sub_comm_number -def local_parse_args(args): - "Parse arguments for forked processes using multiprocessing." - nworkers = args.nworkers or 4 - libE_specs = {'nprocesses': nworkers, 'comms': 'local'} - return nworkers, True, libE_specs, args.tester_args - - -def tcp_parse_args(args): - "Parse arguments for local TCP connections" - nworkers = args.nworkers or 4 - cmd = [ - sys.executable, sys.argv[0], "--comms", "client", "--server", - "{manager_ip}", "{manager_port}", "{authkey}", "--workerID", - "{workerID}", "--nworkers", - str(nworkers)] - libE_specs = {'nprocesses': nworkers, 'worker_cmd': cmd, 'comms': 'tcp'} - return nworkers, True, libE_specs, args.tester_args - - -def ssh_parse_args(args): - "Parse arguments for SSH with reverse tunnel." - nworkers = len(args.workers) - worker_pwd = args.worker_pwd or os.getcwd() - script_dir, script_name = os.path.split(sys.argv[0]) - worker_script_name = os.path.join(worker_pwd, script_name) - ssh = [ - "ssh", "-R", "{tunnel_port}:localhost:{manager_port}", "{worker_ip}"] - cmd = [ - args.worker_python, worker_script_name, "--comms", "client", - "--server", "localhost", "{tunnel_port}", "{authkey}", "--workerID", - "{workerID}", "--nworkers", - str(nworkers)] - cmd = " ".join(cmd) - cmd = "( cd {} ; {} )".format(worker_pwd, cmd) - ssh.append(cmd) - libE_specs = {'workers': args.workers, - 'worker_cmd': ssh, - 'ip': 'localhost', - 'comms': 'tcp'} - return nworkers, True, libE_specs, args.tester_args - - -def client_parse_args(args): - "Parse arguments for a TCP client." - nworkers = args.nworkers or 4 - ip, port, authkey = args.server - libE_specs = {'ip': ip, - 'port': int(port), - 'authkey': authkey.encode('utf-8'), - 'workerID': args.workerID, - 'nprocesses': nworkers, - 'comms': 'tcp'} - return nworkers, False, libE_specs, args.tester_args - - -def parse_args(): - "Unified parsing interface for regression test arguments" - args = parser.parse_args(sys.argv[1:]) - front_ends = { - 'mpi': mpi_parse_args, - 'local': local_parse_args, - 'tcp': tcp_parse_args, - 'ssh': ssh_parse_args, - 'client': client_parse_args} - if args.pwd is not None: - os.chdir(args.pwd) - return front_ends[args.comms or 'mpi'](args) - - -def save_libE_output(H, persis_info, calling_file, nworkers): - script_name = os.path.splitext(os.path.basename(calling_file))[0] - short_name = script_name.split("test_", 1).pop() - filename = short_name + '_results_History_length=' + str(len(H)) \ - + '_evals=' + str(sum(H['returned'])) \ - + '_ranks=' + str(nworkers) - - print("\n\n\nRun completed.\nSaving results to file: "+filename) - np.save(filename, H) - - with open(filename + ".pickle", "wb") as f: - pickle.dump(persis_info, f) - - -def per_worker_stream(persis_info, nworkers): - for i in range(nworkers): - if i in persis_info: - persis_info[i].update({ - 'rand_stream': np.random.RandomState(i), - 'worker_num': i}) - else: - persis_info[i] = { - 'rand_stream': np.random.RandomState(i), - 'worker_num': i} - return persis_info - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - def build_simfunc(): import subprocess diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 6db317c1d..8f0e671b0 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -11,7 +11,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import per_worker_stream +from libensemble.util import per_worker_stream mpi4py.rc.recv_mprobe = False # Disable matching probes diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 626d08ed6..657c140bb 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() libE_specs['save_every_k_gens'] = 300 diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index 8e25f531c..f48687131 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -23,7 +23,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index 3b0252e99..51601673e 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 106dcd9e5..88c15d6a0 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_to_aposmm import give_sim_work_first as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_1 as persis_info, aposmm_gen_out as gen_out, six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index 46e968542..28a965189 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream, eprint +from libensemble.util import parse_args, save_libE_output, per_worker_stream, eprint nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index b8e7a27ec..86e8a2f59 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index f3c554ca9..365e98c12 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 87b46b2e4..912daf34b 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 6fd1b0d5b..549242cef 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py index 769d0e10a..3e29a043e 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py index a1ab4f107..929aa147a 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.alloc_funcs.give_pregenerated_work import give_pregenerated_sim_work as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output +from libensemble.util import parse_args, save_libE_output nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index d6263f0dc..c99edd32d 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index 0cfe45ca4..795d1f286 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index de7890af4..0bab4daf2 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel_with_different_ranks_and_nodes as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index 0de243fe7..fd7cec063 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.branin.branin_obj import call_branin as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out, branin_vals_and_minima as M -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index 58247bb9c..fb2dfffe4 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -18,7 +18,7 @@ from libensemble.libE import libE from libensemble.libE_manager import ManagerException from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import parse_args, per_worker_stream +from libensemble.util import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py index 3df028cb0..44e5c71e1 100644 --- a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_3 as persis_info, aposmm_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders.py b/libensemble/tests/regression_tests/test_chwirut_pounders.py index d438ea63e..d724874da 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 2cfe788e6..943319409 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.gen_funcs.sampling import lhs_sample from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 6abcf01ea..2db4612a2 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream, mpi_comm_split +from libensemble.util import parse_args, save_libE_output, per_worker_stream, mpi_comm_split num_comms = 2 # Must have atleast num_comms*2 processors nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 33f4b5d69..585faca30 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream, mpi_comm_excl +from libensemble.util import parse_args, save_libE_output, per_worker_stream, mpi_comm_excl nworkers, is_master, libE_specs, _ = parse_args() libE_specs['comm'], mpi_comm_null = mpi_comm_excl() diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 054a60224..d973557f8 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -27,7 +27,7 @@ from libensemble.gen_funcs.sampling import uniform_random_sample_obj_components as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.support import persis_info_3 as persis_info -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() if libE_specs['comms'] == 'tcp': diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index 619ae2d65..7187fb320 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.comms_testing import float_x1000 as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream from libensemble.mpi_controller import MPIJobController # Only used to get workerID in float_x1000 jobctrl = MPIJobController(auto_resources=False) diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 012039a29..0120a64c7 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -18,10 +18,10 @@ # Import libEnsemble items for this test from libensemble.libE import libE -from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f +from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_fß from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as alloc_f -from libensemble.tests.regression_tests.common import parse_args, per_worker_stream +from libensemble.util import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index a117a83f9..bcd5278cb 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -24,7 +24,7 @@ from libensemble.sim_funcs.inverse_bayes import likelihood_calculator as sim_f from libensemble.gen_funcs.persistent_inverse_bayes import persistent_updater_after_likelihood as gen_f from libensemble.alloc_funcs.inverse_bayes_allocf import only_persistent_gens_for_inverse_bayes as alloc_f -from libensemble.tests.regression_tests.common import parse_args, per_worker_stream +from libensemble.util import parse_args, per_worker_stream # Parse args for test code nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 73abc3748..e5f916344 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -21,7 +21,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.tests.regression_tests.common import build_simfunc, parse_args, per_worker_stream +from libensemble.util import build_simfunc, parse_args, per_worker_stream # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local tcp diff --git a/libensemble/tests/regression_tests/test_mpi_comms.py b/libensemble/tests/regression_tests/test_mpi_comms.py index 11ae9964c..fc2c3cda2 100644 --- a/libensemble/tests/regression_tests/test_mpi_comms.py +++ b/libensemble/tests/regression_tests/test_mpi_comms.py @@ -10,7 +10,7 @@ import sys from mpi4py import MPI from libensemble.comms.mpi import MPIComm, Timeout -from libensemble.tests.regression_tests.common import parse_args +from libensemble.util import parse_args # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi diff --git a/libensemble/tests/regression_tests/test_nan_func_aposmm.py b/libensemble/tests/regression_tests/test_nan_func_aposmm.py index d78e2424a..2cfff3de6 100644 --- a/libensemble/tests/regression_tests/test_nan_func_aposmm.py +++ b/libensemble/tests/regression_tests/test_nan_func_aposmm.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from support import nan_func as sim_f, aposmm_gen_out as gen_out from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.util import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() n = 2 diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 982ce2781..96e51ae79 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -19,7 +19,7 @@ from libensemble.tests.regression_tests.support import nan_func as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE_manager import ManagerException -from libensemble.tests.regression_tests.common import parse_args, per_worker_stream +from libensemble.util import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() n = 2 diff --git a/libensemble/utils.py b/libensemble/utils.py index 435ef1cd1..396366940 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -7,6 +7,7 @@ import traceback import logging import numpy as np +import argparse import pickle # Only used when saving output on error logger = logging.getLogger(__name__) @@ -234,3 +235,138 @@ def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=No assert sim_specs is not None and alloc_specs is not None and gen_specs is not None, \ "Can't check H0 without sim_specs, alloc_specs, gen_specs" check_H(H0, sim_specs, alloc_specs, gen_specs) + +# ==================== Command-line argument parsing =========================== + +parser = argparse.ArgumentParser(prog='test_...') +parser.add_argument('--comms', type=str, nargs='?', + choices=['local', 'tcp', 'ssh', 'client', 'mpi'], + default='mpi', help='Type of communicator') +parser.add_argument('--nworkers', type=int, nargs='?', + help='Number of local forked processes') +parser.add_argument('--workers', type=str, nargs='+', + help='List of worker nodes') +parser.add_argument('--workerID', type=int, nargs='?', help='Client worker ID') +parser.add_argument('--server', type=str, nargs=3, + help='Triple of (ip, port, authkey) used to reach manager') +parser.add_argument('--pwd', type=str, nargs='?', + help='Working directory to be used') +parser.add_argument('--worker_pwd', type=str, nargs='?', + help='Working directory on remote client') +parser.add_argument('--worker_python', type=str, nargs='?', + default=sys.executable, + help='Python version on remote client') +parser.add_argument('--tester_args', type=str, nargs='*', + help='Additional arguments for use by specific testers') + + +def mpi_parse_args(args): + "Parse arguments for MPI comms." + from mpi4py import MPI + nworkers = MPI.COMM_WORLD.Get_size()-1 + is_master = MPI.COMM_WORLD.Get_rank() == 0 + libE_specs = {'comm': MPI.COMM_WORLD, 'comms': 'mpi'} + return nworkers, is_master, libE_specs, args.tester_args + + +def local_parse_args(args): + "Parse arguments for forked processes using multiprocessing." + nworkers = args.nworkers or 4 + libE_specs = {'nprocesses': nworkers, 'comms': 'local'} + return nworkers, True, libE_specs, args.tester_args + + +def tcp_parse_args(args): + "Parse arguments for local TCP connections" + nworkers = args.nworkers or 4 + cmd = [ + sys.executable, sys.argv[0], "--comms", "client", "--server", + "{manager_ip}", "{manager_port}", "{authkey}", "--workerID", + "{workerID}", "--nworkers", + str(nworkers)] + libE_specs = {'nprocesses': nworkers, 'worker_cmd': cmd, 'comms': 'tcp'} + return nworkers, True, libE_specs, args.tester_args + + +def ssh_parse_args(args): + "Parse arguments for SSH with reverse tunnel." + nworkers = len(args.workers) + worker_pwd = args.worker_pwd or os.getcwd() + script_dir, script_name = os.path.split(sys.argv[0]) + worker_script_name = os.path.join(worker_pwd, script_name) + ssh = [ + "ssh", "-R", "{tunnel_port}:localhost:{manager_port}", "{worker_ip}"] + cmd = [ + args.worker_python, worker_script_name, "--comms", "client", + "--server", "localhost", "{tunnel_port}", "{authkey}", "--workerID", + "{workerID}", "--nworkers", + str(nworkers)] + cmd = " ".join(cmd) + cmd = "( cd {} ; {} )".format(worker_pwd, cmd) + ssh.append(cmd) + libE_specs = {'workers': args.workers, + 'worker_cmd': ssh, + 'ip': 'localhost', + 'comms': 'tcp'} + return nworkers, True, libE_specs, args.tester_args + + +def client_parse_args(args): + "Parse arguments for a TCP client." + nworkers = args.nworkers or 4 + ip, port, authkey = args.server + libE_specs = {'ip': ip, + 'port': int(port), + 'authkey': authkey.encode('utf-8'), + 'workerID': args.workerID, + 'nprocesses': nworkers, + 'comms': 'tcp'} + return nworkers, False, libE_specs, args.tester_args + + +def parse_args(): + "Unified parsing interface for regression test arguments" + args = parser.parse_args(sys.argv[1:]) + front_ends = { + 'mpi': mpi_parse_args, + 'local': local_parse_args, + 'tcp': tcp_parse_args, + 'ssh': ssh_parse_args, + 'client': client_parse_args} + if args.pwd is not None: + os.chdir(args.pwd) + return front_ends[args.comms or 'mpi'](args) + +# =================== save libE output to pickle and np ======================== + +def save_libE_output(H, persis_info, calling_file, nworkers): + script_name = os.path.splitext(os.path.basename(calling_file))[0] + short_name = script_name.split("test_", 1).pop() + filename = short_name + '_results_History_length=' + str(len(H)) \ + + '_evals=' + str(sum(H['returned'])) \ + + '_ranks=' + str(nworkers) + + print("\n\n\nRun completed.\nSaving results to file: "+filename) + np.save(filename, H) + + with open(filename + ".pickle", "wb") as f: + pickle.dump(persis_info, f) + +# ===================== per-worker numpy random-streams ======================== + +def per_worker_stream(persis_info, nworkers): + for i in range(nworkers): + if i in persis_info: + persis_info[i].update({ + 'rand_stream': np.random.RandomState(i), + 'worker_num': i}) + else: + persis_info[i] = { + 'rand_stream': np.random.RandomState(i), + 'worker_num': i} + return persis_info + + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) From 8bbd9847d8d77cc506b4cd0c3f3623261938ac0d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 14:49:51 -0600 Subject: [PATCH 507/644] Debugging readthedocs --- docs/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b6f35cbcb..ef36addc6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1 @@ sphinxcontrib-bibtex -libensemble From 59f0c86abd2b8a17130c51b7b78973e7a8a7fb63 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 14:51:53 -0600 Subject: [PATCH 508/644] Debugging readthedocs --- .readthedocs.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 36218d02f..81da37806 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,13 +1,7 @@ version: 2 sphinx: - configuration: docs/conf.py + configuration: docs/conf.py build: image: latest - -python: - version: 3.6 - install: - - requirements: docs/requirements.txt - system_packages: true From d5602fbcd064c32b009a40bdaf4415144ca61f18 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 14:53:18 -0600 Subject: [PATCH 509/644] Debugging readthedocs --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 39ec59570..693833f64 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ def __getattr__(cls, name): # If your documentation needs a minimal Sphinx version, state it here. # -needs_sphinx = '2.0' +# needs_sphinx = '2.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom From 8f716a73349fa218b16a33640e740d0834a43266 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 14:55:02 -0600 Subject: [PATCH 510/644] Debugging readthedocs --- .readthedocs.yml | 3 +++ conda/environment.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 81da37806..1e71c897a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,5 +1,8 @@ version: 2 +conda: + environment: conda/environment.yml + sphinx: configuration: docs/conf.py diff --git a/conda/environment.yml b/conda/environment.yml index 12c3875f2..8d2d32006 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -3,4 +3,4 @@ dependencies: - pip - pip: - sphinxcontrib.bibtex - - sphinxcontrib-bibtex \ No newline at end of file + - sphinxcontrib-bibtex From 3c89be66958c92d268eea4904916327f8a4d9b2d Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 15:06:08 -0600 Subject: [PATCH 511/644] bump navigation depth --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 693833f64..b45b78d79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -138,7 +138,7 @@ def __getattr__(cls, name): # further. For a list of options available for each theme, see the # documentation. # -html_theme_options = {'navigation_depth': 2, +html_theme_options = {'navigation_depth': 3, 'collapse_navigation': False} # Add any paths that contain custom static files (such as style sheets) here, From f1030261cc2b5813aea749653a678e572628c68c Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 15:06:46 -0600 Subject: [PATCH 512/644] Try going back to conda with dep --- .readthedocs.yml | 9 +++------ conda/environment.yml | 5 +---- docs/requirements.txt | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 36218d02f..45ad794c7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,13 +1,10 @@ version: 2 +conda: + environment: conda/environment.yml + sphinx: configuration: docs/conf.py build: image: latest - -python: - version: 3.6 - install: - - requirements: docs/requirements.txt - system_packages: true diff --git a/conda/environment.yml b/conda/environment.yml index 12c3875f2..d28afbb1b 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,6 +1,3 @@ dependencies: - python>=3.6 - - pip - - pip: - - sphinxcontrib.bibtex - - sphinxcontrib-bibtex \ No newline at end of file + - sphinxcontrib-bibtex \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index b6f35cbcb..063fc0e58 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinxcontrib-bibtex -libensemble + From ca7f75b68ac4631ff127574d8bf86c95c3320b36 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 15:28:25 -0600 Subject: [PATCH 513/644] Explicit install line in environment.yaml --- conda/environment.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda/environment.yml b/conda/environment.yml index d28afbb1b..2a28ef215 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,3 +1,6 @@ +before_install: + conda install sphinxcontrib-bibtex + dependencies: - python>=3.6 - - sphinxcontrib-bibtex \ No newline at end of file + \ No newline at end of file From 9782dbc0d06ec1d059b421089437242d1c7137d3 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 15:50:14 -0600 Subject: [PATCH 514/644] Add channels to environment.yml --- conda/environment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda/environment.yml b/conda/environment.yml index 2a28ef215..01fd94b91 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,6 +1,7 @@ -before_install: - conda install sphinxcontrib-bibtex +channels: + - conda-forge + - defaults dependencies: + - sphinxcontrib-bibtex - python>=3.6 - \ No newline at end of file From 361dde868540e8a421dffeebad0b67829d4e6ad5 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 15:55:14 -0600 Subject: [PATCH 515/644] flake8 --- examples/tutorials/tutorial_calling.py | 8 ++++---- examples/tutorials/tutorial_calling_mpi.py | 8 ++++---- libensemble/tests/regression_tests/common.py | 3 --- libensemble/tests/regression_tests/test_fast_alloc.py | 2 +- libensemble/utils.py | 7 ++++++- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 38d87866f..6681f20db 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -9,10 +9,10 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'user': { 'lower': np.array([-3]), # random sampling lower bound - 'upper': np.array([3]), # random sampling upper bound - 'gen_batch_size': 5 # number of values gen_f will generate per call - } + 'user': {'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } } sim_specs = {'sim_f': sim_find_sine, # Our simulator function diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 99d5e628a..82805aee6 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -13,10 +13,10 @@ gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). - 'user': { 'lower': np.array([-3]), # random sampling lower bound - 'upper': np.array([3]), # random sampling upper bound - 'gen_batch_size': 5 # number of values gen_f will generate per call - } + 'user': {'lower': np.array([-3]), # random sampling lower bound + 'upper': np.array([3]), # random sampling upper bound + 'gen_batch_size': 5 # number of values gen_f will generate per call + } } sim_specs = {'sim_f': sim_find_sine, # Our simulator function diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index c5f490af9..a1883c4b7 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -2,11 +2,8 @@ Common plumbing for regression tests """ -import sys import os import os.path -import numpy as np -import pickle def mpi_comm_excl(exc=[0], comm=None): diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 0120a64c7..8764eaee1 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -18,7 +18,7 @@ # Import libEnsemble items for this test from libensemble.libE import libE -from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_fß +from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as alloc_f from libensemble.util import parse_args, per_worker_stream diff --git a/libensemble/utils.py b/libensemble/utils.py index 396366940..30e9b5b2f 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -5,6 +5,8 @@ """ import traceback +import os +import sys import logging import numpy as np import argparse @@ -238,7 +240,9 @@ def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=No # ==================== Command-line argument parsing =========================== + parser = argparse.ArgumentParser(prog='test_...') + parser.add_argument('--comms', type=str, nargs='?', choices=['local', 'tcp', 'ssh', 'client', 'mpi'], default='mpi', help='Type of communicator') @@ -339,6 +343,7 @@ def parse_args(): # =================== save libE output to pickle and np ======================== + def save_libE_output(H, persis_info, calling_file, nworkers): script_name = os.path.splitext(os.path.basename(calling_file))[0] short_name = script_name.split("test_", 1).pop() @@ -354,6 +359,7 @@ def save_libE_output(H, persis_info, calling_file, nworkers): # ===================== per-worker numpy random-streams ======================== + def per_worker_stream(persis_info, nworkers): for i in range(nworkers): if i in persis_info: @@ -367,6 +373,5 @@ def per_worker_stream(persis_info, nworkers): return persis_info - def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) From 5768f6b0ecb512681ecffcea10f538465a640bed Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:01:11 -0600 Subject: [PATCH 516/644] Add python segment back into .readthedocs.yml --- .readthedocs.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 45ad794c7..279fa2a21 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,3 +8,8 @@ sphinx: build: image: latest + +python: + version: 3.6 + pip_install: true + setup_py_install: true \ No newline at end of file From 16ff9959573bd4ae820682fb6a0aa888ab2d56f2 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:08:58 -0600 Subject: [PATCH 517/644] Try pip install via environment.yml --- .readthedocs.yml | 8 ++++---- conda/environment.yml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 279fa2a21..9cca83e70 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ sphinx: build: image: latest -python: - version: 3.6 - pip_install: true - setup_py_install: true \ No newline at end of file +#python: + #version: 3.6 + #pip_install: true + #setup_py_install: true \ No newline at end of file diff --git a/conda/environment.yml b/conda/environment.yml index 01fd94b91..05403b9a9 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -5,3 +5,5 @@ channels: dependencies: - sphinxcontrib-bibtex - python>=3.6 + - pip: + - . From 345a60b87d766b6d01c5c4bb3252b0ba08b5b695 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:12:41 -0600 Subject: [PATCH 518/644] Try pip install via environment.yml --- conda/environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/environment.yml b/conda/environment.yml index 05403b9a9..3f8afb6e7 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -5,5 +5,4 @@ channels: dependencies: - sphinxcontrib-bibtex - python>=3.6 - - pip: - - . + From fc62d225170b7832fbdfd15aabecc44b7d39559f Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 16:27:03 -0600 Subject: [PATCH 519/644] silly typo --- .../tests/regression_tests/script_test_balsam_hworld.py | 2 +- libensemble/tests/regression_tests/test_1d_sampling.py | 2 +- .../regression_tests/test_1d_uniform_sampling_with_comm_dup.py | 2 +- .../test_6-hump_camel_active_persistent_worker_abort.py | 2 +- .../tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py | 2 +- .../regression_tests/test_6-hump_camel_elapsed_time_abort.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_1.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_2.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_3.py | 2 +- .../regression_tests/test_6-hump_camel_persistent_aposmm_4.py | 2 +- .../test_6-hump_camel_persistent_uniform_sampling.py | 2 +- .../regression_tests/test_6-hump_camel_pregenerated_sample.py | 2 +- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 2 +- ...ump_camel_uniform_sampling_with_persistent_localopt_gens.py | 2 +- .../test_6-hump_camel_with_different_nodes_uniform_sample.py | 2 +- .../test_branin_aposmm_nlopt_and_then_scipy.py | 2 +- libensemble/tests/regression_tests/test_calc_exception.py | 2 +- .../test_chwirut_aposmm_one_residual_at_a_time.py | 2 +- libensemble/tests/regression_tests/test_chwirut_pounders.py | 2 +- .../tests/regression_tests/test_chwirut_pounders_persistent.py | 2 +- .../tests/regression_tests/test_chwirut_pounders_splitcomm.py | 2 +- .../tests/regression_tests/test_chwirut_pounders_subcomm.py | 2 +- .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 2 +- libensemble/tests/regression_tests/test_comms.py | 2 +- libensemble/tests/regression_tests/test_fast_alloc.py | 2 +- .../tests/regression_tests/test_inverse_bayes_example.py | 2 +- .../tests/regression_tests/test_jobcontroller_hworld.py | 3 ++- libensemble/tests/regression_tests/test_mpi_comms.py | 2 +- libensemble/tests/regression_tests/test_nan_func_aposmm.py | 2 +- libensemble/tests/regression_tests/test_worker_exceptions.py | 2 +- 30 files changed, 31 insertions(+), 30 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 8f0e671b0..4b02032c1 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -11,7 +11,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import per_worker_stream +from libensemble.utils import per_worker_stream mpi4py.rc.recv_mprobe = False # Disable matching probes diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index 657c140bb..beb826a62 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() libE_specs['save_every_k_gens'] = 300 diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index f48687131..d677df940 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -23,7 +23,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index 51601673e..7a48aeaa7 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 88c15d6a0..52ad3004c 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_to_aposmm import give_sim_work_first as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_1 as persis_info, aposmm_gen_out as gen_out, six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index 28a965189..c4021df37 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.util import parse_args, save_libE_output, per_worker_stream, eprint +from libensemble.utils import parse_args, save_libE_output, per_worker_stream, eprint nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 86e8a2f59..7c2b51f28 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index 365e98c12..53b3c19fd 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index 912daf34b..b016f7a1e 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 549242cef..7d8b16eca 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py index 3e29a043e..3344eff6b 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py index 929aa147a..a992d5123 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_pregenerated_sample.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.alloc_funcs.give_pregenerated_work import give_pregenerated_sim_work as alloc_f -from libensemble.util import parse_args, save_libE_output +from libensemble.utils import parse_args, save_libE_output nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index c99edd32d..6c9948fa4 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index 795d1f286..4d48ed7c2 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index 0bab4daf2..9b50710cb 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel_with_different_ranks_and_nodes as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index fd7cec063..71767a494 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.branin.branin_obj import call_branin as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out, branin_vals_and_minima as M -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index fb2dfffe4..f7cf98b19 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -18,7 +18,7 @@ from libensemble.libE import libE from libensemble.libE_manager import ManagerException from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import parse_args, per_worker_stream +from libensemble.utils import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py index 44e5c71e1..fd59b09e6 100644 --- a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import persis_info_3 as persis_info, aposmm_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders.py b/libensemble/tests/regression_tests/test_chwirut_pounders.py index d724874da..aca432e06 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index 943319409..cb013ef0e 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.gen_funcs.sampling import lhs_sample from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 2db4612a2..53bc681a6 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.util import parse_args, save_libE_output, per_worker_stream, mpi_comm_split +from libensemble.utils import parse_args, save_libE_output, per_worker_stream, mpi_comm_split num_comms = 2 # Must have atleast num_comms*2 processors nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 585faca30..73f6a07d0 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.util import parse_args, save_libE_output, per_worker_stream, mpi_comm_excl +from libensemble.utils import parse_args, save_libE_output, per_worker_stream, mpi_comm_excl nworkers, is_master, libE_specs, _ = parse_args() libE_specs['comm'], mpi_comm_null = mpi_comm_excl() diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index d973557f8..83fb5e8a2 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -27,7 +27,7 @@ from libensemble.gen_funcs.sampling import uniform_random_sample_obj_components as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f from libensemble.tests.regression_tests.support import persis_info_3 as persis_info -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() if libE_specs['comms'] == 'tcp': diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index 7187fb320..7161c6883 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.comms_testing import float_x1000 as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.mpi_controller import MPIJobController # Only used to get workerID in float_x1000 jobctrl = MPIJobController(auto_resources=False) diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 8764eaee1..5a2043b0f 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as alloc_f -from libensemble.util import parse_args, per_worker_stream +from libensemble.utils import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index bcd5278cb..f9b45b60e 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -24,7 +24,7 @@ from libensemble.sim_funcs.inverse_bayes import likelihood_calculator as sim_f from libensemble.gen_funcs.persistent_inverse_bayes import persistent_updater_after_likelihood as gen_f from libensemble.alloc_funcs.inverse_bayes_allocf import only_persistent_gens_for_inverse_bayes as alloc_f -from libensemble.util import parse_args, per_worker_stream +from libensemble.utils import parse_args, per_worker_stream # Parse args for test code nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index e5f916344..9c4d89e75 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -21,7 +21,8 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import build_simfunc, parse_args, per_worker_stream +from libensemble.util import parse_args, per_worker_stream +from libensemble.tests.regression_tests.common import build_simfunc # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local tcp diff --git a/libensemble/tests/regression_tests/test_mpi_comms.py b/libensemble/tests/regression_tests/test_mpi_comms.py index fc2c3cda2..1c29b3a26 100644 --- a/libensemble/tests/regression_tests/test_mpi_comms.py +++ b/libensemble/tests/regression_tests/test_mpi_comms.py @@ -10,7 +10,7 @@ import sys from mpi4py import MPI from libensemble.comms.mpi import MPIComm, Timeout -from libensemble.util import parse_args +from libensemble.utils import parse_args # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi diff --git a/libensemble/tests/regression_tests/test_nan_func_aposmm.py b/libensemble/tests/regression_tests/test_nan_func_aposmm.py index 2cfff3de6..bb8d5f1dc 100644 --- a/libensemble/tests/regression_tests/test_nan_func_aposmm.py +++ b/libensemble/tests/regression_tests/test_nan_func_aposmm.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from support import nan_func as sim_f, aposmm_gen_out as gen_out from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f -from libensemble.util import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() n = 2 diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 96e51ae79..a8db08e2a 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -19,7 +19,7 @@ from libensemble.tests.regression_tests.support import nan_func as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE_manager import ManagerException -from libensemble.util import parse_args, per_worker_stream +from libensemble.utils import parse_args, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() n = 2 From 8fc6f20383b20532933ae21fce699c23a298d576 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:33:00 -0600 Subject: [PATCH 520/644] Remove version2 - last try with conda --- .readthedocs.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 9cca83e70..52dec1e2d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,4 @@ -version: 2 +#version: 2 conda: environment: conda/environment.yml @@ -9,7 +9,7 @@ sphinx: build: image: latest -#python: - #version: 3.6 - #pip_install: true - #setup_py_install: true \ No newline at end of file +python: + version: 3.6 + pip_install: true + setup_py_install: true \ No newline at end of file From 28d212d86627ae33c84cb790e60617006bed98fb Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:37:33 -0600 Subject: [PATCH 521/644] Remove version2 - last try with conda --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 52dec1e2d..fc115834f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,7 @@ #version: 2 conda: - environment: conda/environment.yml + file: conda/environment.yml sphinx: configuration: docs/conf.py From 8179b197356392904383228b8d2762dc017f724b Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:44:55 -0600 Subject: [PATCH 522/644] Last may have wrked on ver1. Try version 2 --- .readthedocs.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index fc115834f..56981be9d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,4 @@ -#version: 2 +version: 2 conda: file: conda/environment.yml @@ -10,6 +10,4 @@ build: image: latest python: - version: 3.6 - pip_install: true - setup_py_install: true \ No newline at end of file + install: true \ No newline at end of file From ac602ea7afbc01149aaf07a8d2c0d1f3cf3d79fb Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:45:45 -0600 Subject: [PATCH 523/644] Fix environemnt keyword. Try version 2 --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 56981be9d..df05b2ff7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,7 @@ version: 2 conda: - file: conda/environment.yml + environment: conda/environment.yml sphinx: configuration: docs/conf.py From f00395432290b9abde47a1d97b2d27a60d1391e0 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:49:52 -0600 Subject: [PATCH 524/644] python install give path. Try version 2 --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index df05b2ff7..ab5b7e05f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,4 +10,4 @@ build: image: latest python: - install: true \ No newline at end of file + install: . \ No newline at end of file From cb6816c8de8daf546959d32fda2f1b49447badac Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 16:54:23 -0600 Subject: [PATCH 525/644] Correct install section. Try version 2 --- .readthedocs.yml | 9 ++++++++- conda/environment.yml | 1 - setup.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index ab5b7e05f..e2f282634 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,4 +10,11 @@ build: image: latest python: - install: . \ No newline at end of file + install: + - method: pip + path: . + #- method: setuptools + #This would do pip install .[docs] - if have a 'docs' dep in setup.py + #extra_requirements: + #- docs + \ No newline at end of file diff --git a/conda/environment.yml b/conda/environment.yml index 3f8afb6e7..01fd94b91 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -5,4 +5,3 @@ channels: dependencies: - sphinxcontrib-bibtex - python>=3.6 - diff --git a/setup.py b/setup.py index cb55be1d2..b6439cba8 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def run_tests(self): ], extras_require={ - 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py', 'sphinxcontrib.bibtex']}, + 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py']}, classifiers=[ 'Development Status :: 4 - Beta', From c61a63dbd8af723b9328b166d38148669d64fefa Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 17:01:21 -0600 Subject: [PATCH 526/644] account for split/subcomm not being moved. forgot one. --- .../tests/regression_tests/test_chwirut_pounders_splitcomm.py | 3 ++- .../tests/regression_tests/test_chwirut_pounders_subcomm.py | 3 ++- .../tests/regression_tests/test_jobcontroller_hworld.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 53bc681a6..9cd7a7fb7 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -21,7 +21,8 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.utils import parse_args, save_libE_output, per_worker_stream, mpi_comm_split +from libensemble.tests.regression_tests.common import mpi_comm_split +from libensemble.utils import parse_args, save_libE_output, per_worker_stream num_comms = 2 # Must have atleast num_comms*2 processors nworkers, is_master, libE_specs, _ = parse_args() diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 73f6a07d0..91664c4a6 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -20,7 +20,8 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.utils import parse_args, save_libE_output, per_worker_stream, mpi_comm_excl +from libEnsemble.tests.regression_tests.common import mpi_comm_excl +from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() libE_specs['comm'], mpi_comm_null = mpi_comm_excl() diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 9c4d89e75..95289cfe2 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -21,7 +21,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.util import parse_args, per_worker_stream +from libensemble.utils import parse_args, per_worker_stream from libensemble.tests.regression_tests.common import build_simfunc # Do not change these lines - they are parsed by run-tests.sh From 8f95e595189d808397505bb4cc967bd926f9a1c1 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 17:07:26 -0600 Subject: [PATCH 527/644] Add extras require docs. Try version 2 --- .readthedocs.yml | 7 ++----- setup.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index e2f282634..ec511c9a7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,8 +13,5 @@ python: install: - method: pip path: . - #- method: setuptools - #This would do pip install .[docs] - if have a 'docs' dep in setup.py - #extra_requirements: - #- docs - \ No newline at end of file + extra_requirements: + - docs diff --git a/setup.py b/setup.py index b6439cba8..a6dd88a10 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def run_tests(self): extras_require={ 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py']}, + 'docs': ['sphinxcontrib.bibtex'] classifiers=[ 'Development Status :: 4 - Beta', From 5900422195528ad3d4b7a1fda6b724c0f103f850 Mon Sep 17 00:00:00 2001 From: shudson Date: Thu, 14 Nov 2019 17:20:24 -0600 Subject: [PATCH 528/644] Fix format extras require docs. Try version 2 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a6dd88a10..e8c6236ef 100644 --- a/setup.py +++ b/setup.py @@ -61,8 +61,8 @@ def run_tests(self): ], extras_require={ - 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py']}, - 'docs': ['sphinxcontrib.bibtex'] + 'extras': ['scipy', 'nlopt', 'mpi4py', 'petsc', 'petsc4py'], + 'docs': ['sphinxcontrib.bibtex']}, classifiers=[ 'Development Status :: 4 - Beta', From 477432d7ca7b99502485066901226f9affe33fd0 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 14 Nov 2019 17:28:27 -0600 Subject: [PATCH 529/644] E -> e --- .../tests/regression_tests/test_chwirut_pounders_subcomm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 91664c4a6..87ae4ead8 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libEnsemble.tests.regression_tests.common import mpi_comm_excl +from libensemble.tests.regression_tests.common import mpi_comm_excl from libensemble.utils import parse_args, save_libE_output, per_worker_stream nworkers, is_master, libE_specs, _ = parse_args() From 2d8625d8315f3c55748979491fa8155e5922559b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 14 Nov 2019 20:13:37 -0600 Subject: [PATCH 530/644] Debugging readthedocs --- .readthedocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index ec511c9a7..e942b9a2e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,6 +9,9 @@ sphinx: build: image: latest +formats: + - pdf + python: install: - method: pip From 2b94e32a15b17145d43d0c8e7e2d9f1257cdcc9f Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 10:25:00 -0600 Subject: [PATCH 531/644] update parse_args detail in theta doc --- docs/platforms/theta.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 95da5a6fd..d95482971 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -135,7 +135,7 @@ Batch Runs Batch scripts specify run-settings using ``#COBALT`` statements. The following simple example depicts configuring and launching libEnsemble to a MOM node with multiprocessing. This script also assumes the user is using the ``parse_args()`` -convenience function within libEnsemble's ``/regression_tests/common.py``. +convenience function within libEnsemble's ``utils.py``. .. code-block:: bash From 032bbdebfaaeb753d1c460414ffd8d0ca36830ee Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 15 Nov 2019 11:01:07 -0600 Subject: [PATCH 532/644] Removing user_funcs.rst --- docs/programming_libE.rst | 8 +++++++- docs/user_funcs.rst | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 docs/user_funcs.rst diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index 175d9bfee..2a4d42b0a 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -8,6 +8,12 @@ Programming with libEnsemble .. toctree:: history_output data_structures/data_structures - user_funcs + +.. toctree:: + :caption: libEnsemble User Functions: + + sim_gen_alloc_funcs + +.. toctree:: job_controller/jc_index logging diff --git a/docs/user_funcs.rst b/docs/user_funcs.rst deleted file mode 100644 index 65a516a45..000000000 --- a/docs/user_funcs.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. toctree:: - :caption: libEnsemble User Functions: - - sim_gen_alloc_funcs From d7efa17f6bb2b3a03945089b23e9392aa8994866 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 11:38:38 -0600 Subject: [PATCH 533/644] replace nprocesses instances with nworkers. #235 --- docs/data_structures/libE_specs.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 2 +- examples/tutorials/tutorial_calling.py | 2 +- libensemble/libE.py | 8 ++++---- .../tests/scaling_tests/forces/run_libe_forces.py | 2 +- libensemble/utils.py | 12 ++++++------ 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index 5fbff8752..f68ce1c4f 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -9,7 +9,7 @@ Specifications for libEnsemble:: 'comms' [string] : Manager/Worker communications mode. Default: mpi Options are 'mpi', 'local', 'tcp' - 'nprocesses' [int]: + 'nworkers' [int]: Number of worker processes to spawn (in local/tcp modes) 'comm' [MPI communicator] : libEnsemble communicator. Default: MPI.COMM_WORLD diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 1ea6d3856..335fba391 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -173,7 +173,7 @@ use. Our communication method, ``'local'``, refers to Python's Multiprocessing. from simulator import sim_find_sine nworkers = 4 - libE_specs = {'nprocesses': nworkers, 'comms': 'local'} + libE_specs = {'nworkers': nworkers, 'comms': 'local'} We configure the settings and specifications for our ``sim_f`` and ``gen_f`` functions in the :ref:`gen_specs` and diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 6681f20db..88757f53b 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -5,7 +5,7 @@ from tutorial_sim import sim_find_sine nworkers = 4 -libE_specs = {'nprocesses': nworkers, 'comms': 'local'} +libE_specs = {'nworkers': nworkers, 'comms': 'local'} gen_specs = {'gen_f': gen_random_sample, # Our generator function 'out': [('x', float, (1,))], # gen_f output (name, type, size). diff --git a/libensemble/libE.py b/libensemble/libE.py index 089c87e07..0980e9d94 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -39,7 +39,7 @@ def libE(sim_specs, gen_specs, exit_criteria, We dispatch to different types of worker teams depending on the contents of libE_specs. If 'comm' is a field, we use MPI; - if 'nthreads' is a field, we use threads; if 'nprocesses' is a + if 'nthreads' is a field, we use threads; if 'nworkers' is a field, we use multiprocessing. If an exception is encountered by the manager or workers, the @@ -262,7 +262,7 @@ def libE_local(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs, H0): "Main routine for thread/process launch of libE." - nworkers = libE_specs['nprocesses'] + nworkers = libE_specs['nworkers'] check_inputs(libE_specs, alloc_specs, sim_specs, gen_specs, exit_criteria, H0) jobctl = JobController.controller @@ -375,9 +375,9 @@ def libE_tcp_mgr(sim_specs, gen_specs, exit_criteria, launchf = libE_tcp_worker_launcher(libE_specs) # Get worker launch parameters and fill in defaults for TCP/IP conn - if 'nprocesses' in libE_specs: + if 'nworkers' in libE_specs: workers = None - nworkers = libE_specs['nprocesses'] + nworkers = libE_specs['nworkers'] elif 'workers' in libE_specs: workers = libE_specs['workers'] nworkers = len(workers) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index d0ed23af8..27ab74cca 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -30,7 +30,7 @@ print("WARNING: nworkers not passed to script - defaulting to 4") nworkers = 4 is_master = True # processes are forked in libE - libE_specs = {'nprocesses': nworkers, 'comms': 'local'} + libE_specs = {'nworkers': nworkers, 'comms': 'local'} if is_master: print('\nRunning with {} workers\n'.format(nworkers)) diff --git a/libensemble/utils.py b/libensemble/utils.py index 30e9b5b2f..b14a27491 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -51,7 +51,7 @@ 'port', # 'authkey', # 'workerID', # - 'nprocesses', # + 'nworkers', # 'worker_cmd', # 'abort_on_exception', # 'sim_dir', # @@ -107,10 +107,10 @@ def check_libE_specs(libE_specs, serial_check=False): if not serial_check: assert libE_specs['comm'].Get_size() > 1, "Manager only - must be at least one worker (2 MPI tasks)" elif comms_type in ['local']: - assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" + assert libE_specs['nworkers'] >= 1, "Must specify at least one worker" elif comms_type in ['tcp']: # TODO, differentiate and test SSH/Client - assert libE_specs['nprocesses'] >= 1, "Must specify at least one worker" + assert libE_specs['nworkers'] >= 1, "Must specify at least one worker" for k in libE_specs.keys(): assert k in allowed_libE_spec_keys, "Key %s is not allowed in libE_specs. Supported keys are: %s " % (k, allowed_libE_spec_keys) @@ -276,7 +276,7 @@ def mpi_parse_args(args): def local_parse_args(args): "Parse arguments for forked processes using multiprocessing." nworkers = args.nworkers or 4 - libE_specs = {'nprocesses': nworkers, 'comms': 'local'} + libE_specs = {'nworkers': nworkers, 'comms': 'local'} return nworkers, True, libE_specs, args.tester_args @@ -288,7 +288,7 @@ def tcp_parse_args(args): "{manager_ip}", "{manager_port}", "{authkey}", "--workerID", "{workerID}", "--nworkers", str(nworkers)] - libE_specs = {'nprocesses': nworkers, 'worker_cmd': cmd, 'comms': 'tcp'} + libE_specs = {'nworkers': nworkers, 'worker_cmd': cmd, 'comms': 'tcp'} return nworkers, True, libE_specs, args.tester_args @@ -323,7 +323,7 @@ def client_parse_args(args): 'port': int(port), 'authkey': authkey.encode('utf-8'), 'workerID': args.workerID, - 'nprocesses': nworkers, + 'nworkers': nworkers, 'comms': 'tcp'} return nworkers, False, libE_specs, args.tester_args From 35ada944a358c50fcc711dcb43f33b043bfcc78a Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 15 Nov 2019 12:36:09 -0600 Subject: [PATCH 534/644] Removing unnecessary files --- docs/appendix.rst | 10 ---------- docs/dev_info.rst | 8 -------- docs/latex_index.rst | 25 +++++++++++++++++++++++-- 3 files changed, 23 insertions(+), 20 deletions(-) delete mode 100644 docs/appendix.rst delete mode 100644 docs/dev_info.rst diff --git a/docs/appendix.rst b/docs/appendix.rst deleted file mode 100644 index c642ae13e..000000000 --- a/docs/appendix.rst +++ /dev/null @@ -1,10 +0,0 @@ -Appendices -========== - -.. toctree:: - tutorials/tutorials - FAQ - known_issues - examples/examples_index - release_notes - bibliography diff --git a/docs/dev_info.rst b/docs/dev_info.rst deleted file mode 100644 index 4c4fdab68..000000000 --- a/docs/dev_info.rst +++ /dev/null @@ -1,8 +0,0 @@ -Developer's Guide -================= - -.. toctree:: - - contributing - dev_guide/dev_API/developer_API.rst - dev_guide/release_management/release_index.rst diff --git a/docs/latex_index.rst b/docs/latex_index.rst index 8ca9be61b..f8643b832 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -8,5 +8,26 @@ Introduction programming_libE platforms/platforms_index - dev_info - appendix + +Developer's Guide +================= + +.. toctree:: + :maxdepth: 3 + + contributing + dev_guide/dev_API/developer_API.rst + dev_guide/release_management/release_index.rst + +Appendices +========== + +.. toctree:: + :maxdepth: 3 + + tutorials/tutorials + FAQ + known_issues + examples/examples_index + release_notes + bibliography From eeef9bb4d34009ea21a5362eaf721cb68e859caf Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 15 Nov 2019 13:34:35 -0600 Subject: [PATCH 535/644] Slight renames, adding bibliography to html --- docs/index.rst | 1 + docs/introduction_latex.rst | 4 ++-- docs/latex_index.rst | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 9d784d684..a4e1e59eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ known_issues release_notes contributing + bibliography .. toctree:: :maxdepth: 2 diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index 8a85f4686..e287bb7a7 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -13,8 +13,8 @@ Example Use Cases .. include:: overview_usecases.rst :start-after: begin_usecases_rst_tag -Further Information -=================== +Quickstart Guide +================ .. include:: ../README.rst :start-after: before_dependencies_rst_tag :end-before: after_resources_rst_tag diff --git a/docs/latex_index.rst b/docs/latex_index.rst index f8643b832..76e999e3b 100644 --- a/docs/latex_index.rst +++ b/docs/latex_index.rst @@ -30,4 +30,5 @@ Appendices known_issues examples/examples_index release_notes - bibliography + +.. bibliography:: references.bib From aa16c8a6afd2136dd5414357edd2a66b54e1c550 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 14:20:12 -0600 Subject: [PATCH 536/644] parse_args in __all__ for addition from other PR --- libensemble/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index a9bced316..7bb74e657 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -4,7 +4,7 @@ """ -__all__ = ['check_inputs'] +__all__ = ['check_inputs', 'parse_args'] import traceback import logging From 9d99157a4ad6eea5d81dc0dcf7e3f5538ce92f10 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 15 Nov 2019 14:23:41 -0600 Subject: [PATCH 537/644] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index a4e1e59eb..fd47b94a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ .. toctree:: :maxdepth: 2 - :caption: Additional Reference: + :caption: Additional References: FAQ known_issues From 81065f87b2d6847edc8e2c0897bf95371751f32e Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 14:32:28 -0600 Subject: [PATCH 538/644] Revert "parse_args in __all__ for addition from other PR" This reverts commit aa16c8a6afd2136dd5414357edd2a66b54e1c550. --- libensemble/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 7bb74e657..a9bced316 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -4,7 +4,7 @@ """ -__all__ = ['check_inputs', 'parse_args'] +__all__ = ['check_inputs'] import traceback import logging From 257459d509f132b38197434dbe4579b60682d89b Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:12:26 -0600 Subject: [PATCH 539/644] Main libE module with own heading under Programming --- docs/libe_module.rst | 6 ++++++ docs/programming_libE.rst | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 docs/libe_module.rst diff --git a/docs/libe_module.rst b/docs/libe_module.rst new file mode 100644 index 000000000..4b5f6c024 --- /dev/null +++ b/docs/libe_module.rst @@ -0,0 +1,6 @@ +The Main libE Module +==================== + +.. automodule:: libE + :members: + :no-undoc-members: diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index 2a4d42b0a..4c99ee5ba 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -1,11 +1,8 @@ Programming with libEnsemble ============================ -.. automodule:: libE - :members: - :no-undoc-members: - .. toctree:: + libe_module history_output data_structures/data_structures From 960c7035447fdac488a04d7a2732c5f077d7fddb Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:15:46 -0600 Subject: [PATCH 540/644] small wording changes --- docs/overview_usecases.rst | 2 +- libensemble/libE.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 6fdc1e69e..47a552a75 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -7,7 +7,7 @@ Overview libEnsemble is a Python library to coordinate the concurrent evaluation of dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to -various workers. A libEnsemble worker is the smallest indivisible unit to +multiple workers. A libEnsemble worker is the smallest indivisible unit to perform some calculation. The work performed by libEnsemble is governed by three routines: diff --git a/libensemble/libE.py b/libensemble/libE.py index a4468e1b3..39c5022c4 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -1,13 +1,13 @@ """ This is the outer libEnsemble routine. -We dispatch to different types of worker teams depending on the contents of -``libE_specs``. If ``'comm'`` is a field, MPI is used; if ``'nthreads'`` is a -field, threads are used ; if ``'nprocesses'`` is a field, multiprocessing is -used. +libEnsemble dispatches to different types of worker teams depending on the +contents of ``libE_specs``. If ``'comm'`` is a field, MPI is used; if +``'nthreads'`` is a field, threads are used ; if ``'nworkers'`` is a field, +multiprocessing is used. If an exception is encountered by the manager or workers, the -history array is dumped to file and MPI abort is called. +History array is dumped to file and MPI abort is called. """ __all__ = ['libE'] From 05e3d353ffcb2cb00f1b274a75d55adb81c70720 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:40:04 -0600 Subject: [PATCH 541/644] reword libE module description --- libensemble/libE.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 39c5022c4..141962ed5 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -1,13 +1,15 @@ """ This is the outer libEnsemble routine. -libEnsemble dispatches to different types of worker teams depending on the -contents of ``libE_specs``. If ``'comm'`` is a field, MPI is used; if -``'nthreads'`` is a field, threads are used ; if ``'nworkers'`` is a field, -multiprocessing is used. - -If an exception is encountered by the manager or workers, the -History array is dumped to file and MPI abort is called. +This module sets up the manager and the team of workers, configured according +to the contents of the 'libE_specs' dictionary. The Manager/Worker +communications scheme used within libEnsemble is parsed from the 'comms' key +if present, with valid values being 'MPI', 'local' (for multiprocessing), or +'TCP'. MPI is the default. If no communication scheme or MPI communicator is +specified, a duplicate of COMM_WORLD will be used. + +If an exception is encountered by the manager or workers, the history array +is dumped to file and MPI abort is called. """ __all__ = ['libE'] From 20b7bb9da289ef2907c7202871f85e8eb6d9a09e Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:43:30 -0600 Subject: [PATCH 542/644] formatting --- libensemble/libE.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 141962ed5..9d04db23d 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -2,10 +2,10 @@ This is the outer libEnsemble routine. This module sets up the manager and the team of workers, configured according -to the contents of the 'libE_specs' dictionary. The Manager/Worker -communications scheme used within libEnsemble is parsed from the 'comms' key -if present, with valid values being 'MPI', 'local' (for multiprocessing), or -'TCP'. MPI is the default. If no communication scheme or MPI communicator is +to the contents of the ``libE_specs`` dictionary. The Manager/Worker +communications scheme used within libEnsemble is parsed from the ``comms`` key +if present, with valid values being ``MPI``, ``local`` (for multiprocessing), or +``TCP``. MPI is the default. If no communication scheme or MPI communicator is specified, a duplicate of COMM_WORLD will be used. If an exception is encountered by the manager or workers, the history array From 100b295fd59497f160eaa07fc65072cb3d770679 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:55:59 -0600 Subject: [PATCH 543/644] rewording comms statement --- libensemble/libE.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 9d04db23d..c30c03361 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -5,8 +5,8 @@ to the contents of the ``libE_specs`` dictionary. The Manager/Worker communications scheme used within libEnsemble is parsed from the ``comms`` key if present, with valid values being ``MPI``, ``local`` (for multiprocessing), or -``TCP``. MPI is the default. If no communication scheme or MPI communicator is -specified, a duplicate of COMM_WORLD will be used. +``TCP``. MPI is the default; if no communicator is specified, a duplicate of +COMM_WORLD will be used. If an exception is encountered by the manager or workers, the history array is dumped to file and MPI abort is called. From 32fe9011966aae89514b759d73dc81f14419e3c1 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 15:59:58 -0600 Subject: [PATCH 544/644] lowercase --- libensemble/libE.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index c30c03361..c63229e82 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -4,8 +4,8 @@ This module sets up the manager and the team of workers, configured according to the contents of the ``libE_specs`` dictionary. The Manager/Worker communications scheme used within libEnsemble is parsed from the ``comms`` key -if present, with valid values being ``MPI``, ``local`` (for multiprocessing), or -``TCP``. MPI is the default; if no communicator is specified, a duplicate of +if present, with valid values being ``mpi``, ``local`` (for multiprocessing), or +``tcp``. MPI is the default; if no communicator is specified, a duplicate of COMM_WORLD will be used. If an exception is encountered by the manager or workers, the history array From 1318f3fefe77cc9cb9f02678daf04916da9068d6 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 15 Nov 2019 17:06:08 -0600 Subject: [PATCH 545/644] fix single test --- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index f43f35567..f0d1bb809 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel from libensemble.gen_funcs.sampling import uniform_random_sample -from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream +from libensemble.tests.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() From 511a9f8b28b52026229e11debde9da0fed7f972d Mon Sep 17 00:00:00 2001 From: shudson Date: Fri, 15 Nov 2019 17:50:25 -0600 Subject: [PATCH 546/644] Fix another single test --- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index f0d1bb809..e030d972c 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel from libensemble.gen_funcs.sampling import uniform_random_sample -from libensemble.tests.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, per_worker_stream from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() From d71b8f0098201a0e6ea7be020d01d06522cb5a37 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 18 Nov 2019 11:20:08 -0600 Subject: [PATCH 547/644] Renaming per_worker_streams, add a doc string, and changing an argument name --- .../tests/regression_tests/script_test_balsam_hworld.py | 4 ++-- libensemble/tests/regression_tests/test_1d_sampling.py | 4 ++-- .../test_1d_uniform_sampling_with_comm_dup.py | 4 ++-- .../test_6-hump_camel_active_persistent_worker_abort.py | 4 ++-- .../regression_tests/test_6-hump_camel_aposmm_LD_MMA.py | 4 ++-- .../test_6-hump_camel_elapsed_time_abort.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_1.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_2.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_3.py | 4 ++-- .../test_6-hump_camel_persistent_aposmm_4.py | 4 ++-- .../test_6-hump_camel_persistent_uniform_sampling.py | 4 ++-- .../regression_tests/test_6-hump_camel_uniform_sampling.py | 4 ++-- ..._camel_uniform_sampling_with_persistent_localopt_gens.py | 4 ++-- ...test_6-hump_camel_with_different_nodes_uniform_sample.py | 4 ++-- .../test_branin_aposmm_nlopt_and_then_scipy.py | 4 ++-- libensemble/tests/regression_tests/test_calc_exception.py | 4 ++-- .../test_chwirut_aposmm_one_residual_at_a_time.py | 4 ++-- libensemble/tests/regression_tests/test_chwirut_pounders.py | 4 ++-- .../regression_tests/test_chwirut_pounders_persistent.py | 4 ++-- .../regression_tests/test_chwirut_pounders_splitcomm.py | 4 ++-- .../tests/regression_tests/test_chwirut_pounders_subcomm.py | 4 ++-- .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 4 ++-- libensemble/tests/regression_tests/test_comms.py | 4 ++-- libensemble/tests/regression_tests/test_fast_alloc.py | 4 ++-- .../tests/regression_tests/test_inverse_bayes_example.py | 4 ++-- .../tests/regression_tests/test_jobcontroller_hworld.py | 4 ++-- libensemble/tests/regression_tests/test_nan_func_aposmm.py | 4 ++-- .../tests/regression_tests/test_worker_exceptions.py | 4 ++-- libensemble/utils.py | 6 ++++-- 29 files changed, 60 insertions(+), 58 deletions(-) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 4b02032c1..47bce6751 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -11,7 +11,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.utils import per_worker_stream +from libensemble.utils import add_unique_random_streams mpi4py.rc.recv_mprobe = False # Disable matching probes @@ -56,7 +56,7 @@ def build_simfunc(): 'num_active_gens': 1} } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'elapsed_wallclock_time': 35} diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index beb826a62..992e12b9c 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() libE_specs['save_every_k_gens'] = 300 @@ -34,7 +34,7 @@ } } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'gen_max': 501} diff --git a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py index d677df940..139b4e622 100644 --- a/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/regression_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -23,7 +23,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.one_d_func import one_d_example as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -51,7 +51,7 @@ } } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'gen_max': 501} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py index 7a48aeaa7..63cb9485f 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() @@ -46,7 +46,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': gen_out, 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) # Set sim_max small so persistent worker is quickly terminated exit_criteria = {'sim_max': 10, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py index 52ad3004c..8388189bb 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_to_aposmm import give_sim_work_first as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import persis_info_1 as persis_info, aposmm_gen_out as gen_out, six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() @@ -51,7 +51,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) exit_criteria = {'sim_max': 1000} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py index c4021df37..0a4602523 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.utils import parse_args, save_libE_output, per_worker_stream, eprint +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams, eprint nworkers, is_master, libE_specs, _ = parse_args() @@ -44,7 +44,7 @@ 'out': [('allocated', bool)], 'user': {'batch_mode': False}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'elapsed_wallclock_time': 1} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 7c2b51f28..2ac709b55 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time @@ -60,7 +60,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 1000} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py index 53b3c19fd..d238e2ae0 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_2.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time @@ -61,7 +61,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 1000} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py index b016f7a1e..dfb537f01 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_3.py @@ -23,7 +23,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time @@ -61,7 +61,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 1000} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py index 7d8b16eca..dff42efc3 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_4.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time @@ -61,7 +61,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 2000} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py index 3344eff6b..1aaa1a6ac 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -44,7 +44,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'gen_max': 40, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py index e030d972c..c913d5945 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel from libensemble.gen_funcs.sampling import uniform_random_sample -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() @@ -42,7 +42,7 @@ } # end_gen_specs_rst_tag -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'gen_max': 501, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py index 4d48ed7c2..edfc37371 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.uniform_or_localopt import uniform_or_localopt as gen_f from libensemble.alloc_funcs.start_persistent_local_opt_gens import start_persistent_local_opt_gens as alloc_f from libensemble.tests.regression_tests.support import uniform_or_localopt_gen_out as gen_out -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima nworkers, is_master, libE_specs, _ = parse_args() @@ -51,7 +51,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': gen_out, 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 1000, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py index 9b50710cb..a0575ba09 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py @@ -22,7 +22,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel_with_different_ranks_and_nodes as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample_with_different_nodes_and_ranks as gen_f from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -70,7 +70,7 @@ 'out': [('allocated', bool)], 'user': {'batch_mode': False}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 10, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py index 71767a494..0d575297b 100644 --- a/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py +++ b/libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.branin.branin_obj import call_branin as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out, branin_vals_and_minima as M -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -60,7 +60,7 @@ 'max_active_runs': 3} } -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) # Tell libEnsemble when to stop (stop_val key must be in H) diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index f7cf98b19..869fbc14a 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -18,7 +18,7 @@ from libensemble.libE import libE from libensemble.libE_manager import ManagerException from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.utils import parse_args, per_worker_stream +from libensemble.utils import parse_args, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -38,7 +38,7 @@ def six_hump_camel_err(H, persis_info, sim_specs, _): 'gen_batch_size': 10} } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'elapsed_wallclock_time': 10} diff --git a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py index fd59b09e6..024d67c72 100644 --- a/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.tests.regression_tests.support import persis_info_3 as persis_info, aposmm_gen_out as gen_out nworkers, is_master, libE_specs, _ = parse_args() @@ -64,7 +64,7 @@ 'stop_partial_fvec_eval': True} } -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders.py b/libensemble/tests/regression_tests/test_chwirut_pounders.py index aca432e06..c0bca13bf 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders.py @@ -20,7 +20,7 @@ from libensemble.sim_funcs.chwirut1 import chwirut_eval as sim_f from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -52,7 +52,7 @@ gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py index cb013ef0e..f78019c94 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_persistent.py @@ -23,7 +23,7 @@ from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.gen_funcs.sampling import lhs_sample from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -63,7 +63,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {'batch_mode': True}} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': 500} diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py index 9cd7a7fb7..1ddef0752 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_splitcomm.py @@ -22,7 +22,7 @@ from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out from libensemble.tests.regression_tests.common import mpi_comm_split -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams num_comms = 2 # Must have atleast num_comms*2 processors nworkers, is_master, libE_specs, _ = parse_args() @@ -57,7 +57,7 @@ gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py index 87ae4ead8..8678e25c8 100644 --- a/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py +++ b/libensemble/tests/regression_tests/test_chwirut_pounders_subcomm.py @@ -21,7 +21,7 @@ from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f from libensemble.tests.regression_tests.support import persis_info_2 as persis_info, aposmm_gen_out as gen_out from libensemble.tests.regression_tests.common import mpi_comm_excl -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() libE_specs['comm'], mpi_comm_null = mpi_comm_excl() @@ -61,7 +61,7 @@ gen_specs['user'].update({'grtol': 1e-4, 'gatol': 1e-4, 'frtol': 1e-15, 'fatol': 1e-15}) -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py index 1f9aad506..8d7c3d497 100644 --- a/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py @@ -27,7 +27,7 @@ from libensemble.gen_funcs.sampling import uniform_random_sample_obj_components as gen_f from libensemble.alloc_funcs.fast_alloc_and_pausing import give_sim_work_first from libensemble.tests.regression_tests.support import persis_info_3 as persis_info -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() if libE_specs['comms'] == 'tcp': @@ -72,7 +72,7 @@ } # end_alloc_specs_rst_tag -persis_info = per_worker_stream(persis_info, nworkers + 1) +persis_info = add_unique_random_streams(persis_info, nworkers + 1) persis_info_safe = deepcopy(persis_info) exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py index 7161c6883..9fe421f27 100644 --- a/libensemble/tests/regression_tests/test_comms.py +++ b/libensemble/tests/regression_tests/test_comms.py @@ -20,7 +20,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.comms_testing import float_x1000 as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble.mpi_controller import MPIJobController # Only used to get workerID in float_x1000 jobctrl = MPIJobController(auto_resources=False) @@ -43,7 +43,7 @@ 'num_active_gens': 1} } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': sim_max, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index 5a2043b0f..0f570e3bb 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -21,7 +21,7 @@ from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as alloc_f -from libensemble.utils import parse_args, per_worker_stream +from libensemble.utils import parse_args, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() @@ -40,7 +40,7 @@ alloc_specs = {'alloc_f': alloc_f, 'out': [('allocated', bool)]} -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'sim_max': num_pts, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index f9b45b60e..84009a37a 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -24,7 +24,7 @@ from libensemble.sim_funcs.inverse_bayes import likelihood_calculator as sim_f from libensemble.gen_funcs.persistent_inverse_bayes import persistent_updater_after_likelihood as gen_f from libensemble.alloc_funcs.inverse_bayes_allocf import only_persistent_gens_for_inverse_bayes as alloc_f -from libensemble.utils import parse_args, per_worker_stream +from libensemble.utils import parse_args, add_unique_random_streams # Parse args for test code nworkers, is_master, libE_specs, _ = parse_args() @@ -45,7 +45,7 @@ 'num_batches': 10} } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) # Tell libEnsemble when to stop exit_criteria = { diff --git a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py index 95289cfe2..ecbece153 100644 --- a/libensemble/tests/regression_tests/test_jobcontroller_hworld.py +++ b/libensemble/tests/regression_tests/test_jobcontroller_hworld.py @@ -21,7 +21,7 @@ from libensemble.libE import libE from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.utils import parse_args, per_worker_stream +from libensemble.utils import parse_args, add_unique_random_streams from libensemble.tests.regression_tests.common import build_simfunc # Do not change these lines - they are parsed by run-tests.sh @@ -82,7 +82,7 @@ } } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {'elapsed_wallclock_time': 10} diff --git a/libensemble/tests/regression_tests/test_nan_func_aposmm.py b/libensemble/tests/regression_tests/test_nan_func_aposmm.py index bb8d5f1dc..0f9afa640 100644 --- a/libensemble/tests/regression_tests/test_nan_func_aposmm.py +++ b/libensemble/tests/regression_tests/test_nan_func_aposmm.py @@ -19,7 +19,7 @@ from libensemble.libE import libE from support import nan_func as sim_f, aposmm_gen_out as gen_out from libensemble.gen_funcs.aposmm import aposmm_logic as gen_f -from libensemble.utils import parse_args, save_libE_output, per_worker_stream +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() n = 2 @@ -44,7 +44,7 @@ gen_specs['user']['components'] = 1 gen_specs['user']['combine_component_func'] = np.linalg.norm -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) # Tell libEnsemble when to stop exit_criteria = {'sim_max': 100, 'elapsed_wallclock_time': 300} diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index a8db08e2a..8645c0ca2 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -19,7 +19,7 @@ from libensemble.tests.regression_tests.support import nan_func as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE_manager import ManagerException -from libensemble.utils import parse_args, per_worker_stream +from libensemble.utils import parse_args, add_unique_random_streams nworkers, is_master, libE_specs, _ = parse_args() n = 2 @@ -35,7 +35,7 @@ 'num_active_gens': 1} } -persis_info = per_worker_stream({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) libE_specs['abort_on_exception'] = False diff --git a/libensemble/utils.py b/libensemble/utils.py index 527852476..63c992cf0 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -362,8 +362,10 @@ def save_libE_output(H, persis_info, calling_file, nworkers): # ===================== per-worker numpy random-streams ======================== -def per_worker_stream(persis_info, nworkers): - for i in range(nworkers): +def add_unique_random_streams(persis_info, size): + # Used to create a random number stream for the libE manager and workers + # (when size = 1 + num_workers). Stream i is initialized with seed=i. + for i in range(size): if i in persis_info: persis_info[i].update({ 'rand_stream': np.random.RandomState(i), From 771e0a1de5d59037cef3589e72ba2c1df71b9e63 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 18 Nov 2019 11:30:45 -0600 Subject: [PATCH 548/644] one space removed --- libensemble/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 63c992cf0..39cb72727 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -364,7 +364,7 @@ def save_libE_output(H, persis_info, calling_file, nworkers): def add_unique_random_streams(persis_info, size): # Used to create a random number stream for the libE manager and workers - # (when size = 1 + num_workers). Stream i is initialized with seed=i. + # (when size = 1 + num_workers). Stream i is initialized with seed=i. for i in range(size): if i in persis_info: persis_info[i].update({ From 95d0f25f54ac84d31cc08f829096c36777c1f55f Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 18 Nov 2019 12:01:41 -0600 Subject: [PATCH 549/644] move manager exception back to libe --- libensemble/libE.py | 31 ++++++++++++++++++++++++++++--- libensemble/utils.py | 23 +---------------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 12bed8b3e..0634bb2f5 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -18,6 +18,9 @@ import logging import random import socket +import traceback +import numpy as np +import pickle # Only used when saving output on error import libensemble.util.launcher as launcher from libensemble.util.timer import Timer @@ -29,7 +32,7 @@ from libensemble.comms.logs import manager_logging_config from libensemble.comms.tcp_mgr import ServerQCommManager, ClientQCommManager from libensemble.controller import JobController -from libensemble.utils import check_inputs, _USER_SIM_ID_WARNING, report_manager_exception +from libensemble.utils import check_inputs, _USER_SIM_ID_WARNING logger = logging.getLogger(__name__) # To change logging level for just this module @@ -136,12 +139,12 @@ def libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, persis_info, logger.info("libE_manager total time: {}".format(elapsed_time)) except ManagerException as e: - report_manager_exception(hist, persis_info, e) + _report_manager_exception(hist, persis_info, e) if libE_specs.get('abort_on_exception', True) and on_abort is not None: on_abort() raise except Exception: - report_manager_exception(hist, persis_info) + _report_manager_exception(hist, persis_info) if libE_specs.get('abort_on_exception', True) and on_abort is not None: on_abort() raise @@ -421,3 +424,25 @@ def libE_tcp_worker(sim_specs, gen_specs, libE_specs): worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=workerID, log_comm=True) logger.debug("Worker {} exiting".format(workerID)) + + +# ==================== Additional Internal Functions =========================== + + +def _report_manager_exception(hist, persis_info, mgr_exc=None): + "Write out exception manager exception to log." + if mgr_exc is not None: + from_line, msg, exc = mgr_exc.args + logger.error("---- {} ----".format(from_line)) + logger.error("Message: {}".format(msg)) + logger.error(exc) + else: + logger.error(traceback.format_exc()) + logger.error("Manager exception raised .. aborting ensemble:") + logger.error("Dumping ensemble history with {} sims evaluated:". + format(hist.sim_count)) + + filename = 'libE_history_at_abort_' + str(hist.sim_count) + np.save(filename + '.npy', hist.trim_H()) + with open(filename + '.pickle', "wb") as f: + pickle.dump(persis_info, f) diff --git a/libensemble/utils.py b/libensemble/utils.py index 527852476..688d30faa 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -6,13 +6,12 @@ __all__ = ['check_inputs'] -import traceback import os import sys import logging import numpy as np import argparse -import pickle # Only used when saving output on error +import pickle logger = logging.getLogger(__name__) # To change logging level for just this module @@ -64,26 +63,6 @@ 'save_every_k_gens', # 'profile_worker'] # - -def report_manager_exception(hist, persis_info, mgr_exc=None): - "Write out exception manager exception to log." - if mgr_exc is not None: - from_line, msg, exc = mgr_exc.args - logger.error("---- {} ----".format(from_line)) - logger.error("Message: {}".format(msg)) - logger.error(exc) - else: - logger.error(traceback.format_exc()) - logger.error("Manager exception raised .. aborting ensemble:") - logger.error("Dumping ensemble history with {} sims evaluated:". - format(hist.sim_count)) - - filename = 'libE_history_at_abort_' + str(hist.sim_count) - np.save(filename + '.npy', hist.trim_H()) - with open(filename + '.pickle', "wb") as f: - pickle.dump(persis_info, f) - - # ==================== Common input checking ================================= _USER_SIM_ID_WARNING = \ ('\n' + 79*'*' + '\n' + From d67782fd66c4291c4e18874ed371292c4893f9b9 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 18 Nov 2019 13:07:19 -0600 Subject: [PATCH 550/644] Changing stream docstring --- libensemble/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 39cb72727..a2deb082b 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -363,8 +363,8 @@ def save_libE_output(H, persis_info, calling_file, nworkers): def add_unique_random_streams(persis_info, size): - # Used to create a random number stream for the libE manager and workers - # (when size = 1 + num_workers). Stream i is initialized with seed=i. + # Creates size random number streams for the libE manager and workers when + # size is num_workers + 1. Stream i is initialized with seed i. for i in range(size): if i in persis_info: persis_info[i].update({ From b143637afb38b9bd71d8106b1bfce6c6b6cea157 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 18 Nov 2019 13:11:47 -0600 Subject: [PATCH 551/644] Fixing ==== line --- libensemble/libE_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/libE_manager.py b/libensemble/libE_manager.py index 0bbfc48d9..4915f0500 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/libE_manager.py @@ -1,6 +1,6 @@ """ libEnsemble manager routines -==================================================== +============================ """ import sys From ab05816dc407cb4292bbb6fb82c73033843f04a5 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 14:15:38 -0600 Subject: [PATCH 552/644] Update forces example --- libensemble/libE_worker.py | 4 +- .../tests/scaling_tests/forces/forces_simf.py | 20 ++---- .../scaling_tests/forces/run_libe_forces.py | 64 +++++++++---------- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/libensemble/libE_worker.py b/libensemble/libE_worker.py index a133b6d1d..cddd0079b 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/libE_worker.py @@ -250,13 +250,13 @@ def _handle_calc(self, Work, calc_in): calc_type_strings[calc_type], timer, job.timer, - calc_status_strings.get(calc_status, "Completed")) + calc_status_strings.get(calc_status, "Not set")) else: calc_msg = "Calc {:5d}: {} {} Status: {}".\ format(calc_id, calc_type_strings[calc_type], timer, - calc_status_strings.get(calc_status, "Completed")) + calc_status_strings.get(calc_status, "Not set")) logging.getLogger(LogConfig.config.stats_name).info(calc_msg) diff --git a/libensemble/tests/scaling_tests/forces/forces_simf.py b/libensemble/tests/scaling_tests/forces/forces_simf.py index 7988d031a..fcd9ff85e 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simf.py +++ b/libensemble/tests/scaling_tests/forces/forces_simf.py @@ -37,16 +37,16 @@ def make_unique_simdir(simdir, count=0): return make_unique_simdir(".".join([simdir.split('.')[0], str(count)]), count) -def run_forces(x, persis_info, sim_specs, libE_info): +def run_forces(H, persis_info, sim_specs, libE_info): # Setting up variables needed for input and output # keys = variable names # x = variable values # output = what will be returned to libE calc_status = 0 # Returns to worker - + + x = H['x'] simdir_basename = sim_specs['user']['simdir_basename'] - # cores = sim_specs['user']['cores'] keys = sim_specs['user']['keys'] sim_particles = sim_specs['user']['sim_particles'] sim_timesteps = sim_specs['user']['sim_timesteps'] @@ -58,15 +58,6 @@ def run_forces(x, persis_info, sim_specs, libE_info): particle_variance = sim_specs['user'].get('particle_variance', 0) # Composing variable names and x values to set up simulation - # arguments = [] - # sim_dir = [simdir_basename] - # for i,key in enumerate(keys): - # variable = key+'='+str(x[i]) - # arguments.append(variable) - # sim_dir.append('_'+variable) - # print(os.getcwd(), sim_dir) - - # For one key seed = int(np.rint(x[0][0])) # This is to give a random variance of work-load @@ -91,7 +82,7 @@ def run_forces(x, persis_info, sim_specs, libE_info): job = jobctl.launch(calc_type='sim', app_args=args, stdout='out.txt', stderr='err.txt', wait_on_run=True) # Auto-partition # Stat file to check for bad runs - statfile = simdir_basename+'.stat' + statfile = 'forces.stat' filepath = os.path.join(job.workdir, statfile) line = None @@ -112,6 +103,7 @@ def run_forces(x, persis_info, sim_specs, libE_info): print("Job {} completed".format(job.name)) calc_status = WORKER_DONE if read_last_line(filepath) == "kill": + # Generally mark as complete if want results (completed after poll - before readline) print("Warning: Job completed although marked as a bad run (kill flag set in forces.stat)") elif job.state == 'FAILED': print("Warning: Job {} failed: Error code {}".format(job.name, job.errcode)) @@ -130,7 +122,7 @@ def run_forces(x, persis_info, sim_specs, libE_info): # job.read_file_in_workdir(statfile) final_energy = data[-1] except Exception as e: - print('Caught:', e) + # print('Caught:', e) final_energy = np.nan # print('Warning - Energy Nan') diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 27ab74cca..663b57718 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -7,30 +7,34 @@ # Import libEnsemble modules from libensemble.libE import libE from libensemble.gen_funcs.sampling import uniform_random_sample +from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams from libensemble import libE_logger -libE_logger.set_level('INFO') +libE_logger.set_level('INFO') # Info is now default - but shows usage. USE_BALSAM = False -USE_MPI = False - -if USE_MPI: - # Run with MPI (Add a proc for manager): mpiexec -np python run_libe_forces.py - import mpi4py - mpi4py.rc.recv_mprobe = False # Disable matching probes - from mpi4py import MPI - nworkers = MPI.COMM_WORLD.Get_size() - 1 - is_master = (MPI.COMM_WORLD.Get_rank() == 0) - libE_specs = {} # MPI is default comms, workers are decided at launch -else: - # Run with multi-processing: python run_libe_forces.py - try: - nworkers = int(sys.argv[1]) - except Exception: - print("WARNING: nworkers not passed to script - defaulting to 4") - nworkers = 4 - is_master = True # processes are forked in libE - libE_specs = {'nworkers': nworkers, 'comms': 'local'} +#USE_MPI = False + +#if USE_MPI: + ## Run with MPI (Add a proc for manager): mpiexec -np python run_libe_forces.py + #import mpi4py + #mpi4py.rc.recv_mprobe = False # Disable matching probes + #from mpi4py import MPI + #nworkers = MPI.COMM_WORLD.Get_size() - 1 + #is_master = (MPI.COMM_WORLD.Get_rank() == 0) + #libE_specs = {} # MPI is default comms, workers are decided at launch +#else: + ## Run with multi-processing: python run_libe_forces.py + #try: + #nworkers = int(sys.argv[1]) + #except Exception: + #print("WARNING: nworkers not passed to script - defaulting to 4") + #nworkers = 4 + #is_master = True # processes are forked in libE + #libE_specs = {'nworkers': nworkers, 'comms': 'local'} + + +nworkers, is_master, libE_specs, _ = parse_args() if is_master: print('\nRunning with {} workers\n'.format(nworkers)) @@ -48,7 +52,7 @@ subprocess.check_call(['./build_forces.sh']) # Normally the sim_dir will exist with common input which is copied for each worker. Here it starts empty. -# Create if no ./sim dir. See sim_specs['sim_dir'] +# Create if no ./sim dir. See libE_specs['sim_dir'] if not os.path.isdir('./sim'): os.mkdir('./sim') @@ -61,16 +65,13 @@ jobctrl = MPIJobController() # Use auto_resources=False to oversubscribe jobctrl.register_calc(full_path=sim_app, calc_type='sim') - -# Todo - clarify difference sim 'in' and 'keys' # Note: Attributes such as kill_rate are to control forces tests, this would not be a typical parameter. # State the objective function, its arguments, output, and necessary parameters (and their sizes) sim_specs = {'sim_f': run_forces, # Function whose output is being minimized 'in': ['x'], # Name of input for sim_f 'out': [('energy', float)], # Name, type of output from sim_f - 'user': {'sim_dir_suffix': 'test', - 'simdir_basename': 'force', + 'user': {'simdir_basename': 'forces', 'keys': ['seed'], 'cores': 2, 'sim_particles': 1e3, @@ -95,23 +96,18 @@ libE_specs['save_every_k_gens'] = 1000 # Save every K steps libE_specs['sim_dir'] = './sim' # Sim dir to be copied for each worker -libE_specs['profile'] = False # Don't have libE profile run +libE_specs['profile_worker'] = False # Whether to have libE profile on # Maximum number of simulations sim_max = 8 exit_criteria = {'sim_max': sim_max} -# Create a different random number stream for each worker (used by uniform_random_sample) -np.random.seed(1) +# Create a different random number stream for each worker and the manager persis_info = {} -for i in range(1, nworkers+1): - persis_info[i] = {'rand_stream': np.random.RandomState(i)} +persis_info = add_unique_random_streams(persis_info, nworkers + 1) H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info=persis_info, libE_specs=libE_specs) # Save results to numpy file if is_master: - short_name = script_name.split("test_", 1).pop() - filename = short_name + '_results_History_length=' + str(len(H)) + '_evals=' + str(sum(H['returned'])) - print("\n\nRun completed.\nSaving results to file: " + filename) - np.save(filename, H) + save_libE_output(H, persis_info, __file__, nworkers) From 16ea97bb62beadda164fdfd1f678348af70fde69 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 14:39:20 -0600 Subject: [PATCH 553/644] Differentiate internal functions in utils.py --- libensemble/libE.py | 4 +++- libensemble/utils.py | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/libensemble/libE.py b/libensemble/libE.py index 0634bb2f5..b87dd3fdb 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -241,7 +241,9 @@ def libE_mpi_worker(sim_specs, gen_specs, libE_specs): logger.debug("Worker {} exiting".format(libE_specs['comm'].Get_rank())) -# ==================== Process version ================================= +# ==================== Local version =============================== + + def start_proc_team(nworkers, sim_specs, gen_specs, libE_specs, log_comm=True): "Launch a process worker team." wcomms = [QCommProcess(worker_main, sim_specs, gen_specs, libE_specs, w, log_comm) diff --git a/libensemble/utils.py b/libensemble/utils.py index 5e9debbc8..f24d7683c 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -4,7 +4,7 @@ """ -__all__ = ['check_inputs'] +__all__ = ['check_inputs', 'parse_args', 'save_libE_output', 'add_unique_random_streams'] import os import sys @@ -73,7 +73,7 @@ '\n' + 79*'*' + '\n\n') -def check_consistent_field(name, field0, field1): +def _check_consistent_field(name, field0, field1): "Check that new field (field1) is compatible with an old field (field0)." assert field0.ndim == field1.ndim, \ "H0 and H have different ndim for field {}".format(name) @@ -167,7 +167,7 @@ def check_H(H0, sim_specs, alloc_specs, gen_specs): # Check dimensional compatibility of fields for field in fields: - check_consistent_field(field, H0[field], Dummy_H[field]) + _check_consistent_field(field, H0[field], Dummy_H[field]) def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=None, exit_criteria=None, H0=None, serial_check=False): @@ -245,7 +245,7 @@ def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=No help='Additional arguments for use by specific testers') -def mpi_parse_args(args): +def _mpi_parse_args(args): "Parse arguments for MPI comms." from mpi4py import MPI nworkers = MPI.COMM_WORLD.Get_size()-1 @@ -254,14 +254,14 @@ def mpi_parse_args(args): return nworkers, is_master, libE_specs, args.tester_args -def local_parse_args(args): +def _local_parse_args(args): "Parse arguments for forked processes using multiprocessing." nworkers = args.nworkers or 4 libE_specs = {'nworkers': nworkers, 'comms': 'local'} return nworkers, True, libE_specs, args.tester_args -def tcp_parse_args(args): +def _tcp_parse_args(args): "Parse arguments for local TCP connections" nworkers = args.nworkers or 4 cmd = [ @@ -273,7 +273,7 @@ def tcp_parse_args(args): return nworkers, True, libE_specs, args.tester_args -def ssh_parse_args(args): +def _ssh_parse_args(args): "Parse arguments for SSH with reverse tunnel." nworkers = len(args.workers) worker_pwd = args.worker_pwd or os.getcwd() @@ -296,7 +296,7 @@ def ssh_parse_args(args): return nworkers, True, libE_specs, args.tester_args -def client_parse_args(args): +def _client_parse_args(args): "Parse arguments for a TCP client." nworkers = args.nworkers or 4 ip, port, authkey = args.server @@ -313,11 +313,11 @@ def parse_args(): "Unified parsing interface for regression test arguments" args = parser.parse_args(sys.argv[1:]) front_ends = { - 'mpi': mpi_parse_args, - 'local': local_parse_args, - 'tcp': tcp_parse_args, - 'ssh': ssh_parse_args, - 'client': client_parse_args} + 'mpi': _mpi_parse_args, + 'local': _local_parse_args, + 'tcp': _tcp_parse_args, + 'ssh': _ssh_parse_args, + 'client': _client_parse_args} if args.pwd is not None: os.chdir(args.pwd) return front_ends[args.comms or 'mpi'](args) From cccf946f0b186e8c7e7c16329157bb0c3d34298e Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 14:47:21 -0600 Subject: [PATCH 554/644] Cleanup forces changes --- .../tests/scaling_tests/forces/forces_simf.py | 5 ++--- .../scaling_tests/forces/run_libe_forces.py | 21 ------------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/forces_simf.py b/libensemble/tests/scaling_tests/forces/forces_simf.py index fcd9ff85e..e33a81896 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simf.py +++ b/libensemble/tests/scaling_tests/forces/forces_simf.py @@ -44,7 +44,7 @@ def run_forces(H, persis_info, sim_specs, libE_info): # output = what will be returned to libE calc_status = 0 # Returns to worker - + x = H['x'] simdir_basename = sim_specs['user']['simdir_basename'] keys = sim_specs['user']['keys'] @@ -121,8 +121,7 @@ def run_forces(H, persis_info, sim_specs, libE_info): data = np.loadtxt(filepath) # job.read_file_in_workdir(statfile) final_energy = data[-1] - except Exception as e: - # print('Caught:', e) + except Exception: final_energy = np.nan # print('Warning - Energy Nan') diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 663b57718..578ff0051 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import sys import os import numpy as np from forces_simf import run_forces # Sim func from current dir @@ -13,26 +12,6 @@ libE_logger.set_level('INFO') # Info is now default - but shows usage. USE_BALSAM = False -#USE_MPI = False - -#if USE_MPI: - ## Run with MPI (Add a proc for manager): mpiexec -np python run_libe_forces.py - #import mpi4py - #mpi4py.rc.recv_mprobe = False # Disable matching probes - #from mpi4py import MPI - #nworkers = MPI.COMM_WORLD.Get_size() - 1 - #is_master = (MPI.COMM_WORLD.Get_rank() == 0) - #libE_specs = {} # MPI is default comms, workers are decided at launch -#else: - ## Run with multi-processing: python run_libe_forces.py - #try: - #nworkers = int(sys.argv[1]) - #except Exception: - #print("WARNING: nworkers not passed to script - defaulting to 4") - #nworkers = 4 - #is_master = True # processes are forked in libE - #libE_specs = {'nworkers': nworkers, 'comms': 'local'} - nworkers, is_master, libE_specs, _ = parse_args() From 99b81425d865d6ef2dfd443b3791b5c907841350 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 16:13:08 -0600 Subject: [PATCH 555/644] Add utilities API --- docs/utilities.rst | 9 ++++++- libensemble/utils.py | 62 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/docs/utilities.rst b/docs/utilities.rst index 53f2ae749..3a04cb524 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -1,7 +1,7 @@ Utilities ========= -libEnsemble features several modules and tools to assist in writing consistent +libEnsemble features a utilities module to assist in writing consistent calling scripts and user functions. Input consistency @@ -40,3 +40,10 @@ Usage: [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] [--worker_python [WORKER_PYTHON]] [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] + +Utilities API +------------- + +.. automodule:: utils + :members: + :no-undoc-members: diff --git a/libensemble/utils.py b/libensemble/utils.py index f24d7683c..59624aca7 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -1,6 +1,6 @@ """ libEnsemble utilities -============================================ +===================== """ @@ -310,7 +310,11 @@ def _client_parse_args(args): def parse_args(): - "Unified parsing interface for regression test arguments" + """Parses command line arguments. + + :doc:`(See usage)` + + """ args = parser.parse_args(sys.argv[1:]) front_ends = { 'mpi': _mpi_parse_args, @@ -326,6 +330,35 @@ def parse_args(): def save_libE_output(H, persis_info, calling_file, nworkers): + """ + Writes out history array and persis_info to files. + + Format: _results_History_length=_evals=_ranks= + + Parameters + ---------- + + H: `NumPy structured array `_ + + History array storing rows for each point. + :doc:`(example)` + + persis_info: :obj:`dict` + + Persistent information dictionary + :doc:`(example)` + + calling_file : :obj:`string` + + Name of user calling script (or user chosen name) to prefix output files. + The convention is to send __file__ from user calling script. + + nworkers: :obj:`int` + + The number of workers in this ensemble. Added to output file names. + + """ + script_name = os.path.splitext(os.path.basename(calling_file))[0] short_name = script_name.split("test_", 1).pop() filename = short_name + '_results_History_length=' + str(len(H)) \ @@ -341,10 +374,27 @@ def save_libE_output(H, persis_info, calling_file, nworkers): # ===================== per-worker numpy random-streams ======================== -def add_unique_random_streams(persis_info, size): - # Creates size random number streams for the libE manager and workers when - # size is num_workers + 1. Stream i is initialized with seed i. - for i in range(size): +def add_unique_random_streams(persis_info, nstreams): + """Creates nstreams random number streams for the libE manager and workers + when nstreams is num_workers + 1. Stream i is initialized with seed i. + + The entries are appended to the existing persis_info dictionary. + + Parameters + ---------- + + persis_info: :obj:`dict` + + Persistent information dictionary + :doc:`(example)` + + nstreams: :obj:`int` + + Number of independent random number streams to produce + + """ + + for i in range(nstreams): if i in persis_info: persis_info[i].update({ 'rand_stream': np.random.RandomState(i), From afd0bbe3839bc568bbeeaf18e05890d1c8104054 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 16:38:55 -0600 Subject: [PATCH 556/644] Add output analysis --- docs/history_output.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/history_output.rst b/docs/history_output.rst index d6146fda4..b863cc797 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -53,3 +53,13 @@ Other libEnsemble files produced by default are: set to DEBUG. If this file is not removed, multiple runs will append output. Messages at or above level MANAGER_WARNING are also copied to stderr to alert the user promptly. For more info, see :doc:`Logging`. + +Output Analysis +^^^^^^^^^^^^^^^ +The ``postproc_scripts`` directory, in the libEnsemble project root directory, +contains scripts to compare outputs and create plots based on the ensemble output. + +.. include:: ../postproc_scripts/readme.rst + +.. include:: ../postproc_scripts/balsam/readme.rst + From 5173d57ecef62224580f7bf6b24badcc619ab90f Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 16:42:45 -0600 Subject: [PATCH 557/644] Slight mod to FAQ wording --- docs/FAQ.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 135b41e45..cf27eac86 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -126,9 +126,9 @@ to ``pdb``. How well this works varies by system:: **Can I use the MPI Job Controller when running libEnsemble with multiprocessing?** -Actually, yes! The job controller type only determines how launched jobs communicate -with libEnsemble, and is independent of ``comms`` chosen for manager-worker -communications. +Actually, yes! The job controller type only determines how libEnsemble workers +launch and interact with user applications, and is independent of ``comms`` chosen +for manager-worker communications. macOS-specific Errors --------------------- From 24c10296c96349003cc49c7b89cf1efe12044db1 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 16:49:45 -0600 Subject: [PATCH 558/644] Update parse_args location in Summit guide --- docs/platforms/summit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 6108b1c73..212aeb600 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -81,7 +81,7 @@ Batch Runs Batch scripts specify run-settings using ``#BSUB`` statements. The following simple example depicts configuring and launching libEnsemble to a launch node with multiprocessing. This script also assumes the user is using the ``parse_args()`` -convenience function within libEnsemble's ``/regression_tests/common.py``. +convenience function within libEnsemble's ``utils.py``. .. code-block:: bash From 4e9a1cf15a12a8c03832e195895d264d003fd4d9 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 18 Nov 2019 17:03:11 -0600 Subject: [PATCH 559/644] Update forces submission scripts to use parse_args cmd line --- .../tests/scaling_tests/forces/summit_submit_mproc.sh | 8 +++++--- .../tests/scaling_tests/forces/theta_submit_balsam.sh | 3 ++- .../tests/scaling_tests/forces/theta_submit_mproc.sh | 7 +++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh index 2279b50dd..42e9da68c 100644 --- a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh @@ -15,8 +15,11 @@ # Name of calling script- export EXE=run_libe_forces.py +# Communication Method +export COMMS="--comms local" + # Number of workers. -#export NUM_WORKERS=4 # Optional if pass to script +export NWORKERS="--nworkers 4" # Wallclock for libE. Slightly smaller than job wallclock #export LIBE_WALLCLOCK=15 # Optional if pass to script @@ -40,8 +43,7 @@ hash -r # Check no commands hashed (pip/python...) # Launch libE. #python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 -#python $EXE $NUM_WORKERS > out.txt 2>&1 -python $EXE > out.txt 2>&1 +python $EXE $COMMS $NWORKERS > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index d51a5692f..3f071a9dc 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -25,7 +25,8 @@ export LIBE_WALLCLOCK=25 export WORKFLOW_NAME=libe_workflow #sh - todo - may currently be hardcoded to this in libE - allow user to specify #Tell libE manager to stop workers, dump timing.dat and exit after this time. Script must be set up to receive as argument. -export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) +export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS" +# export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) # export SCRIPT_ARGS='' #Default No args # Name of Conda environment (Need to have set up: https://balsam.alcf.anl.gov/quick/quickstart.html) diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh index 6454e4f7b..d46dc6a5d 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh @@ -14,8 +14,11 @@ # Name of calling script export EXE=run_libe_forces.py +# Communication Method +export COMMS="--comms local" + # Number of workers. -#export NUM_WORKERS=4 # Optional if pass to script +export NWORKERS="--nworkers 4" # Wallclock for libE (allow clean shutdown) #export LIBE_WALLCLOCK=25 # Optional if pass to script @@ -43,7 +46,7 @@ export PYTHONNOUSERSITE=1 # Launch libE #python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 #python $EXE $NUM_WORKERS > out.txt 2>&1 -python $EXE > out.txt 2>&1 +python $EXE $COMMS $NWORKERS > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py From 784b0e6df91c22bcafc374f60204b187bfe844c0 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 07:41:28 -0600 Subject: [PATCH 560/644] Whitespace removal --- docs/history_output.rst | 1 - docs/platforms/summit.rst | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/history_output.rst b/docs/history_output.rst index b863cc797..64cff7b55 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -62,4 +62,3 @@ contains scripts to compare outputs and create plots based on the ensemble outpu .. include:: ../postproc_scripts/readme.rst .. include:: ../postproc_scripts/balsam/readme.rst - diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 212aeb600..e2307bd3a 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -2,7 +2,7 @@ Summit ====== -Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing Facility. +Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing Facility. Each of the approximately 4,600 compute nodes on Summit contains two IBM POWER9 processors and six NVIDIA Volta V100 accelerators. Summit features three tiers of nodes: login, launch, and compute nodes. @@ -19,12 +19,12 @@ Begin by loading the Python 3 Anaconda module:: You can now create your own custom Conda_ environment:: conda create --name myenv python=3.7 - + Now activate environment:: export PYTHONNOUSERSITE=1 # Make sure get python from conda env . activate myenv - + If you are installing any packages with extensions, ensure the correct compiler module is loaded. If using mpi4py_, this must be installed from source, referencing the compiler. At time of writing, mpi4py must be built with gcc:: @@ -35,7 +35,6 @@ With your environment activated:: CC=mpicc MPICC=mpicc pip install mpi4py --no-binary mpi4py - Installing libEnsemble ---------------------- @@ -48,7 +47,6 @@ Your prompt should be similar to the following line: .. note:: If you encounter pip errors, run ``python -m pip install --upgrade pip`` first - Job Submission -------------- @@ -114,7 +112,7 @@ convenience function within libEnsemble's ``utils.py``. export COMMS='--comms local' # Number of workers. - export NWORKERS='--nworkers 128' + export NWORKERS='--nworkers 128' hash -r # Check no commands hashed (pip/python...) From b21dfd9d43c712d05e8089e7feee3f7903d065e9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 10:02:36 -0600 Subject: [PATCH 561/644] Move API to top of utilities docs, examples beneath with redundant info removed --- docs/utilities.rst | 50 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/docs/utilities.rst b/docs/utilities.rst index 3a04cb524..ab06faf0b 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -4,29 +4,34 @@ Utilities libEnsemble features a utilities module to assist in writing consistent calling scripts and user functions. -Input consistency ------------------ -Users can check the formatting and consistency of ``exit_criteria`` and each -``specs`` dictionary with the ``check_inputs()`` function from the ``utils`` -module. Provide any combination of these data structures as keyword arguments. -For example:: +Utilities API +------------- - from libensemble.utils import check_inputs - check_inputs(sim_specs=my-sim_specs, gen_specs=my-gen_specs, exit_criteria=ec) +.. automodule:: utils + :members: + :no-undoc-members: -Parameters as command-line arguments ------------------------------------- +Examples +-------- -The ``parse_args()`` function can be used to pass common libEnsemble parameters as -command-line arguments. +Check inputs +~~~~~~~~~~~~ -In your calling script:: +.. code-block:: python + + from libensemble.utils import check_inputs + check_inputs(sim_specs=my_sim_specs, gen_specs=my_gen_specs, exit_criteria=ec) + +Parse Args +~~~~~~~~~~ + +.. code-block:: python from libensemble.utils import parse_args nworkers, is_master, libE_specs, misc_args = parse_args() -From the shell, for example:: +From the shell:: $ python calling_script --comms local --nworkers 4 @@ -41,9 +46,16 @@ Usage: [--worker_python [WORKER_PYTHON]] [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] -Utilities API -------------- +Save libE Output +~~~~~~~~~~~~~~~~ -.. automodule:: utils - :members: - :no-undoc-members: +.. code-block:: python + + save_libE_output(H, persis_info, __file__, nworkers) + +Add Unique Random Streams +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + persis_info = add_unique_random_streams(old_persis_info, nworkers + 1) From 0eeaef3a9fcb5e1f4c61e922fefa5b5b64182cab Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 10:21:48 -0600 Subject: [PATCH 562/644] add returns info to parse_args --- libensemble/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 59624aca7..3e5beeef1 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -312,7 +312,18 @@ def _client_parse_args(args): def parse_args(): """Parses command line arguments. - :doc:`(See usage)` + Returns + ------- + + nworkers: :obj:`int` + Number of workers libEnsemble will inititate + + is_master: :obj:`boolean` + Indicate if the current process is the manager process + + libE_specs: :obj:`dict` + Settings and specifications for libEnsemble + :doc:`(example)` """ args = parser.parse_args(sys.argv[1:]) From 22a17119523a3048992a40ba57543bf411aa35ac Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 11:25:10 -0600 Subject: [PATCH 563/644] move utilities to programming --- docs/index.rst | 1 - docs/programming_libE.rst | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index fd47b94a3..ca3837201 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,6 @@ Quickstart overview_usecases programming_libE - utilities platforms/platforms_index .. toctree:: diff --git a/docs/programming_libE.rst b/docs/programming_libE.rst index 4c99ee5ba..3fd34e612 100644 --- a/docs/programming_libE.rst +++ b/docs/programming_libE.rst @@ -14,3 +14,6 @@ Programming with libEnsemble .. toctree:: job_controller/jc_index logging + +.. toctree:: + utilities From 0b8af7da45d7bcb2d57db3ac5a7e3e6982545bb7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 11:38:21 -0600 Subject: [PATCH 564/644] moves examples under each autodoc section --- docs/utilities.rst | 52 -------------------------------------------- libensemble/utils.py | 45 +++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 60 deletions(-) diff --git a/docs/utilities.rst b/docs/utilities.rst index ab06faf0b..00f5fe519 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -4,58 +4,6 @@ Utilities libEnsemble features a utilities module to assist in writing consistent calling scripts and user functions. - -Utilities API -------------- - .. automodule:: utils :members: :no-undoc-members: - -Examples --------- - -Check inputs -~~~~~~~~~~~~ - -.. code-block:: python - - from libensemble.utils import check_inputs - check_inputs(sim_specs=my_sim_specs, gen_specs=my_gen_specs, exit_criteria=ec) - -Parse Args -~~~~~~~~~~ - -.. code-block:: python - - from libensemble.utils import parse_args - nworkers, is_master, libE_specs, misc_args = parse_args() - -From the shell:: - - $ python calling_script --comms local --nworkers 4 - -Usage: - -.. code-block:: bash - - usage: test_... [-h] [--comms [{local,tcp,ssh,client,mpi}]] - [--nworkers [NWORKERS]] [--workers WORKERS [WORKERS ...]] - [--workerID [WORKERID]] [--server SERVER SERVER SERVER] - [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] - [--worker_python [WORKER_PYTHON]] - [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] - -Save libE Output -~~~~~~~~~~~~~~~~ - -.. code-block:: python - - save_libE_output(H, persis_info, __file__, nworkers) - -Add Unique Random Streams -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - persis_info = add_unique_random_streams(old_persis_info, nworkers + 1) diff --git a/libensemble/utils.py b/libensemble/utils.py index 3e5beeef1..836cddec4 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -1,9 +1,3 @@ -""" -libEnsemble utilities -===================== - -""" - __all__ = ['check_inputs', 'parse_args', 'save_libE_output', 'add_unique_random_streams'] import os @@ -176,6 +170,11 @@ def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, gen_specs=No sufficient information to perform a run. There is no return value. An exception is raised if any of the checks fail. + .. code-block:: python + + from libensemble.utils import check_inputs + check_inputs(sim_specs=my_sim_specs, gen_specs=my_gen_specs, exit_criteria=ec) + Parameters ---------- @@ -310,7 +309,28 @@ def _client_parse_args(args): def parse_args(): - """Parses command line arguments. + """ + Parses command line arguments. + + .. code-block:: python + + from libensemble.utils import parse_args + nworkers, is_master, libE_specs, misc_args = parse_args() + + From the shell:: + + $ python calling_script --comms local --nworkers 4 + + Usage: + + .. code-block:: bash + + usage: test_... [-h] [--comms [{local,tcp,ssh,client,mpi}]] + [--nworkers [NWORKERS]] [--workers WORKERS [WORKERS ...]] + [--workerID [WORKERID]] [--server SERVER SERVER SERVER] + [--pwd [PWD]] [--worker_pwd [WORKER_PWD]] + [--worker_python [WORKER_PYTHON]] + [--tester_args [TESTER_ARGS [TESTER_ARGS ...]]] Returns ------- @@ -346,6 +366,10 @@ def save_libE_output(H, persis_info, calling_file, nworkers): Format: _results_History_length=_evals=_ranks= + .. code-block:: python + + save_libE_output(H, persis_info, __file__, nworkers) + Parameters ---------- @@ -386,11 +410,16 @@ def save_libE_output(H, persis_info, calling_file, nworkers): def add_unique_random_streams(persis_info, nstreams): - """Creates nstreams random number streams for the libE manager and workers + """ + Creates nstreams random number streams for the libE manager and workers when nstreams is num_workers + 1. Stream i is initialized with seed i. The entries are appended to the existing persis_info dictionary. + .. code-block:: python + + persis_info = add_unique_random_streams(old_persis_info, nworkers + 1) + Parameters ---------- From e07e3f0af28cbac78477464994f86aa6e6073eb6 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 11:43:27 -0600 Subject: [PATCH 565/644] Adding image --- docs/images/libE_logo_only.png | Bin 0 -> 41030 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/images/libE_logo_only.png diff --git a/docs/images/libE_logo_only.png b/docs/images/libE_logo_only.png new file mode 100644 index 0000000000000000000000000000000000000000..174935a1de36dbeac86d807eb6f64c25c5bcd473 GIT binary patch literal 41030 zcmeFYg;!K<_dh%{3^4QzjpWcF=}ij^Lx+fTN-9WqNi&qxPy&K72qH)$DIL-Zh=hQY zlp>AN@E-1OJ!`%H!f&})E-%j6*RIdbW4x}88X55|Vh{*KrlGE^2LeGbz>f)x0600< zI%Wj^BCyp`QwCk%{K2)oO$SbhJk(9SK_DTHn;&q%8wEe$kkCito(ka)5-J2Qvt~bd z1_WXQX(%fi1b~0H-AkSum=gST=5aQ~pjfMleTU%O(qS{~#D93_C)mPd@@ZSq(?$md ze^7BQ-AQ;F&z^P)ZWX~41m#NUbgwAaeXjRB;~BG}n0h7IZ%&oIt9Y8OxQ}daE`FXL z?+SCGQ7{-7$p*!Q!{Lw#m8FhibKgme|2RX3!{XrYSpV-KwDk&bqSK03a`vAuBMn&L zkpDgC>_xzlc>#pFf1+;)|Mxk%ME<{Tx;co5=U_ui?UUnvk^CX?BND)uqgl7S45b{4t0|)>LqWFIb`Y%oX zKL!0i5Bd*?T~k8;%P+7Jkn7RwZWuy`d^N zy*vQ>9(Se5o0tS<2eC^j%?fN<#{TTE(T4Z9#){Ht-u!|M2v*^$dd~nti6r3TWb0_N zj{*PuCLw3#5!;cH0VEg|FTX9Yf|fV#yA~XDlWbV?$oqdb}l-&xq|`_aKFCPi4%~|7A7TT!cno+ zt7j*}9!(_iOpg*`$$?Yrv*f4@03OyRj_4g^n_D55HcEG*HZ%hGWrf3$7hp=}e}k{G zO86qDj^`tfjx^5bwJY#85OGBWp+^G&X4@6B#9-tY62fMBvC=n3t`rkG z-k1SuV}k~%;1cOyUYpwAlMC*k=yi?x(L4b$uiX2HrPMi~PhM35h}y8>{9&=5$*AFKvg5dSr}4^kcMOc`7}jbB6g7Q zB-=`pfFZ>S8u7d?Dl{TS6Y&f67tkdsA-Dog{F@#3tkHQ2tV(!=HXt-T9B?o^r`vD% z0aG5pFz^!LAKhR1h?d5XdY~L6*pyH&`J7%DDwb#$a&)h)c?g z5B+V!g*?Kt*;Yo{ZvgM7mjNg$xht*#CPMMUM(`+?3s>%iUux)|RCuHvJ(_C?q3(qHrZV5)U_GlD4z$*6_xaN%~b#cJad9;Gl?XbNL zqUy{WtK%sG+_I6q`3K-ooh3qstw*f$Z+WSC$nDRxGGTg{y}R-<0#MZmpzPp%@gVbt z4|gc!dTxLw&U{>tS@R_1$Ke_vf-p};{1pRg zA#1i9Z5I_}6a(a=P;43DmW9uKdHH3T-1l^?xk*Q@|R3BMu4>2b5GqeM6oA z!lp|mc1fQts9}Srmt}s@=|lT`T;ls+mPfwthtR_l|o%n7HvPl|2!~UH>l_TdI z>ecU0Ijq%Q9a)P%lt!AS+lS?kp*jVGx2FF$ZXes&zq<`waic%bYPssWscTZN^5<5b z*c5lhO@a^U(taf*j+@VxKJzs0qR?6AgmpI&rU@}dyxcTrTz`WBR;YUMRU-Wvar;BH zBmd8nl6kjBX~v`h@92vSUqCWh<-St|jC_;`GMbi9U$z{5V$ySn_%r2nTNZdGCO+H< z7W>U@$PBRS5rQ{$1u`_*Rxsdz?V8k0V>WwreU*RBqx4zYr}5(sMHOvumnh{y-;>Mf zu-59cCA; z3VH%I-U&4TtgHoN$v5}lQWceYRCE?w-HDnSH8>>3k2sNS-8>_}}{DgA%ByTD6W*Nx2egG-k#*7QTSb%@%cs z+8HM-%==0+Rgx|G5&s}>iVtG!T1=lx63Gm5hWKA) zIwHUopn8~)g>mZl!`x2sjO~VKi7{y(f=>ma?xb!g$J0e^{*z2tL?OUEOSB5%zZQ;C z3NJ!`fj~I*Js7v%Mx0b`5U;v~a8CBy#pNsqgDuf^cZ-FV-$nvbKLAqy3GZzg-hVO9 z0zPiRd?z&O-GH<4k;f(b5RWSIik5tS0EZz*(uukSZh zIB^5ut%J{KmwovDm6p>fS-l{|y`xjjj|g$MoW`X=m$KP7c7rsZBq~Xp8*ubx9i!~+esxRk$claSuB;SV>V}#rUOh>LYK$uPgcy;AM@JnKp<g*0z4>O{LO22jJ}iWq#*$F8uqcodRw_^0gAZMFvdpm zY6Su=F9;1nXsn-3Q7gY!1#j^$YqhBb z&@K$VqDJPqOq8tv;fXU2^?2lob6xtkKxjE;&pe?lswJ`Pd$NslMHVtcJLX0kx@MYz zK>B5a>Tpy`hiE!L;MokW)2gzp?`PlcDSrFC^j@Agf)40V|E-@ReBjlTSRW^mO6~Fv zg_$}sJR>$GTYJ|Hp6ebZAi4**7AM|BcVUPTMC_q*jOqJIBbSO@1UZ%AqwoBHf-1TN zHUQ3G5jyY^+4$FNS?twqb+{p-2m_$N(|*I15erC|t_r&DH%Uq8pTMh4e)n2_0x_l! z<3LMA8yPX*P=`Pck(|;sOsDy(x6@1$NhN|lc7k4LCj*cUV4`CN3Jjp!Ad2{eO=H%r zG_p8s*os@EByEl;nfrYIDmE67dT&;n;oolm7296eb^rB(yfluY241}qG|`o!Kv%=h zr-1&Cfx{OR+kFnQM>VnGzR_5r=TAdtXDm%#ez^9r70>#vepA{%>3L@c+^S6wQI%f$ zcC$>u~^JNvE9z1_EqCsc8L5okg<;8Ln1l{yXO4ZiVYx;kp~69Xh&fs`sp7yyCuUXR-1r(}!<- zM*lPmODCb?^$RfA1$W#vcrP&e_pZ$ITQ1GS+$fhTiiPF%Q|;2xu+4fEH(R}9#ESq2*am_FU{aB z8vu!uVH&#FLnY)p(4BFA{wn)ijSWq|DwZ%C(uJE zn#1zA2H-?zD9|d0(mc}#jtJoz1?q1;IlKd=-ygIxD+VY1wNY;;G*+6{H#(CYBCU{XW9{NVTMooUn$7GG~S5!c5U zvBfgF^xwwk@0V$~%$6jGxhTz*iQzbr;A;g)X*PEv!_2@*K^ebr%k$|%ep=RYz?pkBQVMM0!fg>YJ%SXM$S z$l!V!I(5h;0X%+c1($}e+R{-kH?6^q)NrjX_P)@aR zNDyIWo+_I{2>-8~Km+&C&jB2J&!2^M2^Z~!^wj4kTxr_*dPHnCUOw`8V1L-r-!jlb zoOII@Jdmv7=Yf7u>#RW2#I<6I+h44ZjBUt81-a+9w2bCP{csI+ne|zC?M_){enb<> z@aA>##l1D~U4HK53klDp3S}688RS1*+#CQiKc`Y?*z6qr-3uPgm*t27%AvW=GcOhQ(54rJi@gCrwbNhJ zrTB}W!Q2I)?t4Q$)&z{{^gzsTMEb{njV$AGb|ixgY%k%{Zh8H~5lX@| zS}+~E@$_t7MilzQ?xld^0NI$k+2mGfEN{6afQ<$0ATVKJ1!Kb*oSGwxQP5`MreeR# z0l27n790(`8|oz2)dm_6@9AgKtj*~DWmDD*FTzfP&lkwP8p}tLQu4&Z>7f9jC`WEC zKIjL<8%@4`pk$^6!r`F=k5BnJLYUP;9>;y6>45|h1)^(8x+kSWIRa@W%e)C}2Xc!kgMRh> zWHur&Rb__~L<4C6_d;usI3fUMz4Cdv0^=w5#Jjt)B)d*>M#rpA6?V)>@oX? ztlzj#g|)0Ei63Yoq{?yAgvku)i~I{LOw(#e>|H=%gOaI~)DTaEhk%qoKwHArsi}MZ zgGkQmpoUcpF;33OU>ZG?Zh^cz=~J=+*&ueDPRaPTt4_-3Qb-(ULlk@bR3{9dq5N3h z*%(Gj0D!*hpG-;`AOTzJJ?z#ZlKaynaCbnT!h*~~X;^}L^=(1SDPK6e9k1ABRFUXc zE^-+%G~e7((OGsS!C>tLV0 z01L!?jMS|bR}eB7&t(|-BkBX)Kz?uaaA-5sZ43FMcs?4%2t=`lhyk5A8GsGx0K z>T)B8eM2YdIedrzJ5qEtD_;}aJt@@CfVOtJshlc48ZJIy508IeW~f+M?6Fd=Mr7Zg ziEdegoet9frK{`dbCl(MPaa=jL_#=uC&NV>aCy7ed{#GeFQ(EG_}ROUd+d>d;f!dQ zB0-oF@=cMYjI9ZBFqf8U%?ZhhtTInTPNUhRf!ETVDY-bILTVr?PFByh_WACXRbvwm zmFk0rjIQ@E+$Aw}(6vh!2Y#_S%wI%`{Zo5bo9tBg^C}fAy_@--VAc(lt4s@3O?99i zb^u(F%c9HC8NJF+)Nxx+J~xpV!-kEfzduQPS*!YeBUv|V#@){Mf;RN@-YWizYqc-} zs1Q&JvUhH}*uJ>%T&T2Sy$XRsr$4t?w$JV|$HgGYnIk@YeN0j?Lh-_Q^-m_-;8MyTC-Gir{-ERI+O%;F$Yzy!Fz@&J2R7I1 z`}VCb8~5^iUa}zjdkkJt0>!>a4Z#B$^b$QJJ5h|uN@>o7=IeboCcL&>yF<6`L%MuWFxVgqy=rvceiCH7LVY6bOV zw($1WzGdm7@2L|7v3@Do1YNaSI!GxHQL@bvcv+Fvu|PK;pAv?$+F?$vIj7M%>FEo$ z31#dl2=;jvKKq+={`bTf*Ru=ArB`vupRZ3Xax*7Q{LpMFK&vVB(ddCrx*90`6DeCc z&b3u|ATwQ0StQcT|g^VyLG*$#vQ?`xvR@pPf5uVZw;|q)Bvp-?)_ROvp#-2QX!z zED>SP_Tqw~5gt?|#^Sae!Y5$)jcod@F{VfPP{=(>Q&L`{qf4?3fgpGGx1Xk^oh({Z z>^r=8pfV~bOtpR!jW#S#weudYxMpv#)gI%;WlX%TLYC*PvxlD@4NK`}td`7p5vRty z#v^t_Ri^F6O^Iu|q5m-~jo^Sa#jncgeB_MEUl#6xHpx@!#h$5I(_+z7*0O>67F6P> zC8y=s0qebE;<5qxH655wWcXE9NU-!2%BUT|u^I za5G(5*QkcWEL%NWeNz+35feas2(CegtI#TQ9jD+c;?b3{zWcfMYa~Y9t6*{N|z%Oyi z^*dyS=cJHy#^3f{V2EQDo=?M?mX#NMmsnOF$@Mas4-nIQPz#Hnb(nWzIcKY?q3?FqC=ww^I~czO*rq~J zpdY=&3T0v__vEAzhEzynq#Y09ZiVdE`8SDi2V_7u*N~;Xzu$Ow#SmX;A8DJ=0 zO_fb!{|;7%BCv{p3E8dWurbV6Eo5sf{|Qy84`ToV7w4DBoG-nZWzpl6R2JTe$Baje z{}?C}D1R|6=(d> zLGvT`_8=65;dayCDUUQI72>Gzj40}xG*x4mF|mY*YZV(H;{BDS2CL5Hl^*o`$nI+@ zeefdamE~&VHoN*W(J?~WfvwLRlI;|8okGXe(x%i!9#QEGN5g~Yto$*}z^M`6lpmk#%OXMM<{ z5{dKAJTq0K4GP)e)M?la-PCT@vGEAx&1`dZN`Ge%86F=RQ#3VZVviMwy7&8;t(QgT zhO0}5;CV~T^`l|Yn5A|GbeiBQeUN3FYDbmrYP}kt0VODBhSad$+hd$*NxeY2$lDxNlCTqQd_%-Dp zvbJ&CFacbJW^X2X%Ky7lt(Ooq8Q9?lK&hE2fv5lyQ;!s<78`J)xu9apvk1fQ5)hQP zZj7`fzqGAceBbO{_VQ^gL47V6{~Sx&8ox;Qf*Q;cwy8SJlja!}Wxaq{(c)IAYGUFD6j0;ffiq73L3E<-dHs<`Z_6U1ghts# ztSalUBlsj2*cocwH0`q68XY}z--}^0o-UTRP zEK8GnYBPCYM0fi_d6JRIT?9AD1B~NDL`FbpW^kO)Alt+zJg?;TFKDm$OdO@Osj;a$ zormIFYpbdP+&Zn>m9lmis=-EhMrwr zp4a@4k)VV!1ulcM{=7t;s#g`;ikR8S27+JR3D;Fpvhv|+co~vDCKq{5B3f~9hkLru zDflya^Z4#$vzWu@{gA%1kyFn51+1oo3zAK=!<+&%NWSU&j7>odn>);BAH~MhsJ-ia z@zwkzJ~yR(zBc&^JxgtdOWiDe?xXL_Qzh47L0%|Qq)^?P_L%$R>NGy$`-<`O7;ECg zSpN+fjwn)C&UKsU<>eSHgU`2&nRdxXe~;fexYjYh{x-AqLA9`z)FoTE6stIe_v2Aw zCfc^PPip>86Jmy1jFl4D40)(m_$pS?i)aG-fIMrZcoK`d#DqoDW7lk|_@7utwe9)38UoJXc_1WlA{ z997}=h~qJ&OwhS(o6B$S{#|)1y}#-?r>S1Ig>;mm>uMkE0oVh;AKU+?3poUF(!B%@n=|%{q8zJcV z@ub(|Op<(>ZKt!an5%e;XYG`ZcZ7b4M_tpW(-;$+=Zzdncs^Tt%GlMvX(^>A`Yy=U zw^}*65>G%S%@BI#`28+B-|9JrDVGz?c9R>&alfa=Sr|x})$O=G=c}50^m>kEo#LbZ zVVn*7h51!vaHhhxD;X>^u1K&k5Oe<)`ceF37xz~Uy1zSp9tiq`veGRSE3i1d56owR zA^jrqzf%>-iTQ{+k}Te2=&B2qXo1hiCa3+gLGQ44*}q8uQ^k8>u)WylY8=1<$bg%m zblKhBND0#SovodCfqDpZmYGc`btE{Djs;1(5T7 z+OIjRrat&(JUq76k|r_PlYZ=VQqk4M59?#vfxD}|pwC2;*3MPE=|G(zl2a^|4^!8SrArmwzRgvPyi)q*tKr6%sZnv?w5HU0U>~X61hI;d zQvW#hT}lW`;m%p%l~5zFDpT8+`91DtwuZ7>L62hBFF|^8+WWJJZ#_RWLRBm8a2nxr z_g{;*kRKtwGYRl9O4aK~P46JAj#`%_qb6EF{l4L@R>h`P5$DMXf4QGDCt=k@SJgho z{_&T@_Hrwqq(Exngtadv0PXiUUOUl0Vfo}GjZ=IkWzK3@dyp?$oi(pv@GHF4Xy<`d zctT?`#9jA+jwaeyW~#;(3y{0-J1^or zu>AQ<8hdyi57{cPnIyKVtbAGn_KKAHAIq6wXreG=NK10gC@7e$t^Icx69t;{=d~)O zp6IOY-G{b^T%Du1S3}UR`o)4k(wl*lq*{7~$06O9ID5i-j_0PvwhQ!`$_fM=W4t+I zYvPb$&mQ~H3ulfBHS&w(w_mB%+@E#M?^YXHi0m~~OMc24nGaDtUWmC*hTb~-YEGE1 z*G@iL-p+A79U&T^WDY%}k35BF)K(fT-ya{T=&Q{Y0u)KqC@1ub^k;OrS&d)o+yy>o z6$VJxl!~TlJ*1BoE%mM<#fC|Sjtq;?OMrasl1)DaPPKk~@UN7AXm+LCEUUfay!pHL z)PbVMFB-j@FBs4cVx_AZrPDb0eRr_jOq+Q7lT2({*kh?|KyV08UmC$*l{ktSBj5{( zd^X_zCz3&kY?eW57POId3ns-lB5IIuZwR$=Uz^Lc=jd< zt9iy~0pm>T-x}Em5~^}KO*(%?Y8W$cZ4T-I%*dZzmy&5%8M}F|$4JQ^K|V}hKveZK zNbQzoI`GeJVEo(JAH_%TmRPof@T^<a@4+r$DDX*RZ`yq$sJ}^OK*6VVTVZu-Iem z$(jjCAp%B7n9tYJe@0o79mcqPq6+hI)7N?8yNKf<7dc%ei`Y4E@k92Fvd-3T3o5u2 zsN5J=*2vT~>mzY${mrer&NfD1Z^fK1da>$gjta zpkyqg78Pyawy74dQSzcylKQr&uXwhZLN$}zG{1);sr5?fF!%WWVYu}VEFOvqv$nKS zQ+F7ZI}Wxrjy&sZ8QUT?1u?RS9ldL>vu8ZYP{9g z;JqqgUwZ~hoUS?i)tzt7bupt??tH?9a7iFN_}*3fW?tV<>$My`m?01Ojefe{9cNUcXd7Fx2y=FbZAzDJb`<%XBIuCb-nAY?E5af0^&O0!(f{& zcrM;Z((_CXH`POjPwyPG^7z6d_+-65r0Wkv*e45^8ZrDTTCX>9PjhrYKvMBxJRM3iQ?q~5u)6N@7~CMLEp57={Ea=> zz`~4FzrL%eMcIv2?~}W*@Rh>a>B2_owgg_ONA%r_>FLU6`aIXNOs`x-8@zuL2l`mZ zdzMh3mu{%yp$M*0F>gqoKq8MMN?T-CTf*lFuS-r$zAF-wFj3?*NDlcs zbv;+f1A7`zU$9GbShM+h+k)_S2W-I7N`wgHPFxIZn4H#%3xB!DJfo7;W?0&@Q!A?i z^UdB@B&Q=bcN+~>1M z4N!ocBn^`3DM_P#hpDXF+~KpyF33Vf8J$Hv?y}@#1$FV24l-=lL;hEc)Wc?b_wF$k zCW?1Io;+n<3*#^iG}W#aQMZYAGS4U2kFl3loH1A<0Ww7^s0d7xyGR#M+~L z!mbgo7;wJn(JPycl{UQ~;3Cp7hap$jjsQKGdfH}DQt)9iOp9A;j#0Mdef4~h!`e}Irg`(HS`$LYh>UXf+;*_Jf zQh7A@DQ|&Nx{D)`LY8n`V6s?MzJPyM;AQU!KHK&)YwdS6?rrkTnkHxfRYX9by1%6U ztH;4kf~4?eis_F23GVi;&IVz>+4&~yK#UC)=ZcyM1=QL&1~I)LuK)W{LSTP3HC4in z^T!+_-2T9Z-7V9P8Oe7gnMNPVUL?j#$CO(xaC*V;!bP{ZIT~Qsa^L%GrDz$ozl#5(77`larP3_}?)v8g z>Gt^kZ5ha=OgVfZ$aP!n`q@jbW5u~)MCVMM%FoF)T7``AP^s5=>8eX_cNaVE-wHPn zpr|!qZ+vPOs$f^`nAqU(2|IlY-))rkKvpC2f#T_P7BJB#^7^#of>*C?HX`61CFI8? z3#}Hb>y-B;Z(CsQupS=l@cZCr&iXX_g3ZFx?8@VxE6^N&Oa4;!Q?=Zrgq4RINku^`LBQds(YC>k zpoH?yQKW8R=cRYFPF|JAT?Nn2!Sr|(Aqy{a49M|Q*Rq7QMP>I zNqL9hH1h8YM%PDv;h;04B&8{dasFlStE}K&Uu=N|QCn0WZSs>mJY&mb=b%^-zdL<@ z8p%ESS=o}Lm&CHl!h?gg8Dcq?g})Z;iOvg{?`)s%#LS%az3>Sn^ggM*!D{EF{4&1fayNveOXZmXYB zt42!O1zYijko_Hso!My~-+2R*^3t_1yV7nh2-!N(Hhh)m^eXt*t`ZIMNUf2dsVmch ziA|R!`0pvb?G}{-_K{+rUs+@r4w7fw@>~FVfw?;*ftLVph<9UJ6qV_e8?eL^6w)AC zTOT8Sq@Y+OT=S$;vOo`!u0oYAmT-+US{CCc)2V( z?$;_2dRn0>5G4rsmj}Pkr@g5{%tm?s{N6R zQ~FM#N2)QKG#C2;Ia1m2nx$Q;dU`Br&i0Uv;p%gN^gljJu?g0vLmwDhsR?plYBbM{ zH_$0>k9 zDcM>86W5$M2H)MzEgs58b&|OyP`wQy&!WG5hc&8qSR!PNkSXw?G4PQA zDiY)B8RT0$(5&CMLp0nhmM*=~hrLC!o|p5&<5lJNKMaDBh;B#Sh7A~)_GgIs=AzAsf54fF}Y!8Nk?>TYM(0Pq%9wSWEm^^=52ezkG|0MaB|K7!$OFP_+|U z?y!!9mch!GUfPwYXeNx$xG;^I6+WCgEh14@C;m1yu8a}6l~Ezf5AO;}ZG2_13Uc;; zyS3G@C+@MNjh`XMC1hiQ|e`1E$?H;K+EmuDKD;2-Aiqryn(YFJ$_ zf0#Xug+VJR7n*qlW8L2V4NM|V_jn`Z-=s5aqVWvghpv63kX%hI7#sidmB5QBclD}Z zsxCI8&b|+6%gm#a3Yi^DU^SL!!-6``gA=;ELJxkUbe3ipeO~f=48z|)gBxEbaYEt< z=$IzTNnlz^t>f8Jk-m3u#QO={fr-H?Fgx5#Dup(wGbU)_K@(QQ;=3u^BI(>Bj&+KR^B{a@oN7nHEM)cL7Sh9PZbD!58ogUGm5Cw=Zwy4WgT3SuN!EN z5GuIC6oxM+G44%ix;tmtWMyHpQ=M%@NP*DUJ%sSpSYcqSr6WpAM~>&s!!zV>8xgh# z9jPwtC#fVH5GHRMr`>LbFl@>=+gEy1LY%Ud6IvF>so}T5({>7imu2p7!j3ifq zDUEH7)6zYS*y}jX?zVB%u}~XavQb=ps3@&?4Ot8S^C@%xN-X{<`0Vqyb2ok~yv$u% zF!mV%Vi@+I@U9-w0EZ)vJZ{U`?DQ5B-!;Q_gN&C&><2a6KQXO+Gee?5ntH15Zg1R4 z>kRuLvqbkUz}sN==PWZvyFyFt43xxb-y|jgfh1L|axlm8lYi(?^vArbN?br@B@8Tu zMpl1AkuVM8I+V0Zs;JuuDqH`=RwXUlKUM)n@tmKkm9@S9A%On~mjd@DztSKvqKmuZ zNsLHVNdS81q*zejG{J&AqnuEZU(}t1CYcj(<|xzhoID|PNc?t!7iY|r-z)`HohC23 ziDC_O?AcY7>PJBKQbe%+nz+y^RcWjfd@ksN?@Fp+Vpm04WB-K}f{#vj!3;P{?!}&L z&iYNLpkjx&U^T?r9&@nJRdyE0oAJqv(8yvD+dB8Z#374cubD+hmVfuOEA?Dy&DFq`X+KPB@?p7EXQ(mD0F;$9xEYwx}gu;S<)#^L*{JC_~U zz@*Y1U;+cHch1La+-b~sfKRf2k7l~tsttvqWGhmbrVCgQ9(HxU4|#*M+lI#yI;}aQ z3~?!Rdk;GMV^IoPWo@tmnr7&Bz_QYvqQAsvZ=$LFl|_{DC_lI&V^W5@-|jv9Rxf%? zDZiiuENarAi1V~$xQ9K5PxUI9^0x;==QeJC6*wELkgUT=lcD13QX^F3?Ht>Uk*Fs} zX7t+xb}@|XX^NHQrfc^|#hOTV)R#BDJM=v7C>Oai(*{S@;hr~<4d~#%yd9+5Zw_h17B!V>5)WaTI%gd~D zia?*tYJXYwU_{3Jd|--c^73SWI8*;H*%8n`@zU5K8G3ly5bFFMMXL35TW{E3R^A+e zX00PIqFr^xvqax1y1H{jb3}IZexir`_#^Xm`1Y=DJF7ZnQ@GJdUYZHd(P)X5U5R>N zcMRt~mAVa?piaj4vN~*OCy>Ng1?sO9=rgZevh{$@jix{*=*upfvAUB2Rpt&QI5Du* zO`TX!&m@t+{tm@}FN5{dFVxfeQR!vr_^%XF6|7eKDX-7=vB`-Duv+9h#N?+SWRM-` zwIrTffyiub#nnur1aolDTn~mg4~+a!3+HBKEvn7T6d2ToEXAr@7VnKBwvg6$@J&v%#~3ik58=)@n__n ze;E;-{xiz%osbGEz!(2^l!UU;#)-{(9@GsmC!V}Qvht*Oq3Xja$BW~Z1b(XZyHTd- zehRa&rd+juaOK_GYHa&uHoM}{3?W3oZmcHRDhEOFViMet^}@`~;zRg8Ck3LL$78bx z1?No+G-~%t9`KwLR&Js^Bn%cGgUFD0a?f=jJnUDGQFo<0D)B<@I=qrI#{D>?S;^r} zY`d#~7beg=<6ZbY8BzWBPjp|!TfyinD+RePLvMv_?{W*FPhio0G2tMBH_``n@H*kX z$>;3sLL=vL8ND9(pV-e{6$z+mmZuo{Wq)r{D`^?6X*6|-H3}0KcmqYdx^}!cL9v*&Yeu1O24M-R#l*%n&Ymb0t-RtpSB~GaQ3M~)&}%0 z!S~#3JV#`U1(avri)O>VSG5YvQ-AiQHCJ_-cg@wZ6O}w(PPL1VbJ$t+8njD7 zMd7ok1;xy*<<$+g|0u@`|Oy3sSjW- zg79!(5wr6tGE8C(nn!oB`u3<|BHpZ=*HVkTh4{R+Kp;y5MCF#_fbt$KUIY z7v=2|it_`@YL1IuQjc${dmMC^9J6m}KW(){g*r<+{8JI>2n)P>LEjC0bXxFQW z{(3Ed?RGJ^Z$oyg0*Ll~fntS)+ zR3^Rjj9R4~0uoplo+@!i$-_IBGLe;!bVcNZ{8qOddV}%SI={W0XOm*(Ih-5R+AJTn z*Gne*QmY!e3-Ml6h!w^Q`JHQiw%r~wAip2LumWnhaOWHbb`19^z<Q51`c&Unc9Wyt2e#Ee`v&gyebx|07KcI&9iAtlTx zuFa7zlwrx=u4eMTSqT60+Pq)=xj@VqQ+sGcX!wD7PzcfPtzF>DZqf3BK&$v7dOA+G zgx2!QM;)5MrXdYsxE~=?J)CY|1n>3TPYT`5p;q{$n*SOZzVCwP@cb=nu`@}CSJZ< zL=w0q_l}5)uqo*zo3~$VfMvZn*Y8==RN7d@oNT6+8xQ14k|SROEq3IBh>Nq*3**h{ zoG!O=?pDvd{&!5lsUC}jC+ALW@l;4slx6XEHn>P><=hZfnQF)#gnym^9xkf_h4K{p8LOb4L0T ztM0CN|E%CV3w-R)_Jb7W#_*o8zng30qGtv_NMdp`&_;BA$Wy923;X_kv*dW-BhfAT z97ZP-Mwb}dT8`*uaT|*kt(nlsFUE1@<$X8Rjr}wiq!~G5JdAOQJ9O+00EWpiQB^oA zmivF-d&gD>=Z)lN^PXuGESBtyYcOPgr1@PTzaCK|EYXkY4sC3oS@P*h=@g8~pFsF< zb3x^Ar45F-j=goV%8kh6Nj=AGVI{HO-ZU4wkXJmo+I}P9Ftjh8vOaS9{;e`>w7W1p zgagFhqp>rKRjZ=9rH<^th^Z|E=UsGR!?dYZj#*50Sa_%=24MsPH2mk%)!o59ts*DY zwyvg=w^LaMfkr;s6S0D7h*H8y*pKAr50d#6Ocx;SH@I67*A#sT;&tJVV;Poam_A_K zJa(v#)~gfuy`R)Q6`2_QjKT5D3Bd<|#ebot!5u7m{ej({g{_t#t{`2)A{*nmTRITS zS+5*Q)}W7tWUelqi{WZrI()<1Quw_8WCHuR)Ppm3PBj$6De~{n#u%CP>QcFF6M5Rx zebSIRDI)%=`XF~k=iv6}`@cxw3U0HsmE==MMP;CXPj{t*tv+XClVMs-L#)165#yq< zFYkOlyU`gCd*VM3jmO@>lurF{fbZR&C^5krMCUs0*GZtmw?maQzJhDjG;MvKESNj7 zD0#4ovK4yCtMH$)A*T`3pbkCDBN*njjv;-q5`ol?}e1cw)iRCB@xO&CQo8q zpU_-NQVMv*e=A6FXayf=m#Lo0-e=Sbn+xcJ@qG@&#fUv7(|*+!%8`hGO(wL8)}2zs zKZ9R^;>n95LFz-K%@&LYrDPH>bN4`a*FIxorwI~aiesMTcw;zgHneTqZ(q>jKHgFOhm-Ce>fRY`}imk#f}lx%}8m0XEQQStF}BG znSW*xy<{OiQe%mBF9;dXf`;5v5wf_)AzEI;9Nk1zih3mPYfxaS=85j!>`2AnrV9iI zsQY{8`HoXkpEEKYXU|6fTJ>)3xo)&@1{k-&Q`-N;<_GXt~gr8;44trO9+ecqF(`I0P+Ia0^?d|`- zHjM6B^O^qx6YEBBj23lhSgN8zbUX6)Tp*)BE6nCL=h|B`s8^K!f;9O<5tL+!vc^Z0 zrS))EDV0}*RIFDt-o>{UGz|vr4l~xW_I#5bg z)w`R{%B}Y+fCa`gs%BtyqBFgpYkb4p9wn!w$M;kI^7d!G(~;%jcV9(_wR-Kx>i@^o zS%p>AMGKqQba%t1Te_t|Iz%M|q`N`7ySqzL1nCCpZrDm~>Fx&UIE(N6=Xw4cu3)j| zT%+cQcb39yjGIo~ZRqnMKERnhztyT5a#_SQCh9KO2ggZ~OTs*H46OqbxZAN>Yy}1e zWkzcvO?pOb!#?)d3dZEWjr|Pr$t)-AY=hwQB!hh1iX9Q;$X_%PG>5^g3{n|Vmn`8_ z*p6F6&alpoB;N%uKEY(uv&ggOb1~wyv}p@EEm0G#Ot=ix{DN{P2H6{X1%EwRlt+PX z+1-k+^N2V-&KLn@ybr?-j^rRq#AczdCrtv-CT7ByMlR~Pvh#+gQ2rLg=5KDfMD9Sl zShvmIr(Kk{81irvsXdyG53kG|wZ8piY`n0DO=Y8Nuyit9&IBdUP&^0T`YQGra6wWUHgN-f5rrGvN zg>BOrlw1jK(^C7BuW`qG4K{ZKc6tvW*}Uz>LvPw=#Ce~6Jc|f3in8j22RpU9%O7T= z{3bS`H7Z}XML@8ag`K@G25*_TH*|aYE=o^HhvtZfd2G`ma0ep!eRSGYYk?uAk$M?; zT_JxAML^66TZjut_HyS|tyGuq&#x^FDR`-2>Js`KF}KsQqzsHH`DWy-@jL%lW^~s~)7>+xWo-~#o8rC|ZCW9dD`@Dn zI2#bE9^V^z78j)_W};-QZIWGKm~Y29oGyQ6w?%h3{5;v3oh#DnuKE&diiJK_rOP~O zG{zORNM;#jOd>5%8o@!`(64|~9qrufQTvXn^u5P^ipdwU-qc6j^Pl`VRF}+B9_R!V zO0ckF;JzN)KO359ECJmd1zS);b=(HA5WH>1t#i&ABmd=#(e&0h;;&l7TP}ivd?ZOr zZuI?QT5npvwER)O7yjnud4*=IG?DwoNCY&3p)gm|deh(?Y2%Z2(0Htq9vx-U4lOiP zkbPmQ(U+<&;M$_i`AGMTE9dSOoBB#p$xK8NCRiy!zp@+)b>L%910&qj8h6KXGM9lakBUR9KqukSunZmuDboh36^{h z9mo50^!~#0nx?pd{-4(oJXwX^%~EE!c3bF+2)yCA@k$Ql9!2k#Z1xrsRgT;^XhN1v zsv@Rn6GRQREK$CEOo9iI-?2EgfT&q(zOP8M5vo>Hmkn>iH$A#J$J*ic211vQUvQYI zRZ+{34dmzlIYCG*Pf0MhRsVkR)gwY)%&JkH_`c#KYVTy{C+%r~o6}bkrmE@a3zSwy z_tEB?BgE-lMxQXgj!{5omP%-7LdqIKE}(MY)fdP0=dDh4XX-ajsfw>Lh9@V z?hvsMr@L>|r*W;u+W!*!jg-+LKH?Fk|FDTLH#)g5fplmyVw;Oh7|=(;drO{+vfvh{ z_}=A*7Bo&rd%n>EL4e7>N}R$aokB!#+oWZusnlPT4jpJ;nfW|D!pHNmkqfFk2LizZ4r*zQ6+EkZFI^}(l-&LY$d>3iyONIRIGc0oFYIC<6%D-H6<2CsU zS2bBmy#iXUrGCew!(jJoFfPK8*xdqU$`q#9&AC(LJ+Ck5M3`ZLO?q62*L^fQmKkNl z|2XC4U+@E=75ck=eQrmf#*MTRK&!IK-NH*dP*`sr7*L60iy@U})D}Vr5!3v5f}I|p zEDk8maJw*pY6+Ow0Xu~*weHS)^WV*Ac|$GdMNS?kQ1>iT%^`a<(mEw4FnVe|208RA z$lm@!cra6mm476=y-XePorV(SQ>PG7lS{FxaLA#dSd}1s5aC0FrA<@9uESkKxGC_h zXj%fFU|ifr9ixtN6xh}t%W(EwFw$%L&vn$dM0}F=HFYaK+oL2XVt1Z#`QdV;JBX5j zq3IF)QDhs{kkuD=p4dOL8xD6d(21lR&8B%U6mUCvW-?a zuiAP)Z`h?~;2HvOi1fplTzAe`jmJopaAUWQTFu#*|F{{b#Pu{yAQ{aE(TDcR*nL+(o>TX zh)mVmny9n3kf9?pZ?hM0vQ5LuAbS)*B+HbMf&qo$>A`C7Zxa3rNh)<^rYpSpxOEN-q$o%iJLyiT z9dTb`L7)WJUDx(Sw`C&iUs`uDdwePL2wE`BP)wDKwrbN(M3y{4;Ek;s!|Q6PA<%JC z0zBB}y2hZXWd~A1AP8Nq+Qx(hl%CNXx9{g62yVlPZ$!(P`08R+ereNSFi*O*BT!Yr zE#BYeV^+EUQk*0kott&PKl<@wG3IL}91;$~u2E)Ef%9Yy6$MH3&fP+997yXpmIH{> zpqdc;XEfA5f~*sN>xa~wFTPNU6Wr{rY_5`j+l3sq;3VA)y7SuXh3R5G$=-_CBI=e|_Z?)_`PU8~Fprj?W8ePv5i^5F-Gwhat18U^amn zWpnLYYpjKil<0yWssuC_N=!mc2v?X3-xJ0;cTSv~vtJO*%qubk)?pMP|GfOWy*VGF z>Q65BfA7e59gIJIdk^POLWlPXpP|7JO|UkHANgQ~2!=7%?rTTn`tM|;ul80TVq|C} z=Sp!etDFXU&7PZtozM+OFq!MNjEKBg?)Wd>V}7ZraTIk`Iu$YU;X2Z`H1*R8#}|N6 zM}$rR+c`~30)~qVNR{sZaRF}PRSa^@G$9BmEY8+62zhcB(@|v2q{CjTjksXZA8j1| zZ?Pp3x`aFMp1zVgXHqAT-3@%zD88 zmY|A&y`fE#Xz|9kIjOv{045Dgy`D0TZ{47iZTys@4lm#5Q2VP7$r@%jGd_yZgd|&^ zY_WWPw2*6=SXb4|XwpqSaJIY4jea5iMC%fI*o%udX4s>|J z-1&G8+0lkwD#_XlxTrJCz$o=o{P(|&E>}*D$c}IcG3c-f0;=y=S%!_`j&Z%vL>cJd zW2Z@Vlf)NpfWC?Pbm(qS5c(U(&LGTO^ox!ujD{V}c>h2t;iT`$QQYT1w;qIuD^(36 zaE0M{CV<#R4D8N#FvwA03Qd|B&!+?VERES&Cw))D9$v#0>QG^?M!Q_!TqPZCqP%o? zDezKINdCyLSn_|vt8cM~MMr<`^<%@_*{E;NW|SWfJIt^Ip=&w{rf|d`i62xa-kAD5 zV7aQOWX<4eh#kL9ZoKKdcx7Z=VP||-H^Y|PsSPW69Dosft}Gz4Kfg`a9dX1l%u7bc ze{JaH&-vpgBY`(ei@%@%3iMp{3_v#@Hp1vQ5lIie^;u1xw^f-<7Z2!dp=*5RfAdwh zDi+yj3}(#t5ZP8YJ|9LZpA2+crB@Sxo|G^LN$;8LB|kfN!0-hJ++WBMXVekG*}Zm? z@b_N)$69Pb4_wP8OXz_sSTP`c~><~MG!bY_Op%OMf zJ0D^3>7)!z0TUQkA^C{{n0D^PmbaRn`oVBQ(Qyy>FsvH|nh_d$|6A`z-C>_cU1=(L zu#W&V9pJ!l{{J+FU7S6`*;5}lwW$^AJkt1spFhI;#9N128!sEgc7FSknC(Pt*U|{T zvQQ#kXGonke_8`PipSXQ0L`Zr+Ib?ogx=kIxCpQoJpC)D;`Jj%^0)tF2MtC^7?=AI zW+lyepqo$(3|IH%ZKQ^)B@2(L+M!uv<1foZP*8ZDOK*^N(C;A?|79K zZ#g^Maw7PE{1^bk&`ye^hM6x; z{OG%;rJU4|{51mDeqqpqRjK>l>DZ6(N3p<&gaF6c^Ncg`l(Wa+An@D}>ONE)8D@$^ zj6hXw{XV|uP87GFui=2qTW_;s-e7n7>74C;S52I9Hjx?(%#cwxBst^5i2@_8)N;2% zye_&1e=zG6&=d$6=gX0q+TVGf380q47zo+g>~UT6n#_nWY2mcqp)U(=zi`>thc~w!R~KiY5Fi$?E9Qb}0<_Jsbj_E#&Y22!|K zf2iZn9dq{rmQ*s&FuF$t>NHBnlLP$E&MXNu?p?zH(NapsHETjIs@wITE{>(|?CBkTT)M=frl5=aW5vz; zIUc0HyvfYiF}PwU9kfG0Ki6NxDh2GrTk1EL8DNsP%8#FPi?N^TWjI%#&tQFP1eG=N z!e2RcoC^v%apbV*og_cydz-7ZXsg}E51hV7f(pa*m5o!(wnv4(we8C55t1!WEVyn{O z;C>o-W?+C}4$1*^obXExbzWf_4F#|+eFf&JvI?3m;b(uYZtJhHM)5;PM-{qY&o|v{ z>m)+0nYsdSSlsvAb=Mlu9c31Uc;AV^TIKLrUYz17jLmK1TNVUE*j>6?sp zn&R;Y#FW0t8GbDHo6@?mLb;;)^1#PSh~gbuTNoss9DqMCIg}y?Dq4Ps;PXkOv;TG( z`K$g2yLxr?eb=nZzqYHl_ol&hTIp%-+kUcL<)U^U-{JsAKyPbgIT*B%JZWDSU?AnqNMy}Y?zht2##InTt>t4ZkFEg)Cu-mImwrH}YQHN5 z1X%uK@NsYeIw92a<>p7vQ|$d7@zZ%5EG&)`yk%JiFxd#jjZ?-)R*03Q5p4NHT@G85 z4y50XQ#Uyy4R(OEm*N|_0N?sRG3EB9!aV!h)EtR?1dUrXPR3^jSV`lT>H-tm&@D00`dPPfooKk6>bYcPY?Q!rhY@5G?3#BF^A) z;ZPstQR*wC&!5I;)`Kx#V=!+$n%?8OTe z`b6PJnlchG$As*d*+<+(DZ<_6pdcH^anoNRlCs25JR;!PP|1itFgL)}oxUP+(rSl7 zKhgB#F3eI~6Aku->X@(!0exa$UC1Z%mj;PL*3G#uNcimXvvUip?P@t9BVg{WVBJq? zTYbPN)W^JMa%G8z?ZX(=@y0J)SF)N+pco?PM+l^q#v>K`DZcd)_FrQ4o=IkW;Ajaz z!7x%f4n(uYoF};Y^&NYv)i*ram!lJBxSvMuZ2NH z-Qp)psUuIh=S_cfD(dU=?q&2-%BLoBjTsMO3?x=)*oW74N!kE4 zPNP4({zJ;bPhGTzM$VH?8BZ)Oi%)+4$QYLS;+k3ZRteyiWKnT_-#UWA{$JfX1wg66 zv*R+O-`84SwGmu+c8u-!9wTA*ZuHi%q=Dh?DSC%vgV4%ClvYK`hjrfm+FV-#|HyZb zpzQe99_;1VJCfgq67is=K4JzK{8MDri~t_;k{$JH_JdZxff4^$?n*R9?JMuWNJ&&7 z%8Gmh-YA%G4D$EoxzcjQEvb%$DQ$4c>|i65u9tQqbWE7pXb)bXV2>y<2!qaKjgkG#~Yyq-a}p`1g$iryiNYZxNSW| zi*{&Bp4xc>lql(rhj`wzq7W20Em-PpX*O&afZ6K-okKhThfWtU4m53VM z7F+Kvd5iTPHX_H|HLZMqkceGmA6@XvGR)LdLL~7{GH&I)to=!MWa?j#Xpr>dX_aD$ zf)|n;+NKJ3v7^?|GGT@pbgH1uiLUWee>GTdyp|6Zk-Yqc6Y&<~uF}`;_Yf+y{id<6 z{UW(KcJxOm!0`=lmO~(trVg$+W`E=HthydWB?a;`ugN>L3VR|e-Esys9JmW%Vs+ZsxW~wbn7EBT1 zWcd?JKW5%8WceEP!B=6~%@9|3<}YcOU?s+2W>45=hUpp*86fh45&z)slr1-Y(I^5q zhZ&%zbbND?l-;!?$<~Pil7^`s;&mi?gG4)c;E-0j0;r}P8pAX?mu8^Tt#sd_1eLA- z7Y;fnSCEEw_Y@UG-E$ae-u)X5^VW<-fkGz3dqdkKb(sl9~}EHvF@iy z=MPvjI&OGHHFNIHlq~*2?#pNoZ1aZr;i_DGN_OnMW9`W5tortG@)l`_7T(Gn`Fua; zQ@^`8;u# z0lD>eylcgHb+RyX+g8}Y1RJ3VuvOzJ?Azn*ZmglYO6V~J1Xb%KwyU3yuxtg>67ikC zLsF&!yH>=4=NL5!(uP8xMQVb#$#Cnn7{Q{K8 zwJ0xp?7>z;Y}ZNqy`uUjqXC2Gm|(z%AI)zD%5UX?NkxO zt(9zOyCxDqC&V>KyTgO3;eoufQnVBPVULQ$EkYNw@fvTTy1;I%;uyyJS%)#&mhD}g z0x%UWrR$B)U(cwi=gY*I@2xK%N4}E*N~`gI=r0oMmyjs;<5EBRjl}Rk(yJB7e8yd{|ZC)9*e zcluUph`?1(B|s*4O<7vnB=nkJTgq}1Us)m&t@Sm-uUlaF-tFm#hygZw3aN5E1*3!E z7LTWVcy>AG+#N}yPIi1dliNZ~NJCJrfOreQD89v>sF!r*b|8bhicTQjyVdXspazEkU(lIS9s;wRnru3+1ysDr?b``gm4=OnMDF z+>u&VA;-9*vr}C+4wMYrlnR$dTz+WN%&Rx@YvKQ`GModc5MbU@jnz@X36 zZy+zFD$mDy7al2a2Dw1Pq6-lP zoabixq~PDKj^Q;Q>Wy&&{x!w*YoFgnxd?$9r#Q|PsMN-Eq5Z#V5PW-s2K|ykPGTrO>!=N#g&NKi z+j0j_;NiuHwzHLqbV-~vj!%BMfG ztKhFHvta2r0r+=Kwj1611F0r}l#X6`?+R+@B?n+kOpG=2LsMUTW*U-QYHtJ)>WS?+ zortvm7L^kh2Nx%7l?B8puzV~D=?n=&K0Yl_x|-%|;=4M}-W25T+kGYSZ;&Y6s@zrf z=-AinTYupxX23%O5)Fhik9OSt$);=#2kFWmm~d#;G(Y~7;Qjm%naSfhD=1gEMyn+j zI+f}QoXqJEVM&(#d8iYmR#^h47F`6ra%j2eV1LxDI!~844zyii1dxTI^QZY~J`8Hy zH2?{=qY_^>qHKB|%xlfxKl2EL{<_9Wga+i;R$d^4eV=Wy5y3S~NM+DiBKNblWYd5K zSf&AK8Q953ieb3a$UwFEjFiFIo=~pe3zm%aGQjh$Y%)j(PM?&Y-^8}n2q8|!4$Sp2 zTr#y8q`c%TS+||aJxJw$0xh9Vsu(78QW6LZw(~%B?D9hTT_+7_hn*5k-p7#5ay<(q z!$BWWD3U^agjfVVrntPbt4UVqMcxZ9YqQB|4_2QO>1c|yM*5@`QA-QMolz85LRKeg z`|Cb3vm?R(6tUvMOk5C6P(lOQC62(SpIG{KRXK_{PU9@lZNuZ%8?{w01Aei#rRJ#C zxOoUu%%#F_r(G2bNQp?rze8)MWqB**Nwkeh7gOQ>80#)Uf*gp60|SiO>Jbogn-b7- zUKO>5hM2|ACAx8@C0=c*D~S!&L4A|&pshU=n?)vcPe$NKjtT*?3<1JH;W+@BOCJMh zh5zQ6zDTrQn;akEqLOV|I==Axu;jkA&|>0IhYxA$&%A?PXQ>(pZpa^@wJT>3q{w*J ziy~jPT0NV3?Vg(_K@IIADm17+itZPeOBgDJ%pg+&`8_2Kg{O>h)-9bEkJ}B}IU8eN zClCQSIzZQebd_0WEZt>`YFC^I+y_KT3Wgils47?NsTQ&D%v;m7TFWo~tU52K_o zaBNqjCaHkK8^*Z4e;Gi8gc%kRNWEzOd{UugZ&B11^NXt49DYx*7R7<&0Cyz9B`eCq zqy-i%dqH>tb%ZA&=aE$q338Ay0k?Y4Nt+qoKSpK>L*&?8#v1vf2_7@3^j-+!Ui^-7 z)}e9uON70wKS6ie zZw_+%AooX)4IRrg-9X?T7tiK{T5cw9ftAmrY+d3b2) znb1q1+D{`oo-W`}U(|!>mQalS8GI~&Qaq_eqfto?r2xlz_k&(e8aLX zU$8xRsO{xWz8RNGe86?}gS54teZ^H92^X@8EwbQUV@Wi(KB^x6-!1}mG6+PSnAav_ zphambWPWXft)PBaLo$i?)k4UF*|N``kzH<7*o!^A?wowvm06l zNa_IR`)r+X&+Q0;Wc)OWofR*pD^@Y7FEo%T2o^iy7Wt)!vC}6<3L#aaD=o5PQ+}BL zrQE>$QRa}vu1Bc5r*$^ZTdxk|SFQC-1TtT-l5pz>k}bj+txr9+0S|~HpDAgdWA6Eb z%l1O>5U3lP&M_MupzCzMmMP?|qVIJOmjz1I{(cg5%$>lFKE}2g_Y6I(K45uyv)MwH z0moIwb#{*|Zqwi%YR4LMz1?{JXYFy-a)_>)Ra)QFIRJ3lSTLR2(mb~>2=^i>#c`}Y>Bzf@?hpV?5k_^riuzC)p*voa&=XG+ zT*<4zt@=+S#7Lz!(#U9{TJO!6Cn3qF9GIpSmB$xz|-P>2eWwG9cs?5wJL#YwiF*D02AG7=QK z*vapwbE&;Ue4E78Q!FISjlmn@;nM>WQ7h44#Ff;y$@wMZbU(8`RjTWEV46MT)WN>? z5--ms_`LNCleS=gAJJ~)@Q6dLUf)Ud%?dTwpGry+Bz`u;Cfqm1;=-8Y2Nj-Nrrdkh znz@zh`^f|QAB3lQ;0QLt6C`nC;e<8M=@WiCc6-!@)oUe^*Tve%Dbuv3kYR!3PuMSD zpA2odQYIXM^HI#SX8Ukwztil`bOC!u2R?bENg+v{;u_+S5`9pOag$8QVmFgdxDDQ3 z5$olza*5gv!==)-!yEZ$8B|(WE5|;Y37R>sd0ZuaQ6T6ihV^jYZ3tkk(kNzW?_>R- zGRep>Qv@UoejX`RDC3LOegr)K(!(bQ_#z(=i5|ke{0h1Dwl72EVH~l4L`omuqvdx_ z+>G>=9;kqv(8dYtD&*S6fHswi5^(2<3*(6?)Br#Xn0nO1K#`h>uMPfx8=W zNc1!V?L9jP0>ysuuYRTA)O;%t*H0H(uH8B^RgcV;2e8j2Tpw%V;xK@0T?BhnPZGQ@ z;uG#~yHZv$U<@Re&a&&BD&d81u*f!B&%H7ieG=?g)PR_#G+yTsSo5$gAP zR&d#WeVHg$S*z?^7Rg2{@-+kC`rWTPxnu}7AU|@ktZ1{e38Pc$qR#ts05%f|zg)=J zWa8p7{vA!CiG&8~L_-$~$;%MZl#Rm)PVL}&!@$gjgf`5HYXF`|5Bu1a-ys0R!sxG6 z213$Uc!q|6!_|B`bNw+3(3aX*MH}XlQ~M~f!}>`^><%}YBw$H`pIG{lY=KT~py|IO z1Ig!({*->5R#Iqio%1r1uXA}bi)~Hq4<<08X7~ECFn|fRt^G|-J%+uZA8mB4?>~T4 zg?i!Mne~KcvLr`gGZXE4O|ZgD1ZWNw2JA|ywkE3^NgFice!wAir$*n&L5 zdl8lhx3~^g6U@|CjQ}{8HQVnt^HH`H2La0k(RQDC$V|rXkjgp>F8@)|`p3Ik*q5*gxU zbd6P`^lS2UfgCKD6gm<@tI1rlg_`DnEAx|S z-{WBq)mHd+$)`N?nPk$y`u#eI>L^KeEj~EzO>f;c_kD58nt-X&`(mxr8eJon)L)B_ zkms-Nfj@*Ebc~lk++gDUz1`ndplqoV6#EZ-QGT(9tBSpzqQk3J4OfQ>S^ZJ2i zUgB5!SUyztShuWul%S$_^o7BWyyQNIE@n)WYn80w8qcwGcQsxu(=YlU(pIBpOru54CA;2^cC5&Zs6TxqnCUf$?x5ya zmQ9WP;Z$hw_Au2xI;?<*We|k%Z@IaqSQxZe4MsAlFPBDyNAPZ<#kkMo!fp0Dq_Tv0?Wh_ZfNK3mfr_)g`=$pG97=} z?*9DKBg6AiGBQMI7)im5suO?dWflb4^T1XxA&?~&%xh@WXcL8&x212rh6LY1m? z8eARemfdQuP^M7qsiw=J@}}A%JZ7tx&f@ld6Q+5?)&HcZe5XX~=`k{3f*iYZSBFa4 zNFqZm7EdzhujqEi=`?rlJ;-*pCp!8K3HFMnc8X?5cV*VuG?o}VZuyc$Gk&?nHtuh( z?EEU{7tx+&S8nygmB7BPHwa((NdxnW4QbrZRzz2wMO}rpg75yq^}$xtt5eUb$zj9F z{SyyLoCd5<1b+WJ{5YDs!{BV1d>rZ~*jeKRdN}0XH~$c8WvKH5E$HPsPbKShr11EQ z5kYi_c7|`rGRRr1oAJZ@{d|$7V(kWnN48|veT&y{LNSZ0bntwyo<8B4BhgYMl(k7> zm#hT$`^QK)*~-NpUy_>w`m=!k&5c9e`)R-Cgu|erOx~ z;Z{37&ePXvkDYU&&qk(Lly)>w8q;H#sXf7J3Rqch9N9Ik1FyZju}^g&~VQw<%Nh6(s|Fl}rCvigrS`T+&SGSw`xERHWkAZfq`H*rQ?W zvy|^)5p5qSwd(S<%P!A#a_52Up{2?#Zl($wmfUzzt~l5MFUavMT%rrM1=fNRGK#$z zVBKcNB552v($>useifBzQhMapsz&-i?E1R|;r;C=W2=RMrgw28RVtk?!~ei1W5efb zZ=8Uy-7k`GHiZWXtyX7V4V?cXot#%~CP+_*$Y^bc12xXZA??lg#hgCS3#5G*l51jY z{TrFA+%uHL%;vhX;IuGS++=1AeFzvNfNjdSP677m2Zd7o7aXQ9c4b{Xek?X0yFC1r z>_r9kM0jcgc^b6FZ5KMa^iCa0Tg9q$-eel9;!n~b*bZ6b`A-z1uYKj1D~It`_EvtQx8kdfaRZx0DOwT~sFCQX1oR?!xThruwdR!C-h!`Qnl7yS9au zgtJ|4@HHc26TF_`f(5YgVmEK>L}Uz0Vmq;_7Hy@ShWBBbQmu=YgU}S(jd%Du{vE0` zHf0R&yC!*z%ywHAsvGQ#rUWwkwYe=LH1r%PU<(4fs*pW!G`~JSi;n9hk)nZy6Xi&h{KZjZTmH}nw`P;8x0@xUjv@sd=_kovM@;A6UqdJ z@%u2yU}Pl72(MFmz?&EsI}{RAM3+PDLR!4?1dBBE&WT~wV|c6d0KZDp*0%4z7mDBS zgV5SI95bb`gbbQ>?F${7I;az;zHeOR7BpXS4Dd7l;d?^ClZ57V7Sm54hvmD;=A*(->J4Is2Dw6TfyZbli5rXD|cb zi`cjAf9f?a8|L0laB*vMy51Wy^_ET-W92_w<>oAT3!%J(#gV|qEUmXXcmpeNm~d)u z*I-_6VMZGb0;FI8d?zGT`e6dIiuQKDEjrZ#UWC>B?V`)vcx3P2zd`>{p?5Z?Oh+t3 zQs>KaHlsZnk2E8aY_@OT0pBqOeGy4g0)AWygRHu!brnu|I2Eg zhVzKtGca18NUPRXJn?lh&AK~WK9=)6v^CRD|3gXCp@{_9ac;0|fM^LkT>zNe@W=F) z7RcESr4_>rt4!^Sr;1CTo>BO?qINkIJtEqJgmtQfrFbK$UiM@zQ+xc!@TcUA34Avx zL)N(34dzs74NBvfV@?z>aH-yDgI9C_VtS+IWu=N3m59I?qye~iE@yPd_d{F91w+MU zhh4=t2KC$(U!Usi-=bgghrE~2q0+?+SuIsGcc`xQ3Bk>R`H>20h`n8d@rNmR_t zsC~R{m)*pTuVb2F{A4oLkN$OqRLN%7t{Cz%BlI?Y>#%}sK>U4yqR^ILi{-clnQY(| zpmNEF+;H~U0dFSxu|tsQ9!kW12h>@M%&EWN(r5aFpcRdPIzJj@Xa+(QD@*bny%dUM+B(f`}j;Yib zC?dY6VlTBmP$#)O^v;Y{XYMT|jNLqR3wD)ue9Fffr#_Pkus{~V20I1;=@=@n-<`wt z@{C!v^<$B+*=49I{$~aQyEwvJ#9rvR)ulQpc$YePc@JNM4sCeYW{5`5M4~nKkI38UfSQKJJqMIA zA}9!g3Gc?n(;uAGi|F?W617Vj| zp-IhUh!!Xf!VU>9t-*lv2AuL2%ju_;os#n3_9A?h+VIcX{FfzeNTZigneggO0+Ih@ z4)dCuT^}ziH&}%!O)NcH$=`+wJ6-p5nQzGajA!ymu#34A#6?}YiUdrvT8Dz}XZn0J zrvg+d3sT#&*b6p|zQL^g#=rRq{jBB(zgZ6n4nLN~eVrf zFkhvkWF`A-VK|&|lL@7Q}GZA`sU$HpUT3r-IGi$ulDro?P&eC+-SXdhsVuSo} z9AD_Qt(5j+vX`Gn{5|7d6yOFBySr#S02gZkj`}Art2tUQG~lMP+EDkgs!8DBQd{Ul z({rrODv~JG=TQrX5B-RCc&guY{7`GE33g@mp5uMLitkh|gx1D;L;Km4=nXXG&JUz)%$_ z$Vh&)m=(fw{p0!&v0KeId(Vo7V{0K}{bR*s;a?}cm_OVS_q9d%b^uCC^h-E0auAie zI8L9`gOo-5w`~)*LQJl3UpFd89Y-&@T+KF;53kH(xl`QVy zOc~ge=9)-Wr#hdTUYKk+i}?QTboPeNqlORyCr(|{^F?pC6DbN4?6^0m%eJ%nA*yQx zJbd7&n~wuh7JIe~{N=2mo4zFd0=q8e)99$D!ri$`7liwo)!Z&Bn#KVcBZ$rS9Hyn` z3p01Drg_@XC^BSnz}~za53R&5m6aeml0u-eCJ@tSYGR7vN_b>WtbI$t?@bZ-HbEw3 z@uI4WdjOXP^_0$WDDYUJcPX-$BQdCv10?By?RcIc@B3%>t%hDppaeUx-=8y>K!KZ} zrO=0SkH`gYK$dE)F(UWhZy0|OaYi+VMgG<=j8(JBoaLNvB5#i zpGs#lDxc&?sqM&WQ8(-}`T+11eK-Zk4Fw8w-_SNy(l33)hI6eaRX|SF~U1{jT!SkM)sn#+SCpz`m zoJ&%~!P2F!qu~ped@pN|%J#)BvY`iM!r|M=iuI1mmQ{A~-?=-o5BQTwA(D{b*Wkjm zd3vu2*%pI^OJ;rSQa;ff^>9h5DX_*eS|$GJ9+lDC&2g4HT{hrFA;2!FSOMY}Irrq(!egvG^Ec+z(Z-^K|z3vdy+Lu2r6H@E7pUcz?%}cW_bJ9>Bjk*WN&ZlS63$qZW0f?N~yNEVH{?j#1$ZDC(_6IzK)hKISsd zh0C9>jH(fg5ZQyVV_||gNfpT@*LVX7WePCZSSRpRRp-Z7(ok$)TvU^P)-i;Nj)l$C zUR2bln+yB9@Ez$E51!jFBuZ>BPy*WhHq9fbTL(@RKkgIfT=4UC&9A5W$O*%QDrv)G zo#R~dv%zmot`jA$Xp~IN!Zmh#+0L*Lt|u2Z4ASCR!gGEgqaooC$9px2LI_BvgxtoE z%kmmd?;9&GZo1l70LvxJVsgoeTdnBlwd78u&bq-Llaq@Ljwe}6Ij)`SAVFkcnl|c;?&px{pgq5S$YamI)F(JeD3ZL~Qy)*Zvk zw_;C^X=cekB{q3m@vlhhC4ULi*1U4NH+&y9;7>3M!r4Ke!E5HoimjLw(3>LfKZtFi zwm=_#W&kJ?r7Rw2No*|qsn}3wDE)v!*K<3;0@3U_NqHvj)6GKVjk(iKyBEYWF`ReK zcR%X)3t?Dpx!XgF`xACA@uC}`jf>mS?#LcEkTve4b*a0_u^y`;Zw%gneVQXX#cq`P z7yd?(_bVX<6!2;3i!>^(Dh=!|ce!b`e>x7BjfD=QsB+cjN-{4N@wf)CL%%-`Veb&` zBu{=gw7>M#hpb!UO%1k$_jO@~}-1=4)Fg zyC5PVBn8cPV%9v!N(4C;H-5jEsIDCGiMr04oVTx?BsmPvtcw?X$-gYPvI{P6OY+sn z@v35xSr?KziU;0~7=cz540!{K!HqYo&N4)5+63REo6N7<0$mxl5QoDJ{h;%_ZQ;vT zF1O4wS*7XSALUx5Euy%M{r*{Q_ztin=-*oLv2a1EKuq|pzYn-vDm;M9R8jV->U$~W z_Wbj{>1@(VPfIFOPI)aAo|PkqJ;C84adfj&ypta>-ouZ$?`dB~228FUI>_7kQL|co zj^o&|p@64b+U_9*39CuKPbf?J*>gd>@4ne^>NRW2o$4#$XA!rkRwSWY#5R)+JB-ncgM-0^OzGy(3Hv^q0ZL ze1yrrv%-bQuD&_@?yMWsXHUCWAF7Eo5**uOv&G`^Hp{!e>X{tspM z{*h(I_KYor8vD{?!f1$0V=Mbsh_OqS>^oV;R-sS~#vo)HF^G(vJeDj&3WKo}O)4f* z2&K=h=ll61zOUD3ew#USU+226`<&~3pZ9g$=Wr+*o&d}c^%{D!@g3|15h4hL(DoEt zJ{We>_xoxFG@{qrSId;_lt1d(l%7~44kWGNW2ln+>#sEU17K&Ib$_WJ3R+fAoTQjs zu+2Ds!n@npqeHjh^&?d%5TO{u5Tc6n(eyqUN6MLzn72)UBe>P(1!pUFiSdJp;aXNmRY9~9hQFm zg-?U!6F)OuTF|oJa+HYk_y5(~wU(YauR1V8n0yqD<4FStfg&Y zo_D$q>Ml&;O`05k5mGzVZR7z9iYw-*7(Uum@SV6O6C!%vThwE@QyUV@k9w=aTnK5< z_Ke>hhczQy@|1NaOW@f(I#E5{`P05J;+$$yKCrNhHx%sUNYb1etp#W^51Xw%ovoA! z>6lk}H*^a$y3yeAZb)SQ1aScPx9R z8wO?n%hHv49td3ZhL79XbWurhrGs%blW%3QsnHyBpF>ah6M$W?;u3Dq8}3eya!gI* z6&7k;D_QN3J2dT%-=>4n1P8lZh=Vsy?{yIFti@;;wWdsa_$G^|;|17s+@Xqv{Ll{x zsG6YZ7{3#L#35E3MD;Y`IwAX(b-zc%B~rNAZDOlc7>rLnn1bfMDw_zlufw@Z=dQzHldBbnl1u_RAT zj}jJ0@(FrmL98&7KgyYacr1)zo73+~@vWO-P@O5e);(2wzb4vn<}}}(=?Uv2?o>=h zhY6j0>=l(cx+`8_qc$+e_o!Pc#~^G@u-lNL7hc}NHO`e780cu0w9Xp{zwM4C`Fg+K z9@1rzuBi;lRPM(E|qr4PF4QNM*GeKF0J@_3BpUiFpT1EP8NegClOXcaGJw~8O}Rg2H4>H zFShXgC0xWh8LrDB+{(9CqADcTWa6JIpH9r;-ta!I9&c ztd)%;{3Dy#dMxBFMPj*ODdQe0eLlHl@wC&Nk&mG$$AIYiXuhMG(dbZQHIgr-ZM2aX z|C7>)sVsyf%kQ0Tyv>s(H0Gghn-M1dJ?+Vz$pGG@(NnZC!B6*;n#2A2T`MLQY!DK* zCK|BIHOQ;Q?J@L_kCs9CklJPwl?#<;SmpCul<4sPXYny_((`j0e#|aNO({+_Q&q+W3!>_lqMC?Z%N5j6uig@z zM%XS27PQ*d2|QsL6B2vWZnx)v9QKd)y}VUuP8kY!S}Z(!`{tzy-eSfn+9Hsv+T(82 zmxL;NCE4HSaB9vsQ;*IO8Z+DI`9b7n*L8P+OJJYvJO%e-bIVao#Cmw!+WYweg%x)yO*`oQwzg3gfSO;JNxr`c2-Yk!rK$STTV+c$tgqNnAv%2bak%F7yuOAhb*Zw|#R?nz1$YNc+r( z*!8Os$X^j}3^G0Vm0Jmv0r#~Bo6om94!oMb4yhckMVk-Oy+n8eCphB#yu!&VJ;t0i z6wVT}MgO-it-~ufG$W@73eES)a&ruOj}B$z3i^V6As!ad#uPi|<9BIZr4H50Huk$P z!lj!&Q)`$z`oAk61FF^+<5~Lq{Z<(wDXRJqr)*>x$*6Zx=BR4gXhow3*hqV&-Is!A z&3x$d66@!(zB2ZxFB)qiMa;>A)gQ>HJM@@LAxGUwoo`f9zSlyHJ@7mZRgi{&`=6^) ze3zr<<9QzAs_2IB0c!$g^J{Q@Yg)EILsK>|BZ@XoCbCOImDO`IE4HZ?&VtW27ebOu zsdd^n6huTyV%Q#4S`Wekbb{)9hsZs~T>dRfZBk2oEa$CGcXZmB~1DK0%U8%3`3M%mx>sM-Yg{GxKDtaKyz*|Z9 z=>qUzHq*~KJJmE?OtJ5Yp3IdV>tjptg z%wgw3*8Xtvo779)EB6;`2Tb0LsaC(YxBrrKZw^^G-g3pyw#o1cD8IGEJ+5@sBevY6 zUTysc=)$s~Am%C|gYMq#k5vofup*+FkE+n`AY9CSpw>0kOLwB$(st_H&JnFb(y=}b zBs2zB^WzfjHqF!HwG#OHa=c{=x`;2rm(z-~{mU5^QPxogYI~*iV&Wl9DqMHIWiw-k zgU990mQD%bgB$Y)#p&;@YY}BTq}x;-TTkAUilfFG0-vjI?VkEsXPV-n23L8rMhN!3 z6Kwbyl0a5;#e*F}<%s3FJfk$0Qw`kJ?h;f5fVV#jK|!1tHLP27QEm#^YfYr>Q&UGJ z&p5!%NhffhMQ;7R7F8*_ibzd;qI1Fai(hb(oj+HOx@sK$zPhU1Cs{q0gIF)c>w8ax zo6BH8e0H?TF-}17$Q$q_!oe@H1@+Xj)okb5`d$5vX%?2>oVv$n>Y}3`F4Ru;om%a8 zt<1bi6JkR98wg5ZK%J0}95(QF7sqSUjKKWi4rLNg<_l~JD{V?bRMUoTGMnqm0@=JjDHdJ+x1}@=&w|+g4VTFS=H#u;=lx^jQN`F6(so%f{3!m&BjBN->D%wfW z3Q>O@TvOOKwo!_55TCnXC4bJBfVB5EZ0AmBta<+J2{~t9gVNZz`+VTbf1epa;CvU5 zp|)FK{R(bIcBC7QR^tUu1`BEzG=KfnKTv)eG#Y+uZuTa6zYdwb!Z!=*H6ofaz*cRQ z9Wqz!K3(5sQNq6KqqJ5QO~1?DXcd<>zzANIo%K%cH11u59c><`C*zG`$iOO7e#6ba zCcpl=;TO-1aj3T+iFLZM&=Y3f{9Ry0MJ)6grSIBE_<{rjODVnFJ)%w{hR7q+KIlj8 zZwndOD;PP4ZH|50{Qj|UUvBTpZW;44%zUwXx^}ug(v~+RB=+ZfVe|)YHFY>w+S&8% zp?2wccKs$CqwXGhWzY?2zId6Rf7`bB8aupH`=p*=)UhPC92=UM`Vto$)T}p~0qF@h zdB_VeN+u0&K>Gp-8`HT5B54QuB6)?QeXw~XoFRMgx$D+B#tyM6fzjVGW*Hn{`YR*P`ZK!##j8ayEit z`ii*H?)Pm+cvM9d-44F*bSpQoG!ZFCA%#=tZl0R3(Mzyr&H#(6F=V`#3dul>YLb?A z2A*)d(p$FQ_lTtpOHS9%U;Cc5DKt92g2dzZ zzFo9e$bgk5GWM6HeZau45a5*;Ntf8_)pfgst?+k%ct1^fF7t-7{M~!nIMUJM2^%(oG}G@lkDM2Dpe#(*RPA}cRgA_n2DFJd zLxL(Oho`T(1G0MxUrRVQXX4Rh#VwGuENx)CFD?zFq4T|lco57`AY)c;-b+SZ((_6M zxA#bInFtlU)Wp@^$6A;67;BFh6PHUI_Ba1EQ}x&1CuV0mMuk$3Q8^x zr)r_SHWeHcY)7c_Z_==-xLO#qt4W#dRtmI}DYRyTGa*%iputC8N7>Y?HL@^4K8gT| zm>kSuS$ORISpQDt?oWqaWJoa`=861%Kk~CTrT_2pjIiu9IS#-s<6d$D zE%kY?zd+T=5Vw2h1YwC=!Lpr(lbziGY9PO<_RR((CtnxHwQ9}Q;xD09YkV1*~rcw@dgk~uM7Bw&pwH} zWvSm~=|w9@{ut^&pdX6F8$kE^ohB>>cvH6BmkXEK`oB3!sFiCmTD!9|f7>mQfJ)N zri0r!c+>v*e5BjRUz*Xd2}}6>^T*dG9g{r5%%>n(LTEzdOF#RkWLONG{0iG*rAGMD zIFdUO0YG)Nqkq)-#djee1sS_qqiauMN0f=m!B;is{MH@Z0v)M|iG~2vAv5R49D5I= zXGf$v|3&{cyDNv^E>yZ!#D~Sp_v}@#W(`02JjP9T5jh^I;C~$W*=Zs)#G9rV&YQtheGbB4WIp816F)KN*>-`9-~J44r7 zs&i~xazM?};6ExsvqS;SgB>sT9xHep3sF!QrA9(gfDDM@9S P0r(i|o9PjCToV5S)L@04 literal 0 HcmV?d00001 From c91e42f51e0cc012bcfc018f2655df5d595d4891 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 11:59:39 -0600 Subject: [PATCH 566/644] Using seealso declaration --- libensemble/alloc_funcs/fast_alloc.py | 2 +- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 2 +- libensemble/alloc_funcs/fast_alloc_to_aposmm.py | 2 +- libensemble/alloc_funcs/give_pregenerated_work.py | 2 +- libensemble/alloc_funcs/give_sim_work_first.py | 2 +- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 2 +- libensemble/alloc_funcs/start_only_persistent.py | 2 +- .../alloc_funcs/start_persistent_local_opt_gens.py | 2 +- libensemble/gen_funcs/aposmm.py | 10 +++++----- libensemble/gen_funcs/persistent_aposmm.py | 10 +++++----- libensemble/gen_funcs/persistent_uniform_sampling.py | 2 +- libensemble/gen_funcs/sampling.py | 6 +++--- libensemble/gen_funcs/uniform_or_localopt.py | 2 +- libensemble/sim_funcs/chwirut1.py | 4 ++-- libensemble/sim_funcs/one_d_func.py | 2 +- libensemble/sim_funcs/six_hump_camel.py | 6 +++--- 16 files changed, 29 insertions(+), 29 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index dd7322a57..6b8e0a720 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -9,7 +9,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): is told to call the generator function, provided this wouldn't result in more than ``gen_specs['user']['num_active_gen']`` active generators. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_fast_alloc.py`` """ diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 6bcc9ffec..7f0700195 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -21,7 +21,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): alloc_specs['user']['stop_partial_fvec_eval']: True --- after the value returned from combine_component_func is larger than a known upper bound on the objective. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py`` """ diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index 3a260a6c0..3051b0db8 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -12,7 +12,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): more than ``gen_specs['user']['num_active_gen']`` active generators. Also allows for a 'batch_mode'. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py`` """ diff --git a/libensemble/alloc_funcs/give_pregenerated_work.py b/libensemble/alloc_funcs/give_pregenerated_work.py index c64a6609d..7d29473f3 100644 --- a/libensemble/alloc_funcs/give_pregenerated_work.py +++ b/libensemble/alloc_funcs/give_pregenerated_work.py @@ -6,7 +6,7 @@ def give_pregenerated_sim_work(W, H, sim_specs, gen_specs, alloc_specs, persis_i This allocation function gives (in order) entries in alloc_spec['x'] to idle workers. It is an example use case where no gen_func is used. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_fast_alloc.py`` """ diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 8525706c1..2195bc67d 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -20,7 +20,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): This is the default allocation function if one is not defined. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` """ diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index df0586274..318c4ee5c 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -13,7 +13,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info This function assumes that one persistent APOSMM will be started and never stopped (until some exit_criterion is satisfied). - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py`` """ diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 4b3543b60..a62d0e5a1 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -10,7 +10,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): the persistent generator have been returned from the simulation evaluation, then this information is given back to the persistent generator. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index a56b49906..dcaf31ef1 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -20,7 +20,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per evaluated (oldest first). - If no points are left, call the generation function. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` """ diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 009411429..cb9dcffcb 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -133,10 +133,10 @@ def aposmm_logic(H, persis_info, gen_specs, _): active runs, order of points in each run, etc. The allocation function must ensure it's always given. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_branin_aposmm.py`` for basic APOSMM usage. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` for an example of APOSMM coordinating multiple local optimization runs for an objective with more than one component. @@ -371,7 +371,7 @@ def update_history_dist(H, n, gen_specs, c_flag): """ Updates distances/indices after new points that have been evaluated. - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ @@ -766,7 +766,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): Indices where a local opt run should be started - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ @@ -890,7 +890,7 @@ def initialize_APOSMM(H, gen_specs): """ Computes common values every time that APOSMM is reinvoked - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 7b085afd9..292a3836c 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -141,10 +141,10 @@ def aposmm(H, persis_info, gen_specs, libE_info): active runs, order of points in each run, etc. The allocation function must ensure it's always given. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_branin_aposmm.py`` for basic APOSMM usage. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` for an example of APOSMM coordinating multiple local optimization runs for an objective with more than one component. @@ -671,7 +671,7 @@ def update_history_dist(H, n): """ Updates distances/indices after new points that have been evaluated. - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ @@ -803,7 +803,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): Indices where a local opt run should be started - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ @@ -895,7 +895,7 @@ def initialize_APOSMM(H, user_specs, libE_info): """ Computes common values every time that APOSMM is reinvoked - :See: + .. seealso:: ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` """ n = len(user_specs['ub']) diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index ed51ec599..48fbdb524 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -9,7 +9,7 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): This generation function always enters into persistent mode and returns ``gen_specs['gen_batch_size']`` uniformly sampled points. - :See: + .. seealso:: ``libensemble/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` """ ub = gen_specs['user']['ub'] diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index bc045edbe..eae6c09e2 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -12,7 +12,7 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe ``gen_specs['user']['lb']``. Also randomly requests a different ``number_of_nodes`` and ``ranks_per_node`` to be used in the evaluation of the generated point. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` """ ub = gen_specs['user']['ub'] @@ -46,7 +46,7 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs, _): and ``gen_specs['user']['lb']`` but requests each ``obj_component`` be evaluated separately. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py`` """ ub = gen_specs['user']['ub'] @@ -74,7 +74,7 @@ def uniform_random_sample(H, persis_info, gen_specs, _): Generates ``gen_specs['user']['gen_batch_size']`` points uniformly over the domain defined by ``gen_specs['user']['ub']`` and ``gen_specs['user']['lb']``. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` """ ub = gen_specs['user']['ub'] diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index 53342f15c..05da55ca2 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -15,7 +15,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): ``libE_info['persistent']`` isn't ``True``). Otherwise, the generation function a persistent nlopt local optimization run. - :See: + .. seealso:: ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` """ diff --git a/libensemble/sim_funcs/chwirut1.py b/libensemble/sim_funcs/chwirut1.py index 59c205b33..8fac5e50f 100644 --- a/libensemble/sim_funcs/chwirut1.py +++ b/libensemble/sim_funcs/chwirut1.py @@ -260,10 +260,10 @@ def chwirut_eval(H, persis_info, sim_specs, _): component of the objective will be evaluated. Otherwise, all 214 components are evaluated and returned in the ``'fvec'`` field. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_chwirut_pounders.py`` for an example where the entire fvec is computed. - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` """ diff --git a/libensemble/sim_funcs/one_d_func.py b/libensemble/sim_funcs/one_d_func.py index 07201eb19..bf02ed027 100644 --- a/libensemble/sim_funcs/one_d_func.py +++ b/libensemble/sim_funcs/one_d_func.py @@ -10,7 +10,7 @@ def one_d_example(x, persis_info, sim_specs, _): """ Evaluates the six hump camel function for a single point ``x``. - :See: + .. seealso:: ``/libensemble/libensemble/tests/regression_tests/test_fast_alloc.py`` """ diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index 7f51e14d8..26d1b5b62 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -15,7 +15,7 @@ def six_hump_camel_with_different_ranks_and_nodes(H, persis_info, sim_specs, lib performs a system call with a given number of nodes and ranks per node using a machinefile (to show one way of evaluating a compiled simulation). - :See: + .. seealso:: ``/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` """ @@ -60,7 +60,7 @@ def six_hump_camel(H, persis_info, sim_specs, _): ``sim_specs['out']`` and pauses for ``sim_specs['user']['pause_time']]`` if defined. - :See: + .. seealso:: ``/libensemble/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py`` """ @@ -83,7 +83,7 @@ def six_hump_camel_simple(x, persis_info, sim_specs, _): """ Evaluates the six hump camel function for a single point ``x``. - :See: + .. seealso:: ``/libensemble/libensemble/tests/regression_tests/test_fast_alloc.py`` """ From f6f5bbe843808a8df2c1e533a41809d770ab099b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 12:05:12 -0600 Subject: [PATCH 567/644] Using hyperlinks instead in APOSMM --- libensemble/gen_funcs/aposmm.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index cb9dcffcb..04a457195 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -134,10 +134,12 @@ def aposmm_logic(H, persis_info, gen_specs, _): must ensure it's always given. .. seealso:: - ``libensemble/tests/regression_tests/test_branin_aposmm.py`` + `test_branin_aposmm.py 0: @@ -891,7 +893,7 @@ def initialize_APOSMM(H, gen_specs): Computes common values every time that APOSMM is reinvoked .. seealso:: - ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + `start_persistent_local_opt_gens.py Date: Tue, 19 Nov 2019 12:13:38 -0600 Subject: [PATCH 568/644] Using hyperlinks instead in APOSMM --- libensemble/gen_funcs/aposmm.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 04a457195..20c30d3f8 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -133,13 +133,14 @@ def aposmm_logic(H, persis_info, gen_specs, _): active runs, order of points in each run, etc. The allocation function must ensure it's always given. + `Python `_ + .. seealso:: - `test_branin_aposmm.py`_ for basic APOSMM usage. .. seealso:: - ``libensemble/tests/regression_tests/`` - `test_chwirut_aposmm_one_residual_at_a_time.py`_ for an example of APOSMM coordinating multiple local optimization runs for an objective with more than one component. """ @@ -374,7 +375,7 @@ def update_history_dist(H, n, gen_specs, c_flag): Updates distances/indices after new points that have been evaluated. .. seealso:: - `start_persistent_local_opt_gens.py`_ """ updated_inds = set() @@ -769,7 +770,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): .. seealso:: - `start_persistent_local_opt_gens.py`_ """ if nu > 0: @@ -893,7 +894,7 @@ def initialize_APOSMM(H, gen_specs): Computes common values every time that APOSMM is reinvoked .. seealso:: - `start_persistent_local_opt_gens.py`_ """ user_specs = gen_specs['user'] From d85c9df6efc3cc0dbeca7719bc29dcae5cbe6015 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 12:38:30 -0600 Subject: [PATCH 569/644] Using hyperlinks elsewhere --- libensemble/alloc_funcs/fast_alloc.py | 2 +- libensemble/alloc_funcs/fast_alloc_and_pausing.py | 2 +- libensemble/alloc_funcs/fast_alloc_to_aposmm.py | 2 +- libensemble/alloc_funcs/give_pregenerated_work.py | 2 +- libensemble/alloc_funcs/give_sim_work_first.py | 2 +- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 2 +- libensemble/alloc_funcs/start_only_persistent.py | 2 +- .../alloc_funcs/start_persistent_local_opt_gens.py | 2 +- libensemble/gen_funcs/aposmm.py | 2 -- libensemble/gen_funcs/persistent_aposmm.py | 11 ++++++----- libensemble/gen_funcs/persistent_uniform_sampling.py | 2 +- libensemble/gen_funcs/sampling.py | 6 +++--- libensemble/gen_funcs/uniform_or_localopt.py | 2 +- libensemble/sim_funcs/chwirut1.py | 6 ++++-- libensemble/sim_funcs/one_d_func.py | 2 +- libensemble/sim_funcs/six_hump_camel.py | 6 +++--- 16 files changed, 27 insertions(+), 26 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index 6b8e0a720..c8595207b 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -10,7 +10,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): more than ``gen_specs['user']['num_active_gen']`` active generators. .. seealso:: - ``/libensemble/tests/regression_tests/test_fast_alloc.py`` + `test_fast_alloc.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 7f0700195..c4115f669 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -22,7 +22,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): combine_component_func is larger than a known upper bound on the objective. .. seealso:: - ``/libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py`` + `test_chwirut_uniform_sampling_one_residual_at_a_time.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index 3051b0db8..94ed90c79 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -13,7 +13,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): for a 'batch_mode'. .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py`` + `test_6-hump_camel_aposmm_LD_MMA.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/give_pregenerated_work.py b/libensemble/alloc_funcs/give_pregenerated_work.py index 7d29473f3..4da5edb72 100644 --- a/libensemble/alloc_funcs/give_pregenerated_work.py +++ b/libensemble/alloc_funcs/give_pregenerated_work.py @@ -7,7 +7,7 @@ def give_pregenerated_sim_work(W, H, sim_specs, gen_specs, alloc_specs, persis_i idle workers. It is an example use case where no gen_func is used. .. seealso:: - ``/libensemble/tests/regression_tests/test_fast_alloc.py`` + `test_fast_alloc.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 2195bc67d..54a25aaa1 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -21,7 +21,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): This is the default allocation function if one is not defined. .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + `test_6-hump_camel_uniform_sampling.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 318c4ee5c..5a87850c3 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -14,7 +14,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info stopped (until some exit_criterion is satisfied). .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm.py`` + `test_6-hump_camel_persistent_aposmm.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index a62d0e5a1..3d832de45 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -11,7 +11,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): then this information is given back to the persistent generator. .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` + `test_6-hump_camel_persistent_uniform_sampling.py `_ """ Work = {} diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index dcaf31ef1..698f158b9 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -21,7 +21,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per - If no points are left, call the generation function. .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` + `test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py `_ """ Work = {} diff --git a/libensemble/gen_funcs/aposmm.py b/libensemble/gen_funcs/aposmm.py index 20c30d3f8..d83112f33 100644 --- a/libensemble/gen_funcs/aposmm.py +++ b/libensemble/gen_funcs/aposmm.py @@ -133,8 +133,6 @@ def aposmm_logic(H, persis_info, gen_specs, _): active runs, order of points in each run, etc. The allocation function must ensure it's always given. - `Python `_ - .. seealso:: `test_branin_aposmm_nlopt_and_then_scipy.py `_ for basic APOSMM usage. diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 292a3836c..fea4c2f7b 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -142,10 +142,11 @@ def aposmm(H, persis_info, gen_specs, libE_info): must ensure it's always given. .. seealso:: - ``libensemble/tests/regression_tests/test_branin_aposmm.py`` + `test_branin_aposmm_nlopt_and_then_scipy.py `_ for basic APOSMM usage. + .. seealso:: - ``libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` + `test_chwirut_aposmm_one_residual_at_a_time.py `_ for an example of APOSMM coordinating multiple local optimization runs for an objective with more than one component. """ @@ -672,7 +673,7 @@ def update_history_dist(H, n): Updates distances/indices after new points that have been evaluated. .. seealso:: - ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + `start_persistent_local_opt_gens.py `_ """ new_inds = np.where(~H['known_to_aposmm'])[0] @@ -804,7 +805,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): .. seealso:: - ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + `start_persistent_local_opt_gens.py `_ """ r_k = calc_rk(n, n_s, rk_const, ld) @@ -896,7 +897,7 @@ def initialize_APOSMM(H, user_specs, libE_info): Computes common values every time that APOSMM is reinvoked .. seealso:: - ``/libensemble/alloc_funcs/start_persistent_local_opt_gens.py`` + `start_persistent_local_opt_gens.py `_ """ n = len(user_specs['ub']) diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index 48fbdb524..6f3a826ce 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -10,7 +10,7 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): ``gen_specs['gen_batch_size']`` uniformly sampled points. .. seealso:: - ``libensemble/libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` + `test_6-hump_camel_persistent_uniform_sampling.py `_ """ ub = gen_specs['user']['ub'] lb = gen_specs['user']['lb'] diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index eae6c09e2..9dfe338aa 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -13,7 +13,7 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe and ``ranks_per_node`` to be used in the evaluation of the generated point. .. seealso:: - ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` + `test_6-hump_camel_with_different_nodes_uniform_sample.py `_ """ ub = gen_specs['user']['ub'] lb = gen_specs['user']['lb'] @@ -47,7 +47,7 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs, _): separately. .. seealso:: - ``libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py`` + `test_chwirut_uniform_sampling_one_residual_at_a_time.py `_ """ ub = gen_specs['user']['ub'] lb = gen_specs['user']['lb'] @@ -75,7 +75,7 @@ def uniform_random_sample(H, persis_info, gen_specs, _): defined by ``gen_specs['user']['ub']`` and ``gen_specs['user']['lb']``. .. seealso:: - ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py`` + `test_6-hump_camel_uniform_sampling.py `_ """ ub = gen_specs['user']['ub'] lb = gen_specs['user']['lb'] diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index 05da55ca2..a2830832a 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -16,7 +16,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): function a persistent nlopt local optimization run. .. seealso:: - ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py`` + `test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py `_ """ if libE_info.get('persistent'): diff --git a/libensemble/sim_funcs/chwirut1.py b/libensemble/sim_funcs/chwirut1.py index 8fac5e50f..856ebbaa0 100644 --- a/libensemble/sim_funcs/chwirut1.py +++ b/libensemble/sim_funcs/chwirut1.py @@ -261,10 +261,12 @@ def chwirut_eval(H, persis_info, sim_specs, _): are evaluated and returned in the ``'fvec'`` field. .. seealso:: - ``/libensemble/tests/regression_tests/test_chwirut_pounders.py`` for an example where the entire fvec is computed. + `test_chwirut_pounders.py `_ + for an example where the entire fvec is computed each call. .. seealso:: - ``/libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py`` + `test_chwirut_aposmm_one_residual_at_a_time.py `_ + for an example where one component of fvec is computed per call """ batch = len(H['x']) diff --git a/libensemble/sim_funcs/one_d_func.py b/libensemble/sim_funcs/one_d_func.py index bf02ed027..f0dadb2ec 100644 --- a/libensemble/sim_funcs/one_d_func.py +++ b/libensemble/sim_funcs/one_d_func.py @@ -11,7 +11,7 @@ def one_d_example(x, persis_info, sim_specs, _): Evaluates the six hump camel function for a single point ``x``. .. seealso:: - ``/libensemble/libensemble/tests/regression_tests/test_fast_alloc.py`` + `test_fast_alloc.py `_ """ O = np.zeros(1, dtype=sim_specs['out']) diff --git a/libensemble/sim_funcs/six_hump_camel.py b/libensemble/sim_funcs/six_hump_camel.py index 26d1b5b62..69a44a3e1 100644 --- a/libensemble/sim_funcs/six_hump_camel.py +++ b/libensemble/sim_funcs/six_hump_camel.py @@ -16,7 +16,7 @@ def six_hump_camel_with_different_ranks_and_nodes(H, persis_info, sim_specs, lib using a machinefile (to show one way of evaluating a compiled simulation). .. seealso:: - ``/libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py`` + `test_6-hump_camel_with_different_nodes_uniform_sample.py `_ """ from mpi4py import MPI @@ -61,7 +61,7 @@ def six_hump_camel(H, persis_info, sim_specs, _): defined. .. seealso:: - ``/libensemble/libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py`` + `test_6-hump_camel_aposmm_LD_MMA.py `_ """ batch = len(H['x']) @@ -84,7 +84,7 @@ def six_hump_camel_simple(x, persis_info, sim_specs, _): Evaluates the six hump camel function for a single point ``x``. .. seealso:: - ``/libensemble/libensemble/tests/regression_tests/test_fast_alloc.py`` + `test_fast_alloc.py `_ """ O = np.zeros(1, dtype=sim_specs['out']) From c929d538649f2320deb3a58f553fac5456b22dea Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 12:44:24 -0600 Subject: [PATCH 570/644] Whitespace --- libensemble/gen_funcs/sampling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index 9dfe338aa..06b35c118 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -12,7 +12,7 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe ``gen_specs['user']['lb']``. Also randomly requests a different ``number_of_nodes`` and ``ranks_per_node`` to be used in the evaluation of the generated point. - .. seealso:: + .. seealso:: `test_6-hump_camel_with_different_nodes_uniform_sample.py `_ """ ub = gen_specs['user']['ub'] @@ -46,7 +46,7 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs, _): and ``gen_specs['user']['lb']`` but requests each ``obj_component`` be evaluated separately. - .. seealso:: + .. seealso:: `test_chwirut_uniform_sampling_one_residual_at_a_time.py `_ """ ub = gen_specs['user']['ub'] @@ -74,7 +74,7 @@ def uniform_random_sample(H, persis_info, gen_specs, _): Generates ``gen_specs['user']['gen_batch_size']`` points uniformly over the domain defined by ``gen_specs['user']['ub']`` and ``gen_specs['user']['lb']``. - .. seealso:: + .. seealso:: `test_6-hump_camel_uniform_sampling.py `_ """ ub = gen_specs['user']['ub'] From 055188d20e1123dd7ad82b435d587103de9d3138 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 19 Nov 2019 13:21:06 -0600 Subject: [PATCH 571/644] Updates to Platforms --- docs/platforms/platforms_index.rst | 2 +- docs/platforms/summit.rst | 36 ++++++++++++++++++++---------- docs/platforms/theta.rst | 2 +- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 818c56671..8903175dd 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -6,7 +6,7 @@ scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly parallel scaling capabilities are best exemplified on the resources available within high-performance machines. -libEnsemble's flexible architecture lends it best to two general modes of worker +libEnsemble's flexible architecture lends itself best to two general modes of worker distributions across allocated compute nodes. The first mode we refer to as *centralized* mode, where the libEnsemble manager and worker processes are grouped on one or more nodes, but through the libEnsemble job-controller or a diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index e2307bd3a..5e20bd76a 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -3,11 +3,15 @@ Summit ====== Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing Facility. -Each of the approximately 4,600 compute nodes on Summit contains two IBM POWER9 processors and six NVIDIA Volta V100 accelerators. +Each of the approximately 4,600 compute nodes on Summit contains two IBM POWER9 +processors and six NVIDIA Volta V100 accelerators. Summit features three tiers of nodes: login, launch, and compute nodes. + Users on login nodes submit batch runs to the launch nodes. -Launch nodes execute user batch-scripts to run on the compute nodes via ``jsrun``. +Batch scripts and interactive sessions run on the launch nodes. Only the launch nodes +can submit MPI runs to the compute nodes via ``jsrun``. + Configuring Python ------------------ @@ -16,12 +20,9 @@ Begin by loading the Python 3 Anaconda module:: $ module load python -You can now create your own custom Conda_ environment:: +You can now create and activate your own custom Conda_ environment:: conda create --name myenv python=3.7 - -Now activate environment:: - export PYTHONNOUSERSITE=1 # Make sure get python from conda env . activate myenv @@ -56,8 +57,8 @@ important command is ``bsub``, for submitting batch scripts from the login nodes to execute on the Launch nodes. It is recommended to run libEnsemble on the Launch nodes (assuming workers are submitting -MPI jobs) using ``local`` comm mode (multiprocessing). In the future, Balsam may be used -to run libEnsemble on compute nodes. +MPI jobs) using the ``local`` communications mode (multiprocessing). +In the future, Balsam may be used to run libEnsemble on compute nodes. Interactive Runs ^^^^^^^^^^^^^^^^ @@ -67,8 +68,7 @@ to the following:: $ bsub -W 30 -P [project] -nnodes 8 -Is -This will place the user on a launch node. Then, to launch MPI jobs to the compute -nodes use ``jsrun`` where you would use ``mpirun``. +This will place the user on a launch node. .. note:: You will need to re-activate your conda virtual environment. @@ -79,7 +79,7 @@ Batch Runs Batch scripts specify run-settings using ``#BSUB`` statements. The following simple example depicts configuring and launching libEnsemble to a launch node with multiprocessing. This script also assumes the user is using the ``parse_args()`` -convenience function within libEnsemble's ``utils.py``. +convenience function from libEnsemble's :doc:`utils module<../utilities>`. .. code-block:: bash @@ -122,7 +122,19 @@ convenience function within libEnsemble's ``utils.py``. With this saved as ``myscript.sh``, allocating, configuring, and queueing libEnsemble on Summit becomes:: - $ bsub script myscript.sh + $ bsub myscript.sh + + +Launching user applications from libEnsemble Workers +---------------------------------------------------- + +Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. +This can be accomplished in user sim_f functions directly. However, it is highly +recommended that the :doc:`job_controller<../job_controller/overview>` interface +is used inside the sim_f (or gen_f), as this provides a portable interface +with many advantages including automatic resource detection, portability, +launch failure resilience, and ease of use. + Additional Information ---------------------- diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 094b69ea5..209f4a05c 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -135,7 +135,7 @@ Batch Runs Batch scripts specify run-settings using ``#COBALT`` statements. The following simple example depicts configuring and launching libEnsemble to a MOM node with multiprocessing. This script also assumes the user is using the ``parse_args()`` -convenience function within libEnsemble's ``utils.py``. +convenience function from libEnsemble's :doc:`utils module<../utilities>`. .. code-block:: bash From cf03733ba86c2fea79431d7a53efaada4f947941 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 14:01:45 -0600 Subject: [PATCH 572/644] wrapping adjust --- docs/platforms/summit.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 5e20bd76a..081794d12 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -2,15 +2,15 @@ Summit ====== -Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing Facility. -Each of the approximately 4,600 compute nodes on Summit contains two IBM POWER9 -processors and six NVIDIA Volta V100 accelerators. +Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing +Facility. Each of the approximately 4,600 compute nodes on Summit contains two +IBM POWER9 processors and six NVIDIA Volta V100 accelerators. Summit features three tiers of nodes: login, launch, and compute nodes. Users on login nodes submit batch runs to the launch nodes. -Batch scripts and interactive sessions run on the launch nodes. Only the launch nodes -can submit MPI runs to the compute nodes via ``jsrun``. +Batch scripts and interactive sessions run on the launch nodes. Only the launch +nodes can submit MPI runs to the compute nodes via ``jsrun``. Configuring Python @@ -26,9 +26,9 @@ You can now create and activate your own custom Conda_ environment:: export PYTHONNOUSERSITE=1 # Make sure get python from conda env . activate myenv -If you are installing any packages with extensions, ensure the correct compiler module -is loaded. If using mpi4py_, this must be installed from source, referencing the compiler. -At time of writing, mpi4py must be built with gcc:: +If you are installing any packages with extensions, ensure the correct compiler +module is loaded. If using mpi4py_, this must be installed from source, +referencing the compiler. At time of writing, mpi4py must be built with gcc:: module load gcc @@ -56,15 +56,15 @@ Summit uses LSF_ for job management and submission. For libEnsemble, the most important command is ``bsub``, for submitting batch scripts from the login nodes to execute on the Launch nodes. -It is recommended to run libEnsemble on the Launch nodes (assuming workers are submitting -MPI jobs) using the ``local`` communications mode (multiprocessing). +It is recommended to run libEnsemble on the Launch nodes (assuming workers are +submitting MPI jobs) using the ``local`` communications mode (multiprocessing). In the future, Balsam may be used to run libEnsemble on compute nodes. Interactive Runs ^^^^^^^^^^^^^^^^ -Users can run interactively with ``bsub`` by specifying the ``-Is`` flag, similarly -to the following:: +Users can run interactively with ``bsub`` by specifying the ``-Is`` flag, +similarly to the following:: $ bsub -W 30 -P [project] -nnodes 8 -Is From c5bb8db6bd87636fdcd05d1f8c282a2cac62a4a7 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 14:16:10 -0600 Subject: [PATCH 573/644] formatting, clarify advantages of job controller --- docs/platforms/summit.rst | 6 +++--- docs/platforms/theta.rst | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 081794d12..1cdf7caec 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -129,11 +129,11 @@ Launching user applications from libEnsemble Workers ---------------------------------------------------- Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. -This can be accomplished in user sim_f functions directly. However, it is highly +This can be accomplished in user ``sim_f`` functions directly. However, it is highly recommended that the :doc:`job_controller<../job_controller/overview>` interface -is used inside the sim_f (or gen_f), as this provides a portable interface +is used inside the ``sim_f`` or ``gen_f``, as this provides a portable interface with many advantages including automatic resource detection, portability, -launch failure resilience, and ease of use. +launch failure resilience, and ease-of-use. Additional Information diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 209f4a05c..23d45e72d 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -89,14 +89,14 @@ On Theta, libEnsemble can be launched to two locations: 1. **A MOM Node**: All of libEnsemble's manager and worker processes run on a front-end MOM node. libEnsemble's MPI job-controller takes - responsibility for direct job-submission to allocated compute nodes. + responsibility for direct user-application submission to allocated compute nodes. libEnsemble must be configured to run with *multiprocessing* communications, since mpi4py isn't configured for use on the MOM nodes. 2. **The Compute Nodes**: libEnsemble is submitted to Balsam and all manager and worker processes are tasked to a backend compute node. libEnsemble's Balsam job-controller interfaces with Balsam running on a MOM node for dynamic - task submission to the compute nodes. + user-application submission to the compute nodes. .. image:: ../images/combined_ThS.png :alt: central_MOM @@ -107,6 +107,12 @@ When considering on which nodes to run libEnsemble, consider if your user functions execute computationally expensive code, or code built for specific architectures. Recall also that only the MOM nodes can launch MPI jobs. +Although libEnsemble workers on the MOM nodes can technically submit +user-applications to the compute nodes via ``aprun`` within user functions, it +is highly recommended that the aforementioned :doc:`job_controller<../job_controller/overview>` +interface is used instead. The libEnsemble job-controller features advantages like +automatic resource-detection, portability, launch failure resilience, and ease-of-use. + Theta features one default production queue, ``default``, and two debug queues, ``debug-cache-quad`` and ``debug-flat-quad``. From 4c72258bb78a0ba569eaaa9527b640be288fd302 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 15:19:55 -0600 Subject: [PATCH 574/644] Trying to see if we can get bibliography to build on RTD --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b45b78d79..990ddf338 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ def __getattr__(cls, name): # If your documentation needs a minimal Sphinx version, state it here. # -# needs_sphinx = '2.0' +needs_sphinx = '3.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom From ef595b8961f6012dee778a17e1fdfd5ba23c7ccb Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 15:20:22 -0600 Subject: [PATCH 575/644] rename welcome link --- docs/welcome.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/welcome.rst b/docs/welcome.rst index 7decf8888..47d45369e 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -27,5 +27,5 @@ libEnsemble is a library to coordinate the concurrent evaluation of dynamic ense * New to libEnsemble? Start :doc:`here`. * Try out libEnsemble with a :doc:`tutorial`. -* Go in-depth by reading the :doc:`User Guide`. +* Go in-depth by reading the :doc:`Overview`. * Check the :doc:`FAQ` for common questions and answers, errors and resolutions. From 228827cb7f2f381e86c872ec0674f4c93f6da645 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 15:24:58 -0600 Subject: [PATCH 576/644] Making like to flake8 file --- CONTRIBUTING.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index cb40685d2..86d20f1fc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,9 +8,7 @@ Contributions may be made via GitHub pull request to: libEnsemble uses the Gitflow model. Contributors should branch from, and make pull requests to, the develop branch. The master branch is used only for releases. Code should pass flake8 tests, allowing for the exceptions -given in the ``.flake8`` file in the project directory: - -.. literalinclude:: ../.flake8 +given in the flake8_ file in the project directory. Issues can be raised at: @@ -66,3 +64,5 @@ following: personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. + +.. _flake8: https://github.com/Libensemble/libensemble/blob/develop/.flake8 From 8f0aebdb51e5a1a01473bdc55815e4b0432f73cf Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 15:31:35 -0600 Subject: [PATCH 577/644] Trying sphinx3 --- conda/environment.yml | 3 ++- docs/conf.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conda/environment.yml b/conda/environment.yml index 0637adca0..e6d8c8c8b 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -4,4 +4,5 @@ channels: dependencies: - sphinxcontrib-bibtex - - python>=3.6 \ No newline at end of file + - sphinx>=3 + - python>=3.6 diff --git a/docs/conf.py b/docs/conf.py index 990ddf338..fadab367f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ def __getattr__(cls, name): # If your documentation needs a minimal Sphinx version, state it here. # -needs_sphinx = '3.0' +needs_sphinx = '2.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom From 14bd8bae3aec38e6acc3a6f1a35146be1d88e0b4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 15:34:34 -0600 Subject: [PATCH 578/644] Trying sphinx3 --- conda/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environment.yml b/conda/environment.yml index e6d8c8c8b..f6716c846 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -4,5 +4,5 @@ channels: dependencies: - sphinxcontrib-bibtex - - sphinx>=3 + - sphinx-build>=3 - python>=3.6 From d42981573ed1d25fdd1ceb5718858b7cd8e26d92 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 19 Nov 2019 15:37:10 -0600 Subject: [PATCH 579/644] Trying sphinx3 --- conda/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/environment.yml b/conda/environment.yml index f6716c846..01fd94b91 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -4,5 +4,4 @@ channels: dependencies: - sphinxcontrib-bibtex - - sphinx-build>=3 - python>=3.6 From 4bef803a6401fd7bc3dc8094d2b320450bf59c3c Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 16:16:44 -0600 Subject: [PATCH 580/644] update tutorials with persis_info and random streams util --- docs/tutorials/local_sine_tutorial.rst | 30 ++++++++++------------ examples/tutorials/tutorial_calling.py | 11 +++----- examples/tutorials/tutorial_calling_mpi.py | 17 +++++------- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 1c64bdb12..af92e7c15 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -3,7 +3,8 @@ Simple Local Sine Tutorial ========================== This introductory tutorial demonstrates the capability to perform ensembles of -calculations in parallel using :doc:`libEnsemble<../introduction>` with Python's Multiprocessing. +calculations in parallel using :doc:`libEnsemble<../introduction>` with Python's +Multiprocessing. The foundation of writing libEnsemble routines is accounting for four components: @@ -13,17 +14,18 @@ The foundation of writing libEnsemble routines is accounting for four components 4. The calling script, which defines parameters and information about these functions and the libEnsemble task, then begins execution. libEnsemble initializes a *manager* process and as many *worker* processes as the -user requests. The manager coordinates data-transfer between workers and assigns each -units of work, consisting of a ``gen_f`` or ``sim_f`` function to run and +user requests. The manager coordinates data-transfer between workers and assigns +each units of work, consisting of a ``gen_f`` or ``sim_f`` function to run and accompanying data. These functions can perform their work in-line with Python or by launching and controlling user-applications with a :ref:`job controller`. Finally, workers pass results back to the manager. -For this tutorial, we'll write our ``gen_f`` and ``sim_f`` entirely in Python without -other applications. Our ``gen_f`` will produce uniform randomly-sampled -values, and our ``sim_f`` will calculate the sine of each. By default we don't need to -write a new allocation function. All generated and simulated values alongside other -parameters are stored in :ref:`H`, the History array. +For this tutorial, we'll write our ``gen_f`` and ``sim_f`` entirely in Python +without other applications. Our ``gen_f`` will produce uniform randomly-sampled +values, and our ``sim_f`` will calculate the sine of each. By default we don't +need to write a new allocation function. All generated and simulated values +alongside other parameters are stored in :ref:`H`, +the History array. .. _libEnsemble: https://libensemble.readthedocs.io/en/latest/quickstart.html @@ -199,18 +201,14 @@ inputs and outputs from those functions to expect. Recall that each worker is assigned an entry in the :ref:`persis_info` dictionary that, in this tutorial, contains a ``RandomState()`` random stream for -uniform random sampling. We populate that dictionary here. Finally, we specify the -circumstances where libEnsemble should stop execution in :ref:`exit_criteria`. +uniform random sampling. We populate that dictionary here using a utility from +the :doc:`utils module<../utilities>`. Finally, we specify the circumstances +where libEnsemble should stop execution in :ref:`exit_criteria`. .. code-block:: python :linenos: - persis_info = {} - - for i in range(1, nworkers+1): # Worker numbers start at 1 - persis_info[i] = { - 'rand_stream': np.random.RandomState(i), - 'worker_num': i} + persis_info = add_unique_random_streams({}, nworkers+1) # Worker numbers start at 1 exit_criteria = {'sim_max': 80} # Stop libEnsemble after 80 simulations diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index 88757f53b..ac8620502 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -1,6 +1,7 @@ import numpy as np import matplotlib.pyplot as plt from libensemble.libE import libE +from libensemble.utils import add_unique_random_streams from tutorial_gen import gen_random_sample from tutorial_sim import sim_find_sine @@ -19,19 +20,15 @@ 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') -persis_info = {} - -for i in range(1, nworkers+1): # Worker numbers start at 1. - persis_info[i] = { - 'rand_stream': np.random.RandomState(i), - 'worker_num': i} +persis_info = add_unique_random_streams({}, nworkers+1) # Worker numbers start at 1 exit_criteria = {'sim_max': 80} # Stop libEnsemble after 80 simulations H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) -print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array +# Some (optional) statements to visualize our History array +print([i for i in H.dtype.fields]) print(H) colors = ['b', 'g', 'r', 'y', 'm', 'c', 'k', 'w'] diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 82805aee6..4b71c4d50 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -1,14 +1,14 @@ import numpy as np import matplotlib.pyplot as plt from libensemble.libE import libE +from libensemble.utils import add_unique_random_streams from tutorial_gen import gen_random_sample from tutorial_sim import sim_find_sine from mpi4py import MPI -# nworkers = 4 # nworkers will come from MPI libE_specs = {'comms': 'mpi'} # 'nworkers' removed, 'comms' now 'mpi' -nworkers = MPI.COMM_WORLD.Get_size() - 1 +nworkers = MPI.COMM_WORLD.Get_size() - 1 # one process belongs to manager is_master = (MPI.COMM_WORLD.Get_rank() == 0) # master process has MPI rank 0 gen_specs = {'gen_f': gen_random_sample, # Our generator function @@ -23,20 +23,17 @@ 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') -persis_info = {} - -for i in range(1, nworkers+1): # Worker numbers start at 1. - persis_info[i] = { - 'rand_stream': np.random.RandomState(i), - 'worker_num': i} +persis_info = add_unique_random_streams({}, nworkers+1) # Worker numbers start at 1 exit_criteria = {'sim_max': 80} # Stop libEnsemble after 80 simulations H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) -if is_master: # Only the master process should execute this - print([i for i in H.dtype.fields]) # Some (optional) statements to visualize our History array +# Some (optional) statements to visualize our History array +# Only the master process should execute this +if is_master: + print([i for i in H.dtype.fields]) print(H) colors = ['b', 'g', 'r', 'y', 'm', 'c', 'k', 'w'] From c9c4d76482215b1e84a7bab18996b7fb42ad211f Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 19 Nov 2019 17:20:44 -0600 Subject: [PATCH 581/644] Populate calling scripts, adjust calling script for clarity --- docs/examples/calling_scripts.rst | 23 ++++++++++++++++++- .../script_test_balsam_hworld.py | 8 +++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index 9a975ecb4..5efb5b7a7 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -1,4 +1,25 @@ Calling Scripts =============== -Below are example calling scripts +Below are example calling scripts used to populate specifications for each user +function and libEnsemble before initiating libEnsemble via the primary ``libE()`` +call. + +Local Sine Tutorial +------------------- + +This example is from the Local Sine :doc:`Tutorial<../tutorials/local_sine_tutorial>`, +meant to run with Python's multiprocessing as the primary ``comms`` method. + +.. literalinclude:: ../../examples/tutorials/tutorial_calling.py + :language: python + +Balsam Job Controller +--------------------- + +This example is from the regression tests. It configures MPI as the primary +communication method, and demonstrates registering a user-application with the +libEnsemble job controller to be launched by the job controller within the ``sim_f``. + +.. literalinclude:: ../../libensemble/tests/regression_tests/script_test_balsam_hworld.py + :language: python diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 47bce6751..fc621a2d9 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -9,8 +9,8 @@ from libensemble.balsam_controller import BalsamJobController from libensemble.message_numbers import WORKER_DONE, WORKER_KILL_ON_ERR, WORKER_KILL_ON_TIMEOUT, JOB_FAILED from libensemble.libE import libE -from libensemble.sim_funcs.job_control_hworld import job_control_hworld as sim_f -from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.sim_funcs.job_control_hworld import job_control_hworld +from libensemble.gen_funcs.sampling import uniform_random_sample from libensemble.utils import add_unique_random_streams mpi4py.rc.recv_mprobe = False # Disable matching probes @@ -42,12 +42,12 @@ def build_simfunc(): jobctrl = BalsamJobController(auto_resources=False) jobctrl.register_calc(full_path=sim_app, calc_type='sim') -sim_specs = {'sim_f': sim_f, +sim_specs = {'sim_f': job_control_hworld, 'in': ['x'], 'out': [('f', float), ('cstat', int)], 'user': {'cores': cores_per_job}} -gen_specs = {'gen_f': gen_f, +gen_specs = {'gen_f': uniform_random_sample, 'in': ['sim_id'], 'out': [('x', float, (2,))], 'user': {'lb': np.array([-3, -2]), From 2adf1ec61df57ce7438395c5a5245940d15cf1a2 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 19 Nov 2019 19:12:33 -0600 Subject: [PATCH 582/644] Document calc_status --- docs/data_structures/calc_status.rst | 74 ++++++++++++++++++++++++ docs/data_structures/data_structures.rst | 1 + docs/sim_gen_alloc_funcs.rst | 14 ++--- libensemble/message_numbers.py | 17 +++--- 4 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 docs/data_structures/calc_status.rst diff --git a/docs/data_structures/calc_status.rst b/docs/data_structures/calc_status.rst new file mode 100644 index 000000000..76bec7fd2 --- /dev/null +++ b/docs/data_structures/calc_status.rst @@ -0,0 +1,74 @@ +.. _datastruct-calc-status: + +calc_status +=========== + +The ``calc_status`` is an integer attribute with a corresponding description that can be used in :ref:`sim_f` or :ref:`gen_f` functions to capture the status of a calcuation. This is returned to the manager and printed to the ``libE_stats.txt`` file. This is not currently used by the manager, but can still provide a useful summary in libE_stats.txt. The user determines the status of the calculation, as it could include multiple application runs. It can be added as a third return variable in sim_f or gen_f functions. +The calc_status codes are in the ``libensemble.message_numbers`` module. + +Example of ``calc_status`` used along with :ref:`job controller` in sim_f: + +.. code-block:: python + :linenos: + :emphasize-lines: 4,16,19,22,30 + + from libensemble.message_numbers import WORKER_DONE, WORKER_KILL, JOB_FAILED + + job = jobctl.launch(calc_type='sim', num_procs=cores, wait_on_run=True) + calc_status = UNSET_TAG + poll_interval = 1 # secs + while(not job.finished): + if job.runtime > time_limit: + job.kill() # Timeout + else: + time.sleep(poll_interval) + job.poll() + + if job.finished: + if job.state == 'FINISHED': + print("Job {} completed".format(job.name)) + calc_status = WORKER_DONE + elif job.state == 'FAILED': + print("Warning: Job {} failed: Error code {}".format(job.name, job.errcode)) + calc_status = JOB_FAILED + elif job.state == 'USER_KILLED': + print("Warning: Job {} has been killed".format(job.name)) + calc_status = WORKER_KILL + else: + print("Warning: Job {} in unknown state {}. Error code {}".format(job.name, job.state, job.errcode)) + + outspecs = sim_specs['out'] + output = np.zeros(1, dtype=outspecs) + output['energy'][0] = final_energy + + return output, persis_info, calc_status + +See forces_simf.py_ for a complete example. + +Available values: + +.. literalinclude:: ../../libensemble/message_numbers.py + :start-after: first_calc_status_rst_tag + :end-before: last_calc_status_rst_tag + +The corresponding messages printed to the libE_stats.txt file are: + +.. literalinclude:: ../../libensemble/message_numbers.py + :start-at: calc_status_strings + :end-before: last_calc_status_string_rst_tag + +Example segment of libE_stats.txt: + +.. code-block:: console + + Worker 1: Calc 0: gen Time: 0.00 Start: 2019-11-19 18:53:43 End: 2019-11-19 18:53:43 Status: Not set + Worker 1: Calc 1: sim Time: 4.41 Start: 2019-11-19 18:53:43 End: 2019-11-19 18:53:48 Status: Worker killed + Worker 2: Calc 0: sim Time: 5.42 Start: 2019-11-19 18:53:43 End: 2019-11-19 18:53:49 Status: Completed + Worker 1: Calc 2: sim Time: 2.41 Start: 2019-11-19 18:53:48 End: 2019-11-19 18:53:50 Status: Worker killed + Worker 2: Calc 1: sim Time: 2.41 Start: 2019-11-19 18:53:49 End: 2019-11-19 18:53:51 Status: Worker killed + Worker 1: Calc 3: sim Time: 4.41 Start: 2019-11-19 18:53:50 End: 2019-11-19 18:53:55 Status: Completed + Worker 2: Calc 2: sim Time: 4.41 Start: 2019-11-19 18:53:51 End: 2019-11-19 18:53:56 Status: Completed + Worker 1: Calc 4: sim Time: 4.41 Start: 2019-11-19 18:53:55 End: 2019-11-19 18:53:59 Status: Completed + Worker 2: Calc 3: sim Time: 4.41 Start: 2019-11-19 18:53:56 End: 2019-11-19 18:54:00 Status: Completed + +.. _forces_simf.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/forces_simf.py diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index fff56d816..4831393ab 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -34,3 +34,4 @@ function. history_array worker_array work_dict + calc_status diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index 09780d0ab..7a87ab3e4 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -71,11 +71,10 @@ Returns: **persis_info**: :obj:`dict` :doc:`(example)` - **calc_tag**: :obj:`int`, optional - Used to tell manager why a persistent worker is stopping. + **calc_status**: :obj:`int`, optional. + Provides a job status to the Manager. Currently only used to output to libE_stats.txt file. + :doc:`(example)` -.. literalinclude:: ../libensemble/message_numbers.py - :end-before: last_message_number_rst_tag gen_f API ~~~~~~~~~ @@ -110,11 +109,10 @@ Returns: **persis_info**: :obj:`dict` :doc:`(example)` - **calc_tag**: :obj:`int`, optional - Used to tell manager why a persistent worker is stopping. + **calc_status**: :obj:`int`, optional. + Provides a job status to the Manager. Currently only used to output to libE_stats.txt file. + :doc:`(example)` -.. literalinclude:: ../libensemble/message_numbers.py - :end-before: last_message_number_rst_tag alloc_f API ~~~~~~~~~~~ diff --git a/libensemble/message_numbers.py b/libensemble/message_numbers.py index 59e747179..80c87cf0d 100644 --- a/libensemble/message_numbers.py +++ b/libensemble/message_numbers.py @@ -1,6 +1,6 @@ # --- Tags -UNSET_TAG = 0 # sh temp - this is a libE feature to be reviewed for best solution +UNSET_TAG = 0 EVAL_SIM_TAG = 1 EVAL_GEN_TAG = 2 STOP_TAG = 3 @@ -17,17 +17,17 @@ # --- Signal flags (in message body vs tags) - -# CALC STATUS/SIGNAL FLAGS: In future these will be in a data structure +# first_calc_status_rst_tag +# CALC STATUS/SIGNAL FLAGS MAN_SIGNAL_FINISH = 20 # Kill jobs and shutdown worker MAN_SIGNAL_KILL = 21 # Kill running job - but don't stop worker - -WORKER_KILL = 30 # Worker kills not covered by a more specific case +WORKER_KILL = 30 # Worker kills not covered by a more specific case WORKER_KILL_ON_ERR = 31 WORKER_KILL_ON_TIMEOUT = 32 -JOB_FAILED = 33 -WORKER_DONE = 34 -CALC_EXCEPTION = 35 +JOB_FAILED = 33 # Calc had jobs that failed +WORKER_DONE = 34 # Calculation was successful +CALC_EXCEPTION = 35 # Reserved: Automatically used if gen_f or sim_f raised an exception. +# last_calc_status_rst_tag calc_status_strings = { MAN_SIGNAL_FINISH: "Manager killed on finish", @@ -40,3 +40,4 @@ CALC_EXCEPTION: "Exception occurred", None: "Unknown Status" } +# last_calc_status_string_rst_tag From b3afda8f75de11159261fd7856b8c8259a371d7c Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 10:11:02 -0600 Subject: [PATCH 583/644] add line numbers and line emphasis --- docs/examples/calling_scripts.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index 5efb5b7a7..bf6ab5ec3 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -13,6 +13,8 @@ meant to run with Python's multiprocessing as the primary ``comms`` method. .. literalinclude:: ../../examples/tutorials/tutorial_calling.py :language: python + :linenos: + :emphasize-lines: 8-28 Balsam Job Controller --------------------- @@ -23,3 +25,5 @@ libEnsemble job controller to be launched by the job controller within the ``sim .. literalinclude:: ../../libensemble/tests/regression_tests/script_test_balsam_hworld.py :language: python + :linenos: + :emphasize-lines: 27-31, 33-34, 42-43, 45-57, 59-65 From 0d232e7a031a8d1fdaaf936a38b8ab4310dcbf87 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 10:46:01 -0600 Subject: [PATCH 584/644] revise intro, add third script --- docs/examples/calling_scripts.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index bf6ab5ec3..bebfb382e 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -3,7 +3,9 @@ Calling Scripts Below are example calling scripts used to populate specifications for each user function and libEnsemble before initiating libEnsemble via the primary ``libE()`` -call. +call. The primary libEnsemble-relevant portions have been highlighted in each +example. Non-highlighted portions may include setup routines, compilation steps +for user-applications, or output processing. Local Sine Tutorial ------------------- @@ -26,4 +28,17 @@ libEnsemble job controller to be launched by the job controller within the ``sim .. literalinclude:: ../../libensemble/tests/regression_tests/script_test_balsam_hworld.py :language: python :linenos: - :emphasize-lines: 27-31, 33-34, 42-43, 45-57, 59-65 + :emphasize-lines: 27-34, 42-65 + +6-Hump-Camel Persistent APOSMM +------------------------------ + +This example is also from the regression tests, and demonstrates configuring a +persistent run via a custom allocation function. Note the use of the +``parse_args()`` and ``save_libE_output()`` convenience functions from the +:doc:`utilities<../utilities>`. + +.. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py + :language: python + :linenos: + :emphasize-lines: 30, 38-69 From b8628c09609d099656be3732507f555285e86483 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 10:48:44 -0600 Subject: [PATCH 585/644] emphasize save() --- docs/examples/calling_scripts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index bebfb382e..6b45d754d 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -41,4 +41,4 @@ persistent run via a custom allocation function. Note the use of the .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py :language: python :linenos: - :emphasize-lines: 30, 38-69 + :emphasize-lines: 30, 38-69, 82 From 73e9b63690f5722ccc45728f0948643d470cb0ca Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 11:14:21 -0600 Subject: [PATCH 586/644] adds more captions throughout --- docs/examples/calling_scripts.rst | 3 +++ docs/platforms/example_scripts.rst | 4 ++++ docs/tutorials/local_sine_tutorial.rst | 2 ++ 3 files changed, 9 insertions(+) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index 6b45d754d..edd19c717 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -15,6 +15,7 @@ meant to run with Python's multiprocessing as the primary ``comms`` method. .. literalinclude:: ../../examples/tutorials/tutorial_calling.py :language: python + :caption: examples/tutorials/tutorial_calling.py :linenos: :emphasize-lines: 8-28 @@ -27,6 +28,7 @@ libEnsemble job controller to be launched by the job controller within the ``sim .. literalinclude:: ../../libensemble/tests/regression_tests/script_test_balsam_hworld.py :language: python + :caption: tests/regression_tests/script_test_balsam_hworld.py :linenos: :emphasize-lines: 27-34, 42-65 @@ -40,5 +42,6 @@ persistent run via a custom allocation function. Note the use of the .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py :language: python + :caption: tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py :linenos: :emphasize-lines: 30, 38-69, 82 diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 3c9fe8d1b..c6b300bb6 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -9,22 +9,26 @@ Bebop - Central Mode -------------------- .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh + :caption: /examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh :language: bash Bebop - Distributed Mode ------------------------ .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm.sh + :caption: /examples/job_submission_scripts/bebop_submit_slurm.sh :language: bash Blues ----- .. literalinclude:: ../../examples/job_submission_scripts/blues_script.pbs + :caption: /examples/job_submission_scripts/blues_script.pbs :language: bash Theta - Central Mode with Balsam -------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/theta_submit_balsam.sh + :caption: /examples/job_submission_scripts/theta_submit_balsam.sh :language: bash diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index af92e7c15..30a8beed6 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -90,6 +90,7 @@ For now, create a new Python file named ``generator.py``. Write the following: .. code-block:: python :linenos: + :caption: examples/tutorials/tutorial_gen.py import numpy as np @@ -133,6 +134,7 @@ Create a new Python file named ``simulator.py``. Write the following: .. code-block:: python :linenos: + :caption: examples/tutorials/tutorial_sim.py import numpy as np From 18fc1aacc09c6ec780e5e51e5523d750c8418be8 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 11:48:18 -0600 Subject: [PATCH 587/644] Add cori user guide --- docs/data_structures/calc_status.rst | 2 +- docs/platforms/bebop.rst | 20 +++-- docs/platforms/cori.rst | 125 +++++++++++++++++++++++++++ docs/platforms/platforms_index.rst | 5 +- 4 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 docs/platforms/cori.rst diff --git a/docs/data_structures/calc_status.rst b/docs/data_structures/calc_status.rst index 76bec7fd2..330d2d726 100644 --- a/docs/data_structures/calc_status.rst +++ b/docs/data_structures/calc_status.rst @@ -51,7 +51,7 @@ Available values: :start-after: first_calc_status_rst_tag :end-before: last_calc_status_rst_tag -The corresponding messages printed to the libE_stats.txt file are: +The corresponding messages printed to the ``libE_stats.txt`` file are: .. literalinclude:: ../../libensemble/message_numbers.py :start-at: calc_status_strings diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 75e1fb583..15c0fca8d 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -50,18 +50,21 @@ You can allocate four Knights Landing nodes for thirty minutes through the follo salloc -N 4 -p knl -A [username OR project] -t 00:30:00 -With your nodes allocated, queue your job to start with five MPI ranks:: +With your nodes allocated, queue your job to start with four MPI ranks:: srun -n 4 python calling.py ``mpirun`` should also work. This line launches libEnsemble with a manager and **three** workers to one allocated compute node, with three nodes available for the workers to launch calculations with the job-controller or a job-launch command. +This is an example of running in :doc:`centralized` mode and, +if using the :doc:`job_controller<../job_controller/mpi_controller.rst>`, it should +be intiated with ``central_mode=True`` .. note:: - When performing an MPI libEnsemble run and not oversubscribing, specify one - more MPI process than the number of allocated nodes. The Manager and first - worker run together on a node. + When performing a :doc:`distributed` MPI libEnsemble run + and not oversubscribing, specify one more MPI process than the number of + allocated nodes. The Manager and first worker run together on a node. If you would like to interact directly with the compute nodes via a shell, the following showcases starting a bash session on a Knights Landing node @@ -77,7 +80,8 @@ Batch Runs ^^^^^^^^^^ Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example -for a libEnsemble use-case running on Broadwell nodes resembles the following: +for a libEnsemble use-case running in :doc:`distributed` MPI +mode on Broadwell nodes resembles the following: .. code-block:: bash :linenos: @@ -97,13 +101,16 @@ for a libEnsemble use-case running on Broadwell nodes resembles the following: cat node_list >> machinefile.$SLURM_JOBID export SLURM_HOSTFILE=machinefile.$SLURM_JOBID - srun --ntasks 5 python3 calling_script.py + srun --ntasks 5 python calling_script.py With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble on Bebop becomes:: sbatch myscript.sh +Example submission scripts for running on Bebop in distributed and centralized mode +are also given in the examples_ directory. + Debugging Strategies -------------------- @@ -122,3 +129,4 @@ See the LCRC Bebop docs here_ for more information about Bebop. .. _Slurm: https://slurm.schedmd.com/ .. _here: https://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs/running-jobs-on-bebop/ .. _options: https://slurm.schedmd.com/srun.html +.. _examples: https://github.com/Libensemble/libensemble/tree/develop/examples/job_submission_scripts diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst new file mode 100644 index 000000000..65f7eae71 --- /dev/null +++ b/docs/platforms/cori.rst @@ -0,0 +1,125 @@ +==== +Cori +==== + +Cori_ is a Cray XC40 located at NERSC, featuring both Intel Haswell +and Knights Landing compute nodes. It uses the SLURM schedular to submit +jobs from login nodes to run on the compute nodes. + +Configuring Python +------------------ + +Begin by loading the Python 3 Anaconda_ module:: + + module load python + +Create a Conda_ virtual environment in which to install libEnsemble and all +dependencies:: + + conda config --add channels intel + conda create --name my_env intelpython3_core python=3 + source activate my_env + +Installing libEnsemble and Dependencies +--------------------------------------- + +You should have an indication that the virtual environment is activated. +Install mpi4py_ and libEnsemble in this environment, making sure to reference +the Cray compiler wrappers. Your prompt should be similar to the +following block: + +.. code-block:: console + + (my_env) user@cori07:~$ CC=cc MPICC=cc pip install mpi4py --no-binary mpi4py + (my_env) user@cori07:~$ pip install libensemble + +Job Submission +-------------- + +Cori uses Slurm_ for job submission and management. The two commands you'll +likely use the most to run jobs are ``srun`` and ``sbatch`` for running +interactively and batch, respectively. libEnsemble runs on the compute nodes +only Cori using either ``multi-processing`` or ``mpi4py``. + +Interactive Runs +^^^^^^^^^^^^^^^^ + +You can allocate four Knights Landing nodes for thirty minutes through the following:: + + salloc -N 4 -C knl -q interactive -t 00:30:00 + +With your nodes allocated, queue your job to start with four MPI ranks:: + + srun -n 4 python calling.py + +This line launches libEnsemble with a manager and **three** workers to one +allocated compute node, with three nodes available for the workers to launch +user applications with the job-controller or a job-launch command. + +This is an example of running in :doc:`centralized` mode and, +if using the :doc:`job_controller<../job_controller/mpi_controller.rst>`, it should +be intiated with ``central_mode=True`` + +.. note:: + When performing a :doc:`distributed` MPI libEnsemble run, + specify one more MPI process than the number of allocated nodes. + The Manager and first worker run together on a node. + +.. note:: + You will need to re-activate your conda virtual environment and reload your + modules! Configuring this routine to occur automatically is recommended. + +Batch Runs +^^^^^^^^^^ + +Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example +for a libEnsemble use-case running in :doc:`distributed` MPI +mode on KNL nodes resembles the following: + +.. code-block:: bash + :linenos: + + #!/bin/bash + #SBATCH -J myjob + #SBATCH -N 4 + #SBATCH -q debug + #SBATCH -A myproject + #SBATCH -o myjob.out + #SBATCH -e myjob.error + #SBATCH -t 00:15:00 + #SBATCH -C knl + + # These four lines construct a machinefile for the job controller and slurm + srun hostname | sort -u > node_list + head -n 1 node_list > machinefile.$SLURM_JOBID + cat node_list >> machinefile.$SLURM_JOBID + export SLURM_HOSTFILE=machinefile.$SLURM_JOBID + + srun --ntasks 5 python calling_script.py + +With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble +on Cori becomes:: + + sbatch myscript.sh + +Example submission scripts are also given in the examples_ directory. + +Debugging Strategies +-------------------- + +View the status of your submitted jobs with ``squeue`` and cancel jobs with +``scancel [Job ID]``. + +Additional Information +---------------------- + +See the LCRC Bebop docs here_ for more information about Cori. + +.. _Cori: https://docs.nersc.gov/systems/cori/ +.. _Anaconda: https://www.anaconda.com/distribution/ +.. _Conda: https://conda.io/en/latest/ +.. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ +.. _Slurm: https://slurm.schedmd.com/ +.. _here: https://docs.nersc.gov/jobs/ +.. _options: https://slurm.schedmd.com/srun.html +.. _examples: https://github.com/Libensemble/libensemble/tree/develop/examples/job_submission_scripts diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 8903175dd..6abb5b76b 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -8,7 +8,7 @@ within high-performance machines. libEnsemble's flexible architecture lends itself best to two general modes of worker distributions across allocated compute nodes. The first mode we refer -to as *centralized* mode, where the libEnsemble manager and worker processes +to as **centralized** mode, where the libEnsemble manager and worker processes are grouped on one or more nodes, but through the libEnsemble job-controller or a job-launch command can execute calculations on the other allocated nodes: @@ -17,7 +17,7 @@ job-launch command can execute calculations on the other allocated nodes: :scale: 75 :align: center -Alternatively, in *distributed* mode, each worker process runs independently of +Alternatively, in **distributed** mode, each worker process runs independently of other workers directly on one or more allocated nodes: .. image:: ../images/distributed_Bb.png @@ -42,6 +42,7 @@ Read more about configuring and launching libEnsemble on some HPC systems: :titlesonly: bebop + cori theta summit example_scripts From 93bba0a0fb6c062f7cf5e3789293aecdcaec8b25 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 20 Nov 2019 12:40:31 -0600 Subject: [PATCH 588/644] Better note --- examples/tutorials/tutorial_calling.py | 2 +- examples/tutorials/tutorial_calling_mpi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tutorials/tutorial_calling.py b/examples/tutorials/tutorial_calling.py index ac8620502..682fae7e9 100644 --- a/examples/tutorials/tutorial_calling.py +++ b/examples/tutorials/tutorial_calling.py @@ -20,7 +20,7 @@ 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') -persis_info = add_unique_random_streams({}, nworkers+1) # Worker numbers start at 1 +persis_info = add_unique_random_streams({}, nworkers+1) # Intitialize manager/workers random streams exit_criteria = {'sim_max': 80} # Stop libEnsemble after 80 simulations diff --git a/examples/tutorials/tutorial_calling_mpi.py b/examples/tutorials/tutorial_calling_mpi.py index 4b71c4d50..423f4b59a 100644 --- a/examples/tutorials/tutorial_calling_mpi.py +++ b/examples/tutorials/tutorial_calling_mpi.py @@ -23,7 +23,7 @@ 'in': ['x'], # Input field names. 'x' from gen_f output 'out': [('y', float)]} # sim_f output. 'y' = sine('x') -persis_info = add_unique_random_streams({}, nworkers+1) # Worker numbers start at 1 +persis_info = add_unique_random_streams({}, nworkers+1) # Intitialize manager/workers random streams exit_criteria = {'sim_max': 80} # Stop libEnsemble after 80 simulations From f3e0fc6cfdc760348f122d99f5a5a90e716565b9 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 13:18:15 -0600 Subject: [PATCH 589/644] Fix typos in links --- docs/platforms/bebop.rst | 8 ++++---- docs/platforms/cori.rst | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 15c0fca8d..07eea0a14 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -57,12 +57,12 @@ With your nodes allocated, queue your job to start with four MPI ranks:: ``mpirun`` should also work. This line launches libEnsemble with a manager and **three** workers to one allocated compute node, with three nodes available for the workers to launch calculations with the job-controller or a job-launch command. -This is an example of running in :doc:`centralized` mode and, -if using the :doc:`job_controller<../job_controller/mpi_controller.rst>`, it should +This is an example of running in :doc:`centralized` mode and, +if using the :doc:`job_controller<../job_controller/mpi_controller>`, it should be intiated with ``central_mode=True`` .. note:: - When performing a :doc:`distributed` MPI libEnsemble run + When performing a :doc:`distributed` MPI libEnsemble run and not oversubscribing, specify one more MPI process than the number of allocated nodes. The Manager and first worker run together on a node. @@ -80,7 +80,7 @@ Batch Runs ^^^^^^^^^^ Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example -for a libEnsemble use-case running in :doc:`distributed` MPI +for a libEnsemble use-case running in :doc:`distributed` MPI mode on Broadwell nodes resembles the following: .. code-block:: bash diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index 65f7eae71..3281e2c73 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -56,12 +56,12 @@ This line launches libEnsemble with a manager and **three** workers to one allocated compute node, with three nodes available for the workers to launch user applications with the job-controller or a job-launch command. -This is an example of running in :doc:`centralized` mode and, -if using the :doc:`job_controller<../job_controller/mpi_controller.rst>`, it should +This is an example of running in :doc:`centralized` mode and, +if using the :doc:`job_controller<../job_controller/mpi_controller>`, it should be intiated with ``central_mode=True`` .. note:: - When performing a :doc:`distributed` MPI libEnsemble run, + When performing a :doc:`distributed` MPI libEnsemble run, specify one more MPI process than the number of allocated nodes. The Manager and first worker run together on a node. @@ -73,7 +73,7 @@ Batch Runs ^^^^^^^^^^ Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example -for a libEnsemble use-case running in :doc:`distributed` MPI +for a libEnsemble use-case running in :doc:`distributed` MPI mode on KNL nodes resembles the following: .. code-block:: bash From 5d8e6201046464b6948fc4fdf4b65435fed7353e Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 13:34:45 -0600 Subject: [PATCH 590/644] Updated calc_status documentation to show manager use --- docs/data_structures/calc_status.rst | 4 +++- docs/sim_gen_alloc_funcs.rst | 4 ++-- libensemble/message_numbers.py | 24 ++++++++++++++---------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/data_structures/calc_status.rst b/docs/data_structures/calc_status.rst index 330d2d726..2091f1246 100644 --- a/docs/data_structures/calc_status.rst +++ b/docs/data_structures/calc_status.rst @@ -3,7 +3,7 @@ calc_status =========== -The ``calc_status`` is an integer attribute with a corresponding description that can be used in :ref:`sim_f` or :ref:`gen_f` functions to capture the status of a calcuation. This is returned to the manager and printed to the ``libE_stats.txt`` file. This is not currently used by the manager, but can still provide a useful summary in libE_stats.txt. The user determines the status of the calculation, as it could include multiple application runs. It can be added as a third return variable in sim_f or gen_f functions. +The ``calc_status`` is an integer attribute with named (enumerated) values and a corresponding description that can be used in :ref:`sim_f` or :ref:`gen_f` functions to capture the status of a calcuation. This is returned to the manager and printed to the ``libE_stats.txt`` file. Only the status values ``FINISHED_PERSISTENT_SIM_TAG`` and ``FINISHED_PERSISTENT_GEN_TAG`` are currently used by the manager, but others can still provide a useful summary in libE_stats.txt. The user determines the status of the calculation, as it could include multiple application runs. It can be added as a third return variable in sim_f or gen_f functions. The calc_status codes are in the ``libensemble.message_numbers`` module. Example of ``calc_status`` used along with :ref:`job controller` in sim_f: @@ -44,6 +44,7 @@ Example of ``calc_status`` used along with :ref:`job controller` **calc_status**: :obj:`int`, optional. - Provides a job status to the Manager. Currently only used to output to libE_stats.txt file. + Provides a job status to the Manager and the libE_stats.txt file. :doc:`(example)` @@ -110,7 +110,7 @@ Returns: :doc:`(example)` **calc_status**: :obj:`int`, optional. - Provides a job status to the Manager. Currently only used to output to libE_stats.txt file. + Provides a job status to the Manager and the libE_stats.txt file. :doc:`(example)` diff --git a/libensemble/message_numbers.py b/libensemble/message_numbers.py index 80c87cf0d..27e498713 100644 --- a/libensemble/message_numbers.py +++ b/libensemble/message_numbers.py @@ -5,8 +5,7 @@ EVAL_GEN_TAG = 2 STOP_TAG = 3 PERSIS_STOP = 4 # manager tells persistent worker to desist -FINISHED_PERSISTENT_SIM_TAG = 11 # tells manager sim_f done persistent mode -FINISHED_PERSISTENT_GEN_TAG = 12 # tells manager gen_f done persistent mode + # last_message_number_rst_tag calc_type_strings = { @@ -17,19 +16,24 @@ # --- Signal flags (in message body vs tags) + # first_calc_status_rst_tag # CALC STATUS/SIGNAL FLAGS -MAN_SIGNAL_FINISH = 20 # Kill jobs and shutdown worker -MAN_SIGNAL_KILL = 21 # Kill running job - but don't stop worker -WORKER_KILL = 30 # Worker kills not covered by a more specific case -WORKER_KILL_ON_ERR = 31 -WORKER_KILL_ON_TIMEOUT = 32 -JOB_FAILED = 33 # Calc had jobs that failed -WORKER_DONE = 34 # Calculation was successful -CALC_EXCEPTION = 35 # Reserved: Automatically used if gen_f or sim_f raised an exception. +FINISHED_PERSISTENT_SIM_TAG = 11 # tells manager sim_f done persistent mode +FINISHED_PERSISTENT_GEN_TAG = 12 # tells manager gen_f done persistent mode +MAN_SIGNAL_FINISH = 20 # Kill jobs and shutdown worker +MAN_SIGNAL_KILL = 21 # Kill running job - but don't stop worker +WORKER_KILL = 30 # Worker kills not covered by a more specific case +WORKER_KILL_ON_ERR = 31 # Worker killed due to an error in results +WORKER_KILL_ON_TIMEOUT = 32 # Worker killed on timeout +JOB_FAILED = 33 # Calc had jobs that failed +WORKER_DONE = 34 # Calculation was successful # last_calc_status_rst_tag +CALC_EXCEPTION = 35 # Reserved: Automatically used if gen_f or sim_f raised an exception. calc_status_strings = { + FINISHED_PERSISTENT_SIM_TAG: "Persis gen finished", + FINISHED_PERSISTENT_GEN_TAG: "Persis sim finished", MAN_SIGNAL_FINISH: "Manager killed on finish", MAN_SIGNAL_KILL: "Manager killed job", WORKER_KILL_ON_ERR: " Worker killed job on Error", From ec6358db56e07bcce2c6fbeae45b372b98426cad Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 14:03:13 -0600 Subject: [PATCH 591/644] Minor fix to Cori guide --- docs/platforms/cori.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index 3281e2c73..f666f515b 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -113,7 +113,7 @@ View the status of your submitted jobs with ``squeue`` and cancel jobs with Additional Information ---------------------- -See the LCRC Bebop docs here_ for more information about Cori. +See the NERSC Cori docs here_ for more information about Cori. .. _Cori: https://docs.nersc.gov/systems/cori/ .. _Anaconda: https://www.anaconda.com/distribution/ From 99b68fbe1ebbf3908d84bfb80392ffe29cd2ce6c Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 14:36:29 -0600 Subject: [PATCH 592/644] updates job controller example to use forces, removes support usage in 6HC paposmm 1 --- docs/examples/calling_scripts.rst | 27 ++++++++++--------- .../test_6-hump_camel_persistent_aposmm_1.py | 9 ++++--- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index edd19c717..c608047fe 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -5,7 +5,8 @@ Below are example calling scripts used to populate specifications for each user function and libEnsemble before initiating libEnsemble via the primary ``libE()`` call. The primary libEnsemble-relevant portions have been highlighted in each example. Non-highlighted portions may include setup routines, compilation steps -for user-applications, or output processing. +for user-applications, or output processing. The first two scripts correspond to +random sampling calculations, while the third corresponds to an optimization routine. Local Sine Tutorial ------------------- @@ -19,29 +20,29 @@ meant to run with Python's multiprocessing as the primary ``comms`` method. :linenos: :emphasize-lines: 8-28 -Balsam Job Controller ---------------------- +Electrostatic Forces with Job Controller +---------------------------------------- -This example is from the regression tests. It configures MPI as the primary -communication method, and demonstrates registering a user-application with the -libEnsemble job controller to be launched by the job controller within the ``sim_f``. +This example is from a test for evaluating the scaling capabilities of libEnsemble +by calculating particle electrostatic forces through a user-application. This +application is registered with either the MPI or Balsam job controller, then +launched in the ``sim_f``. Note the use of the ``parse_args()`` and +``save_libE_output()`` convenience functions from the :doc:`utilities<../utilities>`. -.. literalinclude:: ../../libensemble/tests/regression_tests/script_test_balsam_hworld.py +.. literalinclude:: ../../libensemble/tests/scaling_tests/forces/run_libe_forces.py :language: python - :caption: tests/regression_tests/script_test_balsam_hworld.py + :caption: tests/scaling_tests/forces/run_libe_forces.py :linenos: - :emphasize-lines: 27-34, 42-65 + :emphasize-lines: 16, 39-92 6-Hump-Camel Persistent APOSMM ------------------------------ This example is also from the regression tests, and demonstrates configuring a -persistent run via a custom allocation function. Note the use of the -``parse_args()`` and ``save_libE_output()`` convenience functions from the -:doc:`utilities<../utilities>`. +persistent run via a custom allocation function. .. literalinclude:: ../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py :language: python :caption: tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py :linenos: - :emphasize-lines: 30, 38-69, 82 + :emphasize-lines: 29, 42-72, 85 diff --git a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py index 2ac709b55..1323c3d04 100644 --- a/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py +++ b/libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py @@ -24,7 +24,6 @@ from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f from libensemble.utils import parse_args, save_libE_output, add_unique_random_streams -from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima from time import time nworkers, is_master, libE_specs, _ = parse_args() @@ -35,6 +34,10 @@ if nworkers < 2: sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") +six_hump_camel_minima = np.array([[-0.089842, 0.712656], [0.089842, -0.712656], + [-1.70361, 0.796084], [1.70361, -0.796084], + [-1.6071, -0.568651], [1.6071, 0.568651]]) + n = 2 sim_specs = {'sim_f': sim_f, 'in': ['x'], @@ -47,7 +50,7 @@ 'in': [], 'out': gen_out, 'user': {'initial_sample_size': 100, - 'sample_points': np.round(minima, 1), + 'sample_points': np.round(six_hump_camel_minima, 1), 'localopt_method': 'LD_MMA', 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 'xtol_rel': 1e-6, @@ -73,7 +76,7 @@ print('[Manager]: Time taken =', time() - start_time, flush=True) tol = 1e-5 - for m in minima: + for m in six_hump_camel_minima: # The minima are known on this test problem. # We use their values to test APOSMM has identified all minima print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) From cb8379936025b32c6868ae82cca95654c186ba7c Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 16:22:49 -0600 Subject: [PATCH 593/644] formatting changes, remove XC40 note and common.py reference, add utilities reference and xSDK info --- README.rst | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 9d16ae55d..a0248f2ba 100644 --- a/README.rst +++ b/README.rst @@ -42,8 +42,8 @@ libEnsemble aims for: The user selects or supplies a function that generates simulation input as well as a function that performs and monitors the simulations. For example, the generation function may contain an -optimization routine to generate new simulation parameters on-the-fly based on the -results of previous simulations. Examples and templates of such functions are +optimization routine to generate new simulation parameters on-the-fly based on +the results of previous simulations. Examples and templates of such functions are included in the library. libEnsemble employs a manager-worker scheme that can run on various @@ -74,13 +74,12 @@ Optional dependency: * Balsam_ -From v0.2.0, libEnsemble has the option of using the Balsam job manager. This -is required for running libEnsemble on the compute nodes of some supercomputing -platforms (e.g., Cray XC40); platforms that do not support launching jobs from -compute nodes. Note that as of v0.5.0, libEnsemble can also be run on the -launch nodes using multiprocessing. +From v0.2.0, libEnsemble has the option of using the Balsam job manager. Balsam +is required to run libEnsemble on the compute nodes of some supercomputing +platforms that do not support launching jobs from compute nodes. As of v0.5.0, +libEnsemble can also be run on launch nodes using multiprocessing. -The example sim and gen functions and tests require the following dependencies: +The example simulation and generation functions and tests require the following: * SciPy_ * petsc4py_ @@ -89,22 +88,32 @@ The example sim and gen functions and tests require the following dependencies: PETSc and NLopt must be built with shared libraries enabled and present in ``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt -should produce a file ``nlopt.py`` if Python is found on the system. NLopt may also -require SWIG_ to be installed on certain systems. +should produce a file ``nlopt.py`` if Python is found on the system. NLopt may +also require SWIG_ to be installed on certain systems. Installation ~~~~~~~~~~~~ +libEnsemble may be installed or accessed from a variety of sources. + Use pip to install libEnsemble and its dependencies:: pip install libensemble -libEnsemble is also available in the Spack_ distribution. It can be installed from Spack with:: +libEnsemble is also available in the Spack_ distribution. It can be installed +from Spack with:: spack install py-libensemble -The tests and examples can be accessed in the GitHub_ repository. -If necessary, you may install all optional dependencies (listed above) at once with:: +libEnsemble is included in the `xSDK Extreme-scale Scientific Software Development Kit`_ +from version 0.5.0 onward. Install the xSDK and load the environment with:: + + spack install xsdk + spack load -r xsdk + +The codebase, tests and examples can be accessed in the GitHub_ repository. +If necessary, you may install all optional dependencies (listed above) at once +with:: pip install libensemble[extras] @@ -119,8 +128,8 @@ regularly on: * `Travis CI`_ The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ -packages to be installed and can be run from the libensemble/tests directory of -the source distribution by running:: +packages to be installed and can be run from the ``libensemble/tests`` directory +of the source distribution by running:: ./run-tests.sh @@ -154,8 +163,8 @@ available online at Coveralls_. Basic Usage ~~~~~~~~~~~ -The examples directory contains example libEnsemble calling scripts, sim -functions, gen functions, alloc functions and job submission scripts. +The examples directory contains example libEnsemble calling scripts, simulation +functions, generation functions, allocation functions and job submission scripts. The default manager/worker communications mode is MPI. The user script is launched as:: @@ -171,9 +180,8 @@ can then be run as a regular python script:: python myscript.py -When specifying these options via command line options, one may use the -``parse_args`` function used in the regression tests, which can be found in -`common.py`_ in the ``libensemble/tests/regression_tests`` directory. +These options may be specified via the command-line using the ``parse_args()`` +convenience function within ``libensemble/utils.py``. See the `user guide`_ for more information. @@ -210,7 +218,6 @@ Resources .. after_resources_rst_tag .. _Balsam: https://www.alcf.anl.gov/balsam -.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master .. _GitHub: https://github.com/Libensemble/libensemble .. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble @@ -235,3 +242,4 @@ Resources .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble .. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html +.. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info/ From 4cfd4128bf78c802e92ec0029632be52c4fd7466 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 16:50:01 -0600 Subject: [PATCH 594/644] update libE_specs example --- docs/data_structures/libE_specs.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index f68ce1c4f..35dcdbacc 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -32,6 +32,12 @@ Specifications for libEnsemble:: Profile using cProfile. Default: False .. seealso:: - Examples in `common.py`_ + Example ``libE_specs`` from the forces_ scaling test, completely populated:: -.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py + libE_specs = {'comm': MPI.COMM_WORLD, + 'comms': 'mpi', + 'save_every_k_gens': 1000, + 'sim_dir': './sim', + 'profile_worker': False} + +.. _forces: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/scaling_tests/forces/run_libe_forces.py From f9034e01d27ef24e8dc20be66970e6f43c26f5ca Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 17:00:17 -0600 Subject: [PATCH 595/644] update check inputs reference --- docs/data_structures/history_array.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 371362968..48b0a5c9f 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -26,6 +26,6 @@ Below are the protected fields used in ``H`` ``check_inputs()`` and calling it with their ``gen_specs``, ``alloc_specs``, and ``sim_specs`` as keyword arguments:: - from libensemble.libE import check_inputs + from libensemble.utils import check_inputs check_inputs(H0=my_H, sim_specs=sim_specs, alloc_specs=alloc_specs, gen_specs=gen_specs) From ea072da1a6438dd32575e6796aba893d4c8aae66 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 20 Nov 2019 17:03:13 -0600 Subject: [PATCH 596/644] formatting --- docs/logging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/logging.rst b/docs/logging.rst index 6301600e8..7296cfd4f 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -6,7 +6,7 @@ plus one additional custom level (MANAGER_WARNING) between WARNING and ERROR. The default level is INFO, which includes information about how jobs are launched and when jobs are killed. To gain additional diagnostics, the logging level can be set -to DEBUG. libEnsemble produces logging to the file ensemble.log by default. A log +to DEBUG. libEnsemble produces logging to the file ``ensemble.log`` by default. A log file name can also be supplied. To change the logging level to DEBUG, provide the following in your the calling scripts:: From 04c1205068c07b6d6652e11ad8a8f7fb972b7c86 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 22:59:11 -0600 Subject: [PATCH 597/644] Add theta multiprocessing MOM node example and make mods --- .../theta_submit_balsam.sh | 60 ++++++++++--------- .../theta_submit_mproc.sh | 47 +++++++++++++++ .../forces/theta_submit_balsam.sh | 20 +++---- .../forces/theta_submit_mproc.sh | 8 +-- 4 files changed, 93 insertions(+), 42 deletions(-) create mode 100755 examples/job_submission_scripts/theta_submit_mproc.sh diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index a06c7a2ae..2751be89b 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -1,9 +1,9 @@ #!/bin/bash -x -#COBALT -t 60 -#COBALT -O libE_test -#COBALT -n 5 #no. nodes -#COBALT -q debug-flat-quad #For small runs only -##COBALT -q default #For large jobs +#COBALT -t 30 +#COBALT -O libE_MPI_balsam +#COBALT -n 5 # No. nodes +#COBALT -q debug-flat-quad # Up to 8 nodes only +##COBALT -q default # For large jobs >=128 nodes ##COBALT -A <...project code...> # Script to launch libEnsemble using Balsam within Conda. Conda environment must be set up. @@ -11,7 +11,7 @@ # Requires Balsam is installed and a database initialized (this can be the default database). # To be run with central job management -# - Manager and workers run on one node (or more if nec). +# - Manager and workers run on one node (or a dedicated set of nodes). # - Workers submit jobs to the rest of the nodes in the pool. # Constaint: - As set up - only uses one node (up to 63 workers) for libE. To use more, modifiy "balsam job" line to use hyper-threading and/or more than one node. @@ -20,43 +20,47 @@ export EXE=libE_calling_script.py # Number of workers. -export NUM_WORKERS=16 +export NUM_WORKERS=4 -# Wallclock for libE job (supplied to Balsam - make at least several mins smaller than wallclock for this submission to ensure job is launched) -export LIBE_WALLCLOCK=45 +# Wallclock for libE job in minutes (supplied to Balsam - make at least several mins smaller than wallclock for this submission to ensure job is launched) +export BALSAM_WALLCLOCK=25 # Name of working directory where Balsam places running jobs/output (inside the database directory) -export WORKFLOW_NAME=libe_workflow #sh - todo - may currently be hardcoded to this in libE - allow user to specify +export WORKFLOW_NAME=libe_workflow -#Tell libE manager to stop workers, dump timing.dat and exit after this time. Script must be set up to receive as argument. -export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3)) -# export SCRIPT_ARGS='' #Default No args +#Tell libE manager to stop workers, and exit after this time. Script must be set up to receive as argument. +export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) -# Name of Conda environment (Need to have set up: https://balsam.alcf.anl.gov/quick/quickstart.html) -export CONDA_ENV_NAME=balsam +# libEnsemble calling script arguments +# export SCRIPT_ARGS='' # No args +# export SCRIPT_ARGS='$LIBE_WALLCLOCK' # If calling script takes wall-clock as positional arg. + +# If script is using util.parse_args() and takes wall-clock as positional arg. +export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS $LIBE_WALLCLOCK" + +# Name of Conda environment (Need to have set up: https://balsam.readthedocs.io/en/latest/userguide/getting-started.html) +export CONDA_ENV_NAME= + +# Name of database +export DBASE_NAME= # default - to use default database. # Conda location - theta specific -export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -export LD_LIBRARY_PATH=~/.conda/envs/balsam/lib:$LD_LIBRARY_PATH +# export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH +# export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH -export PYTHONNOUSERSITE=1 #Ensure environment isolated +export PYTHONNOUSERSITE=1 # Ensure environment isolated export PMI_NO_FORK=1 # Required for python kills on Theta # Activate conda environment . activate $CONDA_ENV_NAME -# Unload Theta modules that may interfere with Balsam +# Unload Theta modules that may interfere with job monitoring/kills module unload trackdeps module unload darshan module unload xalt -#Location of sim func script (only required if not in sys.path already) -# export PYTHONPATH=/home/shudson/opal_work/opal/emittance_minimization/code:$PYTHONPATH - -. balsamactivate default -# . balsamactivate my_dbase - +. balsamactivate $DBASE_NAME # Currently need atleast one DB connection per worker (for postgres). if [[ $NUM_WORKERS -gt 128 ]] @@ -79,15 +83,15 @@ SCRIPT_BASENAME=${EXE%.*} balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" # Running libE on one node - one manager and upto 63 workers -balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $BALSAM_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes # Hyper-thread libE (note this will not affect HT status of user calcs - only libE itself) # Running 255 workers and one manager on one libE node. -# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node 256 --threads-per-core 4 --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $BALSAM_WALLCLOCK --num-nodes 1 --ranks-per-node 256 --threads-per-core 4 --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes # Multiple nodes for libE # Running 127 workers and one manager - launch script on 129 nodes (if one node per worker) -# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 2 --ranks-per-node 64 --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $BALSAM_WALLCLOCK --num-nodes 2 --ranks-per-node 64 --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes #Run job balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 diff --git a/examples/job_submission_scripts/theta_submit_mproc.sh b/examples/job_submission_scripts/theta_submit_mproc.sh new file mode 100755 index 000000000..74718bfc3 --- /dev/null +++ b/examples/job_submission_scripts/theta_submit_mproc.sh @@ -0,0 +1,47 @@ +#!/bin/bash -x +#COBALT -t 00:30:00 +#COBALT -n 4 +#COBALT -q debug-flat-quad # Up to 8 nodes only +##COBALT -q default # For large jobs >=128 nodes +##COBALT -A <...project code...> + +# Script to run libEnsemble using multiprocessing on launch nodes. +# Assumes Conda environment is set up. + +# To be run with central job management +# - Manager and workers run on launch node. +# - Workers submit jobs to the compute nodes in the node allocation available. + +# Name of calling script +export EXE=run_libe_forces.py + +# Communication Method +export COMMS="--comms local" + +# Number of workers. +export NWORKERS="--nworkers 4" + +# Wallclock for libE (allow clean shutdown) +#export LIBE_WALLCLOCK=25 # Optional if pass to script + +# Name of Conda environment +export CONDA_ENV_NAME= + +# Conda location - theta specific +export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH +export LD_LIBRARY_PATH=~/.conda/envs/balsam/lib:$LD_LIBRARY_PATH +export PMI_NO_FORK=1 # Required for python kills on Theta + +# Unload Theta modules that may interfere with job monitoring/kills +module unload trackdeps +module unload darshan +module unload xalt + +# Activate conda environment +export PYTHONNOUSERSITE=1 +. activate $CONDA_ENV_NAME + +# Launch libE +#python $EXE $NUM_WORKERS > out.txt 2>&1 # No args +#python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 # If user script takes wall-clock as positional arg. +python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using util.parse_args() diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index 3f071a9dc..e8d2f8306 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -18,24 +18,25 @@ export EXE=run_libe_forces.py # Number of workers. export NUM_WORKERS=127 -# Wallclock for libE job (supplied to Balsam - make at least several mins smaller than wallclock for this submission to ensure job is launched) +# Wallclock for libE job in minutes (supplied to Balsam - make at least several mins smaller than wallclock for this submission to ensure job is launched) export LIBE_WALLCLOCK=25 # Name of working directory where Balsam places running jobs/output (inside the database directory) -export WORKFLOW_NAME=libe_workflow #sh - todo - may currently be hardcoded to this in libE - allow user to specify +export WORKFLOW_NAME=libe_workflow -#Tell libE manager to stop workers, dump timing.dat and exit after this time. Script must be set up to receive as argument. -export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS" -# export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) # export SCRIPT_ARGS='' #Default No args +# export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-5)) +export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS" -# Name of Conda environment (Need to have set up: https://balsam.alcf.anl.gov/quick/quickstart.html) +# Name of Conda environment export CONDA_ENV_NAME= + +# Name of database export DBASE_NAME= # Conda location - theta specific -export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -export LD_LIBRARY_PATH=~/.conda/envs/balsam/lib:$LD_LIBRARY_PATH +# export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH +# export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH export PYTHONNOUSERSITE=1 #Ensure environment isolated @@ -48,12 +49,11 @@ export PLOT_DIR=.. # Activate conda environment . activate $CONDA_ENV_NAME -# Unload Theta modules that may interfere with Balsam +# Unload Theta modules that may interfere with job monitoring/kills module unload trackdeps module unload darshan module unload xalt - . balsamactivate $DBASE_NAME # Make sure no existing apps/jobs diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh index d46dc6a5d..c39b8f9e3 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh @@ -18,7 +18,7 @@ export EXE=run_libe_forces.py export COMMS="--comms local" # Number of workers. -export NWORKERS="--nworkers 4" +export NWORKERS="--nworkers 128" # Wallclock for libE (allow clean shutdown) #export LIBE_WALLCLOCK=25 # Optional if pass to script @@ -27,14 +27,14 @@ export NWORKERS="--nworkers 4" export CONDA_ENV_NAME= # Conda location - theta specific -export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -export LD_LIBRARY_PATH=~/.conda/envs/balsam/lib:$LD_LIBRARY_PATH +# export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH +# export LD_LIBRARY_PATH=~/.conda/envs//lib:$LD_LIBRARY_PATH export PMI_NO_FORK=1 # Required for python kills on Theta export LIBE_PLOTS=true # Require plot scripts in $PLOT_DIR (see at end) export PLOT_DIR=.. -# Unload Theta modules that may interfere +# Unload Theta modules that may interfere with job monitoring/kills module unload trackdeps module unload darshan module unload xalt From e81ce567c38b826b2250576f31b53f9271a1bcd0 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 20 Nov 2019 23:08:24 -0600 Subject: [PATCH 598/644] Add summit submission script to examples and fix typo --- .../summit_submit_mproc.sh | 43 +++++++++++++++++++ .../theta_submit_balsam.sh | 4 +- .../theta_submit_mproc.sh | 5 ++- 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 examples/job_submission_scripts/summit_submit_mproc.sh diff --git a/examples/job_submission_scripts/summit_submit_mproc.sh b/examples/job_submission_scripts/summit_submit_mproc.sh new file mode 100644 index 000000000..8872e1811 --- /dev/null +++ b/examples/job_submission_scripts/summit_submit_mproc.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x +#BSUB -P +#BSUB -J libe_mproc +#BSUB -W 30 +#BSUB -nnodes 4 +#BSUB -alloc_flags "smt1" + +# Script to run libEnsemble using multiprocessing on launch nodes. +# Assumes Conda environment is set up. + +# To be run with central job management +# - Manager and workers run on launch node. +# - Workers submit jobs to the nodes in the job available. + +# Name of calling script- +export EXE=run_libe_forces.py + +# Communication Method +export COMMS="--comms local" + +# Number of workers. +export NWORKERS="--nworkers 4" + +# Wallclock for libE. (allow clean shutdown) +#export LIBE_WALLCLOCK=25 # Optional if pass to script + +# Name of Conda environment +export CONDA_ENV_NAME= + +# Need these if not already loaded +# module load python +# module load gcc/4.8.5 + +# Activate conda environment +export PYTHONNOUSERSITE=1 +. activate $CONDA_ENV_NAME + +# hash -d python # Check pick up python in conda env +hash -r # Check no commands hashed (pip/python...) + +# Launch libE. +#python $EXE $LIBE_WALLCLOCK > out.txt 2>&1 # If user script takes wall-clock as positional arg. +python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using utils.parse_args() diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index 2751be89b..a80d2fba6 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -4,7 +4,7 @@ #COBALT -n 5 # No. nodes #COBALT -q debug-flat-quad # Up to 8 nodes only ##COBALT -q default # For large jobs >=128 nodes -##COBALT -A <...project code...> +##COBALT -A # Script to launch libEnsemble using Balsam within Conda. Conda environment must be set up. @@ -35,7 +35,7 @@ export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) # export SCRIPT_ARGS='' # No args # export SCRIPT_ARGS='$LIBE_WALLCLOCK' # If calling script takes wall-clock as positional arg. -# If script is using util.parse_args() and takes wall-clock as positional arg. +# If script is using utilsparse_args() and takes wall-clock as positional arg. export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS $LIBE_WALLCLOCK" # Name of Conda environment (Need to have set up: https://balsam.readthedocs.io/en/latest/userguide/getting-started.html) diff --git a/examples/job_submission_scripts/theta_submit_mproc.sh b/examples/job_submission_scripts/theta_submit_mproc.sh index 74718bfc3..7c638a6a3 100755 --- a/examples/job_submission_scripts/theta_submit_mproc.sh +++ b/examples/job_submission_scripts/theta_submit_mproc.sh @@ -1,9 +1,10 @@ #!/bin/bash -x #COBALT -t 00:30:00 +#COBALT -O libE_mproc_MOM #COBALT -n 4 #COBALT -q debug-flat-quad # Up to 8 nodes only ##COBALT -q default # For large jobs >=128 nodes -##COBALT -A <...project code...> +##COBALT -A # Script to run libEnsemble using multiprocessing on launch nodes. # Assumes Conda environment is set up. @@ -44,4 +45,4 @@ export PYTHONNOUSERSITE=1 # Launch libE #python $EXE $NUM_WORKERS > out.txt 2>&1 # No args #python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 # If user script takes wall-clock as positional arg. -python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using util.parse_args() +python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using utils.parse_args() From 24c8333d7c522ce2facc4b35743adf10141340b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 21 Nov 2019 11:26:33 -0600 Subject: [PATCH 599/644] Trying to trigger RTD From eb25c7a19b182ce33be91e580e9382dc477bed47 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 21 Nov 2019 11:31:56 -0600 Subject: [PATCH 600/644] adds symlinks for example calling scripts and regression tests dir --- examples/calling_scripts/regression_tests | 1 + examples/calling_scripts/run_libe_forces.py | 1 + .../calling_scripts/test_6-hump_camel_persistent_aposmm_1.py | 1 + examples/calling_scripts/tutorial_calling.py | 1 + 4 files changed, 4 insertions(+) create mode 120000 examples/calling_scripts/regression_tests create mode 120000 examples/calling_scripts/run_libe_forces.py create mode 120000 examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py create mode 120000 examples/calling_scripts/tutorial_calling.py diff --git a/examples/calling_scripts/regression_tests b/examples/calling_scripts/regression_tests new file mode 120000 index 000000000..2db4760c5 --- /dev/null +++ b/examples/calling_scripts/regression_tests @@ -0,0 +1 @@ +../../libensemble/tests/regression_tests \ No newline at end of file diff --git a/examples/calling_scripts/run_libe_forces.py b/examples/calling_scripts/run_libe_forces.py new file mode 120000 index 000000000..ec19d93cb --- /dev/null +++ b/examples/calling_scripts/run_libe_forces.py @@ -0,0 +1 @@ +../../libensemble/tests/scaling_tests/forces/run_libe_forces.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py b/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py new file mode 120000 index 000000000..a47e29bc9 --- /dev/null +++ b/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py @@ -0,0 +1 @@ +../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py \ No newline at end of file diff --git a/examples/calling_scripts/tutorial_calling.py b/examples/calling_scripts/tutorial_calling.py new file mode 120000 index 000000000..587851d0e --- /dev/null +++ b/examples/calling_scripts/tutorial_calling.py @@ -0,0 +1 @@ +../tutorials/tutorial_calling.py \ No newline at end of file From 54a7c5955eb4c1b2190f4f15d66857a204ba37ad Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 21 Nov 2019 11:43:03 -0600 Subject: [PATCH 601/644] Removing Jeff from copyright --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b45b78d79..975930867 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -91,7 +91,7 @@ def __getattr__(cls, name): # General information about the project. project = 'libEnsemble' -copyright = '2019, Jeffrey Larson' +copyright = '2019' author = 'Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro' # The version info for the project you're documenting, acts as replacement for From d72ff144dd89d5be81df5d7a957bd19806e0e4d3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 21 Nov 2019 11:51:33 -0600 Subject: [PATCH 602/644] Removing bib from html, link should still work --- docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ca3837201..78fc2af8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,7 +29,6 @@ known_issues release_notes contributing - bibliography .. toctree:: :maxdepth: 2 From 15aed8769e4deacc76db0c7569e1655b2468d24d Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 25 Nov 2019 10:56:29 -0600 Subject: [PATCH 603/644] Make parse_args ignore unrecognized args and utils logger for I/O --- libensemble/utils.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 836cddec4..1d4256912 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -7,9 +7,25 @@ import argparse import pickle +# Create logger logger = logging.getLogger(__name__) -# To change logging level for just this module -# logger.setLevel(logging.DEBUG) +logger.propagate = False +logger.setLevel(logging.INFO) + +# Set up format (Alt. Import LogConfig and base on that) +utils_logformat = '%(name)s: %(message)s' +formatter = logging.Formatter(utils_logformat) + +# Log to file +# util_filename = 'util.log' +# fh = logging.FileHandler(util_filename, mode='w') +# fh.setFormatter(formatter) +# logger.addHandler(fh) + +# Log to standard error +sth = logging.StreamHandler(stream=sys.stderr) +sth.setFormatter(formatter) +logger.addHandler(sth) """ @@ -346,7 +362,8 @@ def parse_args(): :doc:`(example)` """ - args = parser.parse_args(sys.argv[1:]) + #args = parser.parse_args(sys.argv[1:]) + args, unknown = parser.parse_known_args(sys.argv[1:]) front_ends = { 'mpi': _mpi_parse_args, 'local': _local_parse_args, @@ -355,12 +372,15 @@ def parse_args(): 'client': _client_parse_args} if args.pwd is not None: os.chdir(args.pwd) - return front_ends[args.comms or 'mpi'](args) + nworkers, is_master, libE_specs, tester_args = front_ends[args.comms or 'mpi'](args) + if is_master and unknown: + logger.warning('parse_args ignoring unrecognized arguments: {}'.format(' '.join(unknown))) + return nworkers, is_master, libE_specs, tester_args # =================== save libE output to pickle and np ======================== -def save_libE_output(H, persis_info, calling_file, nworkers): +def save_libE_output(H, persis_info, calling_file, nworkers, mess='Run completed'): """ Writes out history array and persis_info to files. @@ -392,6 +412,10 @@ def save_libE_output(H, persis_info, calling_file, nworkers): The number of workers in this ensemble. Added to output file names. + mess: :obj:`String` + + A message to print/log when saving the file. + """ script_name = os.path.splitext(os.path.basename(calling_file))[0] @@ -400,13 +424,14 @@ def save_libE_output(H, persis_info, calling_file, nworkers): + '_evals=' + str(sum(H['returned'])) \ + '_ranks=' + str(nworkers) - print("\n\n\nRun completed.\nSaving results to file: "+filename) + status_mess = ' '.join(['------------------', mess, '-------------------']) + logger.info('{}\nSaving results to file: {}'.format(status_mess, filename)) np.save(filename, H) with open(filename + ".pickle", "wb") as f: pickle.dump(persis_info, f) -# ===================== per-worker numpy random-streams ======================== +# ===================== per-process numpy random-streams ======================= def add_unique_random_streams(persis_info, nstreams): @@ -446,5 +471,7 @@ def add_unique_random_streams(persis_info, nstreams): return persis_info +# A very specific exception to using the logger. def eprint(*args, **kwargs): + """Prints a user message to standard error""" print(*args, file=sys.stderr, **kwargs) From f561dc871b16b62e6983643a5a7f8b552b119db4 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 25 Nov 2019 11:34:28 -0600 Subject: [PATCH 604/644] Whitespace --- docs/platforms/summit.rst | 3 --- docs/sim_gen_alloc_funcs.rst | 2 -- 2 files changed, 5 deletions(-) diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 1cdf7caec..0877b809d 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -12,7 +12,6 @@ Users on login nodes submit batch runs to the launch nodes. Batch scripts and interactive sessions run on the launch nodes. Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. - Configuring Python ------------------ @@ -124,7 +123,6 @@ libEnsemble on Summit becomes:: $ bsub myscript.sh - Launching user applications from libEnsemble Workers ---------------------------------------------------- @@ -135,7 +133,6 @@ is used inside the ``sim_f`` or ``gen_f``, as this provides a portable interface with many advantages including automatic resource detection, portability, launch failure resilience, and ease-of-use. - Additional Information ---------------------- diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index 77a253a2c..137a1991c 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -75,7 +75,6 @@ Returns: Provides a job status to the Manager and the libE_stats.txt file. :doc:`(example)` - gen_f API ~~~~~~~~~ .. _api_gen_f: @@ -113,7 +112,6 @@ Returns: Provides a job status to the Manager and the libE_stats.txt file. :doc:`(example)` - alloc_f API ~~~~~~~~~~~ .. _api_alloc_f: From 424bf084906ff087fe85d8483425d2140e2b7924 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 25 Nov 2019 11:59:53 -0600 Subject: [PATCH 605/644] flake8 --- libensemble/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 1d4256912..f769c6365 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -362,7 +362,7 @@ def parse_args(): :doc:`(example)` """ - #args = parser.parse_args(sys.argv[1:]) + # args = parser.parse_args(sys.argv[1:]) args, unknown = parser.parse_known_args(sys.argv[1:]) front_ends = { 'mpi': _mpi_parse_args, From c5f7ae152b8f7b71326402e1035046ef5eba1253 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 25 Nov 2019 10:56:29 -0600 Subject: [PATCH 606/644] Make parse_args ignore unrecognized args and utils logger for I/O --- libensemble/utils.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index 836cddec4..1d4256912 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -7,9 +7,25 @@ import argparse import pickle +# Create logger logger = logging.getLogger(__name__) -# To change logging level for just this module -# logger.setLevel(logging.DEBUG) +logger.propagate = False +logger.setLevel(logging.INFO) + +# Set up format (Alt. Import LogConfig and base on that) +utils_logformat = '%(name)s: %(message)s' +formatter = logging.Formatter(utils_logformat) + +# Log to file +# util_filename = 'util.log' +# fh = logging.FileHandler(util_filename, mode='w') +# fh.setFormatter(formatter) +# logger.addHandler(fh) + +# Log to standard error +sth = logging.StreamHandler(stream=sys.stderr) +sth.setFormatter(formatter) +logger.addHandler(sth) """ @@ -346,7 +362,8 @@ def parse_args(): :doc:`(example)` """ - args = parser.parse_args(sys.argv[1:]) + #args = parser.parse_args(sys.argv[1:]) + args, unknown = parser.parse_known_args(sys.argv[1:]) front_ends = { 'mpi': _mpi_parse_args, 'local': _local_parse_args, @@ -355,12 +372,15 @@ def parse_args(): 'client': _client_parse_args} if args.pwd is not None: os.chdir(args.pwd) - return front_ends[args.comms or 'mpi'](args) + nworkers, is_master, libE_specs, tester_args = front_ends[args.comms or 'mpi'](args) + if is_master and unknown: + logger.warning('parse_args ignoring unrecognized arguments: {}'.format(' '.join(unknown))) + return nworkers, is_master, libE_specs, tester_args # =================== save libE output to pickle and np ======================== -def save_libE_output(H, persis_info, calling_file, nworkers): +def save_libE_output(H, persis_info, calling_file, nworkers, mess='Run completed'): """ Writes out history array and persis_info to files. @@ -392,6 +412,10 @@ def save_libE_output(H, persis_info, calling_file, nworkers): The number of workers in this ensemble. Added to output file names. + mess: :obj:`String` + + A message to print/log when saving the file. + """ script_name = os.path.splitext(os.path.basename(calling_file))[0] @@ -400,13 +424,14 @@ def save_libE_output(H, persis_info, calling_file, nworkers): + '_evals=' + str(sum(H['returned'])) \ + '_ranks=' + str(nworkers) - print("\n\n\nRun completed.\nSaving results to file: "+filename) + status_mess = ' '.join(['------------------', mess, '-------------------']) + logger.info('{}\nSaving results to file: {}'.format(status_mess, filename)) np.save(filename, H) with open(filename + ".pickle", "wb") as f: pickle.dump(persis_info, f) -# ===================== per-worker numpy random-streams ======================== +# ===================== per-process numpy random-streams ======================= def add_unique_random_streams(persis_info, nstreams): @@ -446,5 +471,7 @@ def add_unique_random_streams(persis_info, nstreams): return persis_info +# A very specific exception to using the logger. def eprint(*args, **kwargs): + """Prints a user message to standard error""" print(*args, file=sys.stderr, **kwargs) From 8f389b7edec7f3efa625d04ffc414357c6ef309a Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 16:19:22 -0600 Subject: [PATCH 607/644] Change script plot_libE_histogram.py to plot_libe_histogram.py --- libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh | 2 +- libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh | 2 +- libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh | 2 +- .../{plot_libE_histogram.py => plot_libe_histogram.py} | 0 postproc_scripts/readme.rst | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename postproc_scripts/{plot_libE_histogram.py => plot_libe_histogram.py} (100%) diff --git a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh index 42e9da68c..0f34cd3a1 100644 --- a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh @@ -48,5 +48,5 @@ python $EXE $COMMS $NWORKERS > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libE_histogram.py + python $PLOT_DIR/plot_libe_histogram.py fi diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index e8d2f8306..f79e5e2a7 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -92,7 +92,7 @@ balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libE_histogram.pyfi + python $PLOT_DIR/plot_libe_histogram.pyfi if [[ $BALSAM_PLOTS = "true" ]]; then # export MPLBACKEND=TkAgg diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh index c39b8f9e3..ddfeafb74 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_mproc.sh @@ -51,5 +51,5 @@ python $EXE $COMMS $NWORKERS > out.txt 2>&1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libE_histogram.py + python $PLOT_DIR/plot_libe_histogram.py fi diff --git a/postproc_scripts/plot_libE_histogram.py b/postproc_scripts/plot_libe_histogram.py similarity index 100% rename from postproc_scripts/plot_libE_histogram.py rename to postproc_scripts/plot_libe_histogram.py diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index ce160f1e8..94d2d9ae1 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -10,7 +10,7 @@ The following scripts must be run in the directory with the **libE_stats.txt** f * **plot_libe_runs_util_v_time.py**: Extract launched job utilization v time plot (with one second sampling). Shows number of workers with active jobs, launched via the job controller, over time. -* **plot_libE_histogram.py**: Create histogram showing the number of completed/killed/failed user calculations binned by run-time. +* **plot_libe_histogram.py**: Create histogram showing the number of completed/killed/failed user calculations binned by run-time. ======================== Results analysis scripts From c296fab060eb3d5b50b36b376627707b659835b9 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 17:34:48 -0600 Subject: [PATCH 608/644] Remove print in resources.py --- libensemble/resources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libensemble/resources.py b/libensemble/resources.py index a5254499c..739194a51 100644 --- a/libensemble/resources.py +++ b/libensemble/resources.py @@ -159,7 +159,6 @@ def get_MPI_variant(): return 'mpich' return 'openmpi' except Exception as e: - print('Testing: Error on MPI command: {}'.format(e)) pass try: From 1a9f54790b25107c42dfc4404aadb6dd31e7f408 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 21:07:49 -0600 Subject: [PATCH 609/644] Remove ref to common and fix xsdk link for pdf --- README.rst | 2 +- docs/introduction_latex.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a0248f2ba..bb473a6be 100644 --- a/README.rst +++ b/README.rst @@ -242,4 +242,4 @@ Resources .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble .. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html -.. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info/ +.. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index e287bb7a7..fd02676cb 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -20,7 +20,6 @@ Quickstart Guide :end-before: after_resources_rst_tag .. _Balsam: https://www.alcf.anl.gov/balsam -.. _common.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/common.py .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master .. _GitHub: https://github.com/Libensemble/libensemble .. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble @@ -45,3 +44,4 @@ Quickstart Guide .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble .. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html +.. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info \ No newline at end of file From 4898abf4a2c9c1b6abb90eaab7d760efe9a63af8 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 21:41:56 -0600 Subject: [PATCH 610/644] Copyright 2019 Argonne National Laboratory --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 975930867..0798e55fa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -91,7 +91,7 @@ def __getattr__(cls, name): # General information about the project. project = 'libEnsemble' -copyright = '2019' +copyright = '2019 Argonne National Laboratory' author = 'Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro' # The version info for the project you're documenting, acts as replacement for From c9d8844ff09208d4d9ad497a2d26d87489fb6859 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 22:21:23 -0600 Subject: [PATCH 611/644] Update example submission scripts --- docs/platforms/example_scripts.rst | 14 ++++++++++++++ examples/job_submission_scripts/blues_script.pbs | 5 +---- .../job_submission_scripts/summit_submit_mproc.sh | 11 ++++++----- .../job_submission_scripts/theta_submit_balsam.sh | 12 +++++------- .../job_submission_scripts/theta_submit_mproc.sh | 12 ++++++------ 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index c6b300bb6..7d6288edc 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -26,9 +26,23 @@ Blues :caption: /examples/job_submission_scripts/blues_script.pbs :language: bash +Theta - On MOM with multiprocessing +----------------------------------- + +.. literalinclude:: ../../examples/job_submission_scripts/theta_submit_mproc.sh + :caption: /examples/job_submission_scripts/theta_submit_mproc.sh + :language: bash + Theta - Central Mode with Balsam -------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/theta_submit_balsam.sh :caption: /examples/job_submission_scripts/theta_submit_balsam.sh :language: bash + +Summit - On Launch nodes with multiprocessing +--------------------------------------------- + +.. literalinclude:: ../../examples/job_submission_scripts/summit_submit_mproc.sh + :caption: /examples/job_submission_scripts/summit_submit_mproc.sh + :language: bash diff --git a/examples/job_submission_scripts/blues_script.pbs b/examples/job_submission_scripts/blues_script.pbs index 1afac76f1..843a86763 100644 --- a/examples/job_submission_scripts/blues_script.pbs +++ b/examples/job_submission_scripts/blues_script.pbs @@ -30,9 +30,6 @@ ##PBS -q ivy #PBS -q shared -export NLOPT_PYTHON_HOME="/home/jlarson/software/nlopt_install/lib/python2.7/site-packages" -export PYTHONPATH="${PYTHONPATH}:${NLOPT_PYTHON_HOME}" - cd $PBS_O_WORKDIR # A little useful information for the log file... @@ -55,7 +52,7 @@ cat $PBS_NODEFILE | sort | uniq >> libE_machinefile echo Starting executation at: `date` pwd -cmd="mpiexec -np 5 -machinefile libE_machinefile python2 libE_calling_script.py libE_machinefile" +cmd="mpiexec -np 5 -machinefile libE_machinefile python libE_calling_script.py libE_machinefile" # This note that this command passes the libE_machinefile to both MPI and the # libE_calling_script, in the latter script, it can be parsed and given to the # alloc_func diff --git a/examples/job_submission_scripts/summit_submit_mproc.sh b/examples/job_submission_scripts/summit_submit_mproc.sh index 8872e1811..5a83ff106 100644 --- a/examples/job_submission_scripts/summit_submit_mproc.sh +++ b/examples/job_submission_scripts/summit_submit_mproc.sh @@ -13,7 +13,7 @@ # - Workers submit jobs to the nodes in the job available. # Name of calling script- -export EXE=run_libe_forces.py +export EXE=libE_calling_script.py # Communication Method export COMMS="--comms local" @@ -22,7 +22,7 @@ export COMMS="--comms local" export NWORKERS="--nworkers 4" # Wallclock for libE. (allow clean shutdown) -#export LIBE_WALLCLOCK=25 # Optional if pass to script +export LIBE_WALLCLOCK=25 # Optional if pass to script # Name of Conda environment export CONDA_ENV_NAME= @@ -38,6 +38,7 @@ export PYTHONNOUSERSITE=1 # hash -d python # Check pick up python in conda env hash -r # Check no commands hashed (pip/python...) -# Launch libE. -#python $EXE $LIBE_WALLCLOCK > out.txt 2>&1 # If user script takes wall-clock as positional arg. -python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using utils.parse_args() +# Launch libE +# python $EXE $NUM_WORKERS > out.txt 2>&1 # No args. All defined in calling script +# python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If calling script is using utils.parse_args() +python $EXE $LIBE_WALLCLOCK $COMMS $NWORKERS > out.txt 2>&1 # If calling script takes wall-clock as positional arg. \ No newline at end of file diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index a80d2fba6..1677b118a 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -32,11 +32,9 @@ export WORKFLOW_NAME=libe_workflow export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) # libEnsemble calling script arguments -# export SCRIPT_ARGS='' # No args -# export SCRIPT_ARGS='$LIBE_WALLCLOCK' # If calling script takes wall-clock as positional arg. - -# If script is using utilsparse_args() and takes wall-clock as positional arg. -export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS $LIBE_WALLCLOCK" +# export SCRIPT_ARGS='' # No args. All defined in calling script +# export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS # If calling script is using utils.parse_args() +export SCRIPT_ARGS="$LIBE_WALLCLOCK --comms mpi --nworkers $NUM_WORKERS" # If calling script takes wall-clock as positional arg. # Name of Conda environment (Need to have set up: https://balsam.readthedocs.io/en/latest/userguide/getting-started.html) export CONDA_ENV_NAME= @@ -45,8 +43,8 @@ export CONDA_ENV_NAME= export DBASE_NAME= # default - to use default database. # Conda location - theta specific -# export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -# export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH +export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH +export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH export PYTHONNOUSERSITE=1 # Ensure environment isolated diff --git a/examples/job_submission_scripts/theta_submit_mproc.sh b/examples/job_submission_scripts/theta_submit_mproc.sh index 7c638a6a3..394091c72 100755 --- a/examples/job_submission_scripts/theta_submit_mproc.sh +++ b/examples/job_submission_scripts/theta_submit_mproc.sh @@ -14,7 +14,7 @@ # - Workers submit jobs to the compute nodes in the node allocation available. # Name of calling script -export EXE=run_libe_forces.py +export EXE=libE_calling_script.py # Communication Method export COMMS="--comms local" @@ -23,14 +23,14 @@ export COMMS="--comms local" export NWORKERS="--nworkers 4" # Wallclock for libE (allow clean shutdown) -#export LIBE_WALLCLOCK=25 # Optional if pass to script +export LIBE_WALLCLOCK=25 # Optional if pass to script # Name of Conda environment export CONDA_ENV_NAME= # Conda location - theta specific export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -export LD_LIBRARY_PATH=~/.conda/envs/balsam/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH export PMI_NO_FORK=1 # Required for python kills on Theta # Unload Theta modules that may interfere with job monitoring/kills @@ -43,6 +43,6 @@ export PYTHONNOUSERSITE=1 . activate $CONDA_ENV_NAME # Launch libE -#python $EXE $NUM_WORKERS > out.txt 2>&1 # No args -#python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 # If user script takes wall-clock as positional arg. -python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If script is using utils.parse_args() +# python $EXE $NUM_WORKERS > out.txt 2>&1 # No args. All defined in calling script +# python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If calling script is using utils.parse_args() +python $EXE $LIBE_WALLCLOCK $COMMS $NWORKERS > out.txt 2>&1 # If calling script takes wall-clock as positional arg. From f5fe36d5acbedeb1f74b7772e142a6c09e75c3e7 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 22:25:20 -0600 Subject: [PATCH 612/644] Remove individual links from example calling scripts to regression tests --- .../test_6-hump_camel_active_persistent_worker_abort.py | 1 - examples/calling_scripts/test_6-hump_camel_aposmm_LD_MMA.py | 1 - examples/calling_scripts/test_6-hump_camel_elapsed_time_abort.py | 1 - .../calling_scripts/test_6-hump_camel_persistent_aposmm_1.py | 1 - .../test_6-hump_camel_persistent_uniform_sampling.py | 1 - examples/calling_scripts/test_6-hump_camel_uniform_sampling.py | 1 - ...-hump_camel_uniform_sampling_with_persistent_localopt_gens.py | 1 - .../test_6-hump_camel_with_different_nodes_uniform_sample.py | 1 - .../calling_scripts/test_branin_aposmm_nlopt_and_then_scipy.py | 1 - .../test_chwirut_aposmm_one_residual_at_a_time.py | 1 - examples/calling_scripts/test_chwirut_pounders.py | 1 - .../test_chwirut_uniform_sampling_one_residual_at_a_time.py | 1 - examples/calling_scripts/test_fast_alloc.py | 1 - examples/calling_scripts/test_jobcontroller_hworld.py | 1 - examples/calling_scripts/test_nan_func_aposmm.py | 1 - 15 files changed, 15 deletions(-) delete mode 120000 examples/calling_scripts/test_6-hump_camel_active_persistent_worker_abort.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_aposmm_LD_MMA.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_elapsed_time_abort.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_persistent_uniform_sampling.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_uniform_sampling.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py delete mode 120000 examples/calling_scripts/test_6-hump_camel_with_different_nodes_uniform_sample.py delete mode 120000 examples/calling_scripts/test_branin_aposmm_nlopt_and_then_scipy.py delete mode 120000 examples/calling_scripts/test_chwirut_aposmm_one_residual_at_a_time.py delete mode 120000 examples/calling_scripts/test_chwirut_pounders.py delete mode 120000 examples/calling_scripts/test_chwirut_uniform_sampling_one_residual_at_a_time.py delete mode 120000 examples/calling_scripts/test_fast_alloc.py delete mode 120000 examples/calling_scripts/test_jobcontroller_hworld.py delete mode 120000 examples/calling_scripts/test_nan_func_aposmm.py diff --git a/examples/calling_scripts/test_6-hump_camel_active_persistent_worker_abort.py b/examples/calling_scripts/test_6-hump_camel_active_persistent_worker_abort.py deleted file mode 120000 index 70e23cd6b..000000000 --- a/examples/calling_scripts/test_6-hump_camel_active_persistent_worker_abort.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_active_persistent_worker_abort.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_aposmm_LD_MMA.py b/examples/calling_scripts/test_6-hump_camel_aposmm_LD_MMA.py deleted file mode 120000 index a757eb29b..000000000 --- a/examples/calling_scripts/test_6-hump_camel_aposmm_LD_MMA.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_aposmm_LD_MMA.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_elapsed_time_abort.py b/examples/calling_scripts/test_6-hump_camel_elapsed_time_abort.py deleted file mode 120000 index a70d98a27..000000000 --- a/examples/calling_scripts/test_6-hump_camel_elapsed_time_abort.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_elapsed_time_abort.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py b/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py deleted file mode 120000 index a47e29bc9..000000000 --- a/examples/calling_scripts/test_6-hump_camel_persistent_aposmm_1.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_aposmm_1.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_persistent_uniform_sampling.py b/examples/calling_scripts/test_6-hump_camel_persistent_uniform_sampling.py deleted file mode 120000 index 0816635dc..000000000 --- a/examples/calling_scripts/test_6-hump_camel_persistent_uniform_sampling.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_uniform_sampling.py b/examples/calling_scripts/test_6-hump_camel_uniform_sampling.py deleted file mode 120000 index e88d8bc54..000000000 --- a/examples/calling_scripts/test_6-hump_camel_uniform_sampling.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py b/examples/calling_scripts/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py deleted file mode 120000 index 1564553fc..000000000 --- a/examples/calling_scripts/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py \ No newline at end of file diff --git a/examples/calling_scripts/test_6-hump_camel_with_different_nodes_uniform_sample.py b/examples/calling_scripts/test_6-hump_camel_with_different_nodes_uniform_sample.py deleted file mode 120000 index 9403febdc..000000000 --- a/examples/calling_scripts/test_6-hump_camel_with_different_nodes_uniform_sample.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py \ No newline at end of file diff --git a/examples/calling_scripts/test_branin_aposmm_nlopt_and_then_scipy.py b/examples/calling_scripts/test_branin_aposmm_nlopt_and_then_scipy.py deleted file mode 120000 index 3868553e8..000000000 --- a/examples/calling_scripts/test_branin_aposmm_nlopt_and_then_scipy.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_branin_aposmm_nlopt_and_then_scipy.py \ No newline at end of file diff --git a/examples/calling_scripts/test_chwirut_aposmm_one_residual_at_a_time.py b/examples/calling_scripts/test_chwirut_aposmm_one_residual_at_a_time.py deleted file mode 120000 index 2ab47bd5b..000000000 --- a/examples/calling_scripts/test_chwirut_aposmm_one_residual_at_a_time.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_chwirut_aposmm_one_residual_at_a_time.py \ No newline at end of file diff --git a/examples/calling_scripts/test_chwirut_pounders.py b/examples/calling_scripts/test_chwirut_pounders.py deleted file mode 120000 index fe1d9884c..000000000 --- a/examples/calling_scripts/test_chwirut_pounders.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_chwirut_pounders.py \ No newline at end of file diff --git a/examples/calling_scripts/test_chwirut_uniform_sampling_one_residual_at_a_time.py b/examples/calling_scripts/test_chwirut_uniform_sampling_one_residual_at_a_time.py deleted file mode 120000 index fc71f5fca..000000000 --- a/examples/calling_scripts/test_chwirut_uniform_sampling_one_residual_at_a_time.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_chwirut_uniform_sampling_one_residual_at_a_time.py \ No newline at end of file diff --git a/examples/calling_scripts/test_fast_alloc.py b/examples/calling_scripts/test_fast_alloc.py deleted file mode 120000 index d81cdd12e..000000000 --- a/examples/calling_scripts/test_fast_alloc.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_fast_alloc.py \ No newline at end of file diff --git a/examples/calling_scripts/test_jobcontroller_hworld.py b/examples/calling_scripts/test_jobcontroller_hworld.py deleted file mode 120000 index 3e864d5b0..000000000 --- a/examples/calling_scripts/test_jobcontroller_hworld.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_jobcontroller_hworld.py \ No newline at end of file diff --git a/examples/calling_scripts/test_nan_func_aposmm.py b/examples/calling_scripts/test_nan_func_aposmm.py deleted file mode 120000 index 9ba82fce2..000000000 --- a/examples/calling_scripts/test_nan_func_aposmm.py +++ /dev/null @@ -1 +0,0 @@ -../../libensemble/tests/regression_tests/test_nan_func_aposmm.py \ No newline at end of file From c85de4cbcb48c23ab5f27e50b8ee17b42c17faba Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 22:42:21 -0600 Subject: [PATCH 613/644] Update theta instructions --- docs/platforms/theta.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 23d45e72d..93e5acc1f 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -78,6 +78,11 @@ Initialize a new database similarly to the following (from the Balsam docs): Read Balsam's documentation here_. +.. note:: + Balsam will create the run directories inside the data sub-directory within the database + directory. From here files can be staged out to the user directory (see the example + batch script below). + Job Submission -------------- @@ -108,7 +113,7 @@ functions execute computationally expensive code, or code built for specific architectures. Recall also that only the MOM nodes can launch MPI jobs. Although libEnsemble workers on the MOM nodes can technically submit -user-applications to the compute nodes via ``aprun`` within user functions, it +user-applications to the compute nodes directly via ``aprun`` within user functions, it is highly recommended that the aforementioned :doc:`job_controller<../job_controller/overview>` interface is used instead. The libEnsemble job-controller features advantages like automatic resource-detection, portability, launch failure resilience, and ease-of-use. @@ -119,6 +124,20 @@ Theta features one default production queue, ``default``, and two debug queues, .. note:: For the default queue, the minimum number of nodes to allocate at once is 128 +Module and environment variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To ensure proper functioning of libEnsemble, including the ability to kill running jobs, it +recommended that the following environment variable is set:: + + export PMI_NO_FORK=1 + +It is also recommended that the following environment modules are unloaded, if present:: + + module unload trackdeps + module unload darshan + module unload xalt + Interactive Runs ^^^^^^^^^^^^^^^^ @@ -183,6 +202,11 @@ convenience function from libEnsemble's :doc:`utils module<../utilities>`. # Required for python kills on Theta export PMI_NO_FORK=1 + # Unload Theta modules that may interfere with job monitoring/kills + module unload trackdeps + module unload darshan + module unload xalt + python $EXE $COMMS $NWORKERS > out.txt 2>&1 With this saved as ``myscript.sh``, allocating, configuring, and queueing @@ -232,6 +256,11 @@ Here is an example Balsam submission script: # Required for python kills on Theta export PMI_NO_FORK=1 + # Unload Theta modules that may interfere with job monitoring/kills + module unload trackdeps + module unload darshan + module unload xalt + # Activate conda environment . activate $CONDA_ENV_NAME From 53697c23fbee3e31bfd52d1be9c4926388782315 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 26 Nov 2019 22:47:09 -0600 Subject: [PATCH 614/644] Remove unused error variable --- libensemble/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/resources.py b/libensemble/resources.py index 739194a51..1910935ca 100644 --- a/libensemble/resources.py +++ b/libensemble/resources.py @@ -158,7 +158,7 @@ def get_MPI_variant(): if 'unrecognized argument npernode' in stdout.decode(): return 'mpich' return 'openmpi' - except Exception as e: + except Exception: pass try: From 608eb9fd577640cdfd38aa5c67144282986d2008 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 27 Nov 2019 07:49:41 -0600 Subject: [PATCH 615/644] Just casing --- docs/platforms/example_scripts.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 7d6288edc..8fa2f63ac 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -5,14 +5,14 @@ Below are some example job-submission scripts used to configure and launch libEn on a variety of high-powered systems. See :doc:`here` for more information about the respective systems and configuration. -Bebop - Central Mode +Bebop - Central mode -------------------- .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh :caption: /examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh :language: bash -Bebop - Distributed Mode +Bebop - Distributed mode ------------------------ .. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm.sh @@ -26,21 +26,21 @@ Blues :caption: /examples/job_submission_scripts/blues_script.pbs :language: bash -Theta - On MOM with multiprocessing +Theta - On MOM nodes with multiprocessing ----------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/theta_submit_mproc.sh :caption: /examples/job_submission_scripts/theta_submit_mproc.sh :language: bash -Theta - Central Mode with Balsam +Theta - Central mode with Balsam -------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/theta_submit_balsam.sh :caption: /examples/job_submission_scripts/theta_submit_balsam.sh :language: bash -Summit - On Launch nodes with multiprocessing +Summit - On launch nodes with multiprocessing --------------------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/summit_submit_mproc.sh From bc70b6ee41b3b1ace8ccc384c871a1136be0e04c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 27 Nov 2019 07:51:15 -0600 Subject: [PATCH 616/644] Comma --- docs/platforms/theta.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 93e5acc1f..2d1ee4c7d 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -80,7 +80,7 @@ Read Balsam's documentation here_. .. note:: Balsam will create the run directories inside the data sub-directory within the database - directory. From here files can be staged out to the user directory (see the example + directory. From here, files can be staged out to the user directory (see the example batch script below). Job Submission From 99ab63de654806d838e91168e4b3e7f573b8db78 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 27 Nov 2019 07:56:19 -0600 Subject: [PATCH 617/644] Removing commented out line --- libensemble/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libensemble/utils.py b/libensemble/utils.py index f769c6365..434904225 100644 --- a/libensemble/utils.py +++ b/libensemble/utils.py @@ -362,7 +362,6 @@ def parse_args(): :doc:`(example)` """ - # args = parser.parse_args(sys.argv[1:]) args, unknown = parser.parse_known_args(sys.argv[1:]) front_ends = { 'mpi': _mpi_parse_args, From 9550a69d464ba39670a65d59db1f51deb42df88c Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 27 Nov 2019 08:07:39 -0600 Subject: [PATCH 618/644] Having fast_alloc send back all of the history (by default) --- libensemble/alloc_funcs/fast_alloc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index c8595207b..8cce6131b 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -28,6 +28,6 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # Give gen work persis_info['total_gen_calls'] += 1 gen_count += 1 - gen_work(Work, i, gen_specs['in'], [], persis_info[i]) + gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) return Work, persis_info From 614fa570c000fddccf10feb208f8fd983b1d47ce Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 27 Nov 2019 11:18:37 -0600 Subject: [PATCH 619/644] Fixing typos --- docs/platforms/example_scripts.rst | 4 ++-- libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh | 0 libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 8fa2f63ac..4af97bc74 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -26,8 +26,8 @@ Blues :caption: /examples/job_submission_scripts/blues_script.pbs :language: bash -Theta - On MOM nodes with multiprocessing ------------------------------------ +Theta - On MOM node with multiprocessing +---------------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/theta_submit_mproc.sh :caption: /examples/job_submission_scripts/theta_submit_mproc.sh diff --git a/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh b/libensemble/tests/scaling_tests/forces/summit_submit_mproc.sh old mode 100644 new mode 100755 diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index f79e5e2a7..c0d437cf0 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -92,7 +92,7 @@ balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 if [[ $LIBE_PLOTS = "true" ]]; then python $PLOT_DIR/plot_libe_calcs_util_v_time.py python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libe_histogram.pyfi + python $PLOT_DIR/plot_libe_histogram.py if [[ $BALSAM_PLOTS = "true" ]]; then # export MPLBACKEND=TkAgg From 6ae6059d5799ec95abd34497f9b8eb496a58d7ef Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 27 Nov 2019 12:27:59 -0600 Subject: [PATCH 620/644] Update CONTRIBUTING.rst to cover fork option --- CONTRIBUTING.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 86d20f1fc..921745e9e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,7 +7,10 @@ Contributions may be made via GitHub pull request to: libEnsemble uses the Gitflow model. Contributors should branch from, and make pull requests to, the develop branch. The master branch is used only -for releases. Code should pass flake8 tests, allowing for the exceptions +for releases. Pull requests may be made from a fork, for those without +repository write access. + +Code should pass flake8 tests, allowing for the exceptions given in the flake8_ file in the project directory. Issues can be raised at: From 5a0fe99a82feca68b9927663eb1a9141c6e0e642 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 27 Nov 2019 13:16:32 -0600 Subject: [PATCH 621/644] Add v0.6.0 release notes --- CHANGELOG.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d268c86a..2f37a8433 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,41 @@ Release Notes ============= + +Release 0.6.0 +------------- + +:Date: November 27, 2019 + +API changes: + +* sim/gen/alloc_specs options that do not directly involve these routines are moved to libE_specs (see docs) (#266, #269) +* sim/gen/alloc_specs now require user defined attributes to be added under a user field (see docs and examples) (#266, #269). +* Addition of a utils module to help users create calling scripts. Includes an argument parser and utility functions (#308). +* check_inputs() function is moved to the utils module (#308). +* The libE_specs option ``nprocesses`` has been changed to ``nworkers`` (#235) + +New example functions: + +* Addition of a new persistent APOSMM generator function (#217). + +Other changes: + +* Overhaul of documentation, including HPC platform guides and a new pdf structure (inc. #232, #282) +* Addition of OpenMP threading and GPU support to forces test (#250). +* Balsam job_controller now tested on Travis (#47) + +:Note: + +* Tested platforms include: Linux, MacOS, Theta (Cray XC40/Cobalt), Summit (IBM Power9/LSF), Bebop (Cray CS400/Slurm), Cori (Cray XC40/Slurm). +* Tested Python versions: (Cpython) 3.5, 3.6, 3.7 + +:Known issues: + +* These are unchanged from v0.5.0 +* A known issues section has now been added to documentation. + + Release 0.5.2 ------------- @@ -44,6 +79,7 @@ Release 0.5.1 * These are unchanged from v0.5.0 + Release 0.5.0 ------------- From 04e303cb0910751d54285618f8b973abfa865a60 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 27 Nov 2019 13:26:06 -0600 Subject: [PATCH 622/644] Update version number to v0.6.0 --- README.rst | 2 +- docs/conf.py | 4 ++-- libensemble/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index bb473a6be..6b88a43a3 100644 --- a/README.rst +++ b/README.rst @@ -210,7 +210,7 @@ Resources David Bindel and John-Luke Navarro}, title = {{libEnsemble} Users Manual}, institution = {Argonne National Laboratory}, - number = {Revision 0.5.2}, + number = {Revision 0.6.0}, year = {2019}, url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} } diff --git a/docs/conf.py b/docs/conf.py index 32ba4367e..d1f26d8c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -99,9 +99,9 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.5.2' +version = '0.6.0' # The full version, including alpha/beta/rc tags. -release = '0.5.2' +release = '0.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/libensemble/__init__.py b/libensemble/__init__.py index c4a9bdf34..317d863e4 100644 --- a/libensemble/__init__.py +++ b/libensemble/__init__.py @@ -4,7 +4,7 @@ Library to coordinate the concurrent evaluation of dynamic ensembles of calculations. """ -__version__ = "0.5.2" +__version__ = "0.6.0" __author__ = 'Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro' __credits__ = 'Argonne National Laboratory' diff --git a/setup.py b/setup.py index e8c6236ef..a2087d11e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def run_tests(self): setup( name='libensemble', - version='0.5.2', + version='0.6.0', description='Library to coordinate the concurrent evaluation of dynamic ensembles of calculations', url='https://github.com/Libensemble/libensemble', author='Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro', From 3d614f1d91e365dd7577ed86807f182098d61d59 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 27 Nov 2019 15:11:18 -0600 Subject: [PATCH 623/644] Slight re-wording --- CHANGELOG.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2f37a8433..32bf21497 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,6 @@ Release Notes ============= - Release 0.6.0 ------------- @@ -11,14 +10,14 @@ Release 0.6.0 API changes: * sim/gen/alloc_specs options that do not directly involve these routines are moved to libE_specs (see docs) (#266, #269) -* sim/gen/alloc_specs now require user defined attributes to be added under a user field (see docs and examples) (#266, #269). +* sim/gen/alloc_specs now require user-defined attributes to be added under the 'user' field (see docs and examples) (#266, #269). * Addition of a utils module to help users create calling scripts. Includes an argument parser and utility functions (#308). * check_inputs() function is moved to the utils module (#308). * The libE_specs option ``nprocesses`` has been changed to ``nworkers`` (#235) New example functions: -* Addition of a new persistent APOSMM generator function (#217). +* Addition of a persistent APOSMM generator function (#217). Other changes: @@ -34,8 +33,7 @@ Other changes: :Known issues: * These are unchanged from v0.5.0 -* A known issues section has now been added to documentation. - +* A known issues section has now been added to the documentation. Release 0.5.2 ------------- @@ -79,7 +77,6 @@ Release 0.5.1 * These are unchanged from v0.5.0 - Release 0.5.0 ------------- From d606b26a405e313149ab28238fbd1b33aa393a2f Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 27 Nov 2019 17:18:48 -0600 Subject: [PATCH 624/644] Update link to user_guide --- README.rst | 2 +- docs/introduction_latex.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6b88a43a3..ca282940f 100644 --- a/README.rst +++ b/README.rst @@ -241,5 +241,5 @@ Resources .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html +.. _user guide: https://libensemble.readthedocs.io/en/latest .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index fd02676cb..d82c1cac0 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -43,5 +43,5 @@ Quickstart Guide .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest/user_guide.html +.. _user guide: https://libensemble.readthedocs.io/en/latest .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info \ No newline at end of file From 4f0849cd1770f707090fb870484ec693bc641f0d Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 29 Nov 2019 15:21:39 -0600 Subject: [PATCH 625/644] Removing two flake8 checks --- .flake8 | 4 ---- libensemble/tests/balsam_tests/setup_balsam_tests.py | 3 ++- libensemble/tests/balsam_tests/test_balsam_1__runjobs.py | 3 ++- libensemble/tests/balsam_tests/test_balsam_2__workerkill.py | 3 ++- libensemble/tests/balsam_tests/test_balsam_3__managerkill.py | 3 ++- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.flake8 b/.flake8 index 6f4334ab3..e4c9b7033 100644 --- a/.flake8 +++ b/.flake8 @@ -20,12 +20,8 @@ exclude = # Keeping bad python format to match PETSc source code libensemble/sim_funcs/chwirut1.py examples/sim_funcs/chwirut1.py - # This is not implemented yet - controller_tests/test_jobcontroller_manager_poll.py # Not checking these scripts yet postproc_scripts - # Do not check these experimental tests - balsam_tests # Note that you can use wildcard exceptions with something such as # libensemble/libensemble/tests/regression_tests/*:F401 diff --git a/libensemble/tests/balsam_tests/setup_balsam_tests.py b/libensemble/tests/balsam_tests/setup_balsam_tests.py index 2e90e66db..07e0ffd5b 100755 --- a/libensemble/tests/balsam_tests/setup_balsam_tests.py +++ b/libensemble/tests/balsam_tests/setup_balsam_tests.py @@ -35,7 +35,8 @@ def run_cmd(cmd, echo=False): print("\nRunning %s ...\n" % cmd) try: subprocess.run(cmd.split(), check=True) - except: + except Exception as e: + print(e) raise("Error: Command %s failed to run" % cmd) diff --git a/libensemble/tests/balsam_tests/test_balsam_1__runjobs.py b/libensemble/tests/balsam_tests/test_balsam_1__runjobs.py index 282e950ad..7cf4eaca0 100644 --- a/libensemble/tests/balsam_tests/test_balsam_1__runjobs.py +++ b/libensemble/tests/balsam_tests/test_balsam_1__runjobs.py @@ -30,7 +30,8 @@ def poll_until_state(job, state, timeout_sec=60.0, delay=2.0): if not os.path.isdir(sim_path): try: os.mkdir(sim_path) - except: + except Exception as e: + print(e) raise("Cannot make simulation directory %s" % sim_path) MPI.COMM_WORLD.Barrier() # Ensure output dir created diff --git a/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py b/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py index 6eeea7ba5..97d0b1de3 100644 --- a/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py +++ b/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py @@ -32,7 +32,8 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): if not os.path.isdir(sim_path): try: os.mkdir(sim_path) - except: + except Exception as e: + print(e) raise("Cannot make simulation directory %s" % sim_path) MPI.COMM_WORLD.Barrier() # Ensure output dir created diff --git a/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py b/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py index 3fc887264..257ac6569 100644 --- a/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py +++ b/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py @@ -32,7 +32,8 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): if not os.path.isdir(sim_path): try: os.mkdir(sim_path) - except: + except Exception as e: + print(e) raise("Cannot make simulation directory %s" % sim_path) MPI.COMM_WORLD.Barrier() # Ensure output dir created From b4bdff28b3668e460c2d796babd53cff37233187 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Fri, 29 Nov 2019 18:29:45 -0600 Subject: [PATCH 626/644] Having these tests be as close as possible. (Not sure why slight differences.) --- libensemble/tests/balsam_tests/test_balsam_2__workerkill.py | 3 --- libensemble/tests/balsam_tests/test_balsam_3__managerkill.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py b/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py index 97d0b1de3..e109ed0be 100644 --- a/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py +++ b/libensemble/tests/balsam_tests/test_balsam_2__workerkill.py @@ -37,10 +37,8 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): raise("Cannot make simulation directory %s" % sim_path) MPI.COMM_WORLD.Barrier() # Ensure output dir created - print("Host job rank is %d Output dir is %s" % (myrank, sim_dir)) - start = time.time() for sim_id in range(steps): jobname = 'outfile_t2_' + 'for_sim_id_' + str(sim_id) + '_ranks_' + str(myrank) + '.txt' @@ -53,7 +51,6 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): ranks_per_node=8, stage_out_url="local:" + sim_path, stage_out_files=jobname + ".out") - if sim_id == 1: dag.kill(current_job) diff --git a/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py b/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py index 257ac6569..8b3ed000e 100644 --- a/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py +++ b/libensemble/tests/balsam_tests/test_balsam_3__managerkill.py @@ -39,7 +39,6 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): print("Host job rank is %d Output dir is %s" % (myrank, sim_dir)) - start = time.time() for sim_id in range(steps): jobname = 'outfile_t3_' + 'for_sim_id_' + str(sim_id) + '_ranks_' + str(myrank) + '.txt' @@ -81,6 +80,5 @@ def poll_until_state(job, state, timeout_sec=120.0, delay=2.0): else: print("Job not completed: %s rank=%d time=%f Status" % (jobname, myrank, time.time()-start), current_job.state) - end = time.time() print("Done: rank=%d time=%f" % (myrank, end-start)) From 1bb3144700cb8c1fb5865308ec565a8886da65e2 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Sun, 1 Dec 2019 08:29:12 -0600 Subject: [PATCH 627/644] Whitespace adding 'Email mailing list' to release process --- .travis.yml | 12 ------------ conda/run_travis_locally/readme.md | 7 ------- docs/conf.py | 14 +++++++++----- .../release_management/release_process.rst | 2 ++ .../job_submission_scripts/bebop_submit_slurm.sh | 1 - .../bebop_submit_slurm_centralmode.sh | 1 - libensemble/tests/run-tests.sh | 8 -------- libensemble/tests/scaling_tests/forces/readme.md | 4 ---- .../scaling_tests/forces/theta_submit_balsam.sh | 1 - 9 files changed, 11 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f2a2f793..f9953b8f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,8 @@ python: - 3.6 - 3.7 - os: linux - env: global: - HYDRA_LAUNCHER=fork @@ -17,7 +15,6 @@ env: matrix: - MPI=mpich - matrix: include: - os: osx @@ -26,16 +23,13 @@ matrix: language: generic python: 3.7 - services: - postgresql - cache: pip: true apt: true - before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh -O miniconda.sh; @@ -55,7 +49,6 @@ before_install: fi - source activate condaenv - install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then COMPILERS=clang_osx-64; @@ -81,25 +74,20 @@ install: - mkdir ../balsam; tar xf 0.3.5.1.tar.gz -C ../balsam; - python conda/install-balsam.py - before_script: - flake8 libensemble - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - source setbalsampath.sh # Imperfect method for env var persist after setup - ulimit -Sn 10000 # More concurrent file descriptors (for persis aposmm) - - # Run test (-z show output) script: - ./libensemble/tests/run-tests.sh -z - # Track code coverage after_success: - mv libensemble/tests/.cov* . - coveralls - after_failure: - cat libensemble/tests/regression_tests/log.err diff --git a/conda/run_travis_locally/readme.md b/conda/run_travis_locally/readme.md index c0476c606..23cb3c75e 100644 --- a/conda/run_travis_locally/readme.md +++ b/conda/run_travis_locally/readme.md @@ -33,7 +33,6 @@ Add user name to docker group so dont need sudo before docker: su - ${USER} id -nG # Confirm username added to docker users - ### Install Travis Image for Python and create a container Name your new travis container: @@ -52,7 +51,6 @@ $CONTAINER is the name you are assigning to the new container made from the imag Alternative travisCI docker images can be found [here](https://hub.docker.com/r/travisci/ci-garnet/tags/). - ### Start a shell session in the container Then open a shell in running container: @@ -61,7 +59,6 @@ Then open a shell in running container: Prompt should say travis@ rather than root@: - ### Build and run libEnsemble for a given Python version You now want to copy the latest build script to the container and run it. @@ -69,7 +66,6 @@ You now want to copy the latest build script to the container and run it. The default build script is build_mpich_libE.sh. If this is not up to date, check the installs against .travis.yml in the top level libEnsemble package directory. - #### Copy build script from host system to the running container Run the following **from your host machine environment**. This copies the given file into an existing @@ -82,7 +78,6 @@ On the docker side you may need to set ownership of the script: chown travis:travis /home/travis/build_mpich_libE.sh - #### Run the libEnsemble build-and-run script Now, in the docker container, become user travis: @@ -102,7 +97,6 @@ be left in the same environment for debugging. The script should stop running if If the conda output is too verbose, remove the "set -x" line in the script. - ### Saving images from built containers and re-starting containers To exit the container session (client). @@ -153,7 +147,6 @@ You can restart from your new image with docker run and docker exec or to run se where / is as used above to save (or from first column in "docker images" output). - ### Notes The CPU cores available will be all those on your machine, not just what Travis supports. diff --git a/docs/conf.py b/docs/conf.py index d1f26d8c4..a56d6389a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -180,11 +180,15 @@ def __getattr__(cls, name): \protected\def\sphinxcrossref#1{\texttt{#1}} \newsavebox\mytempbox -\definecolor{sphinxnoteBgColor}{RGB}{221,233,239} -\renewenvironment{sphinxnote}[1] - {\begin{lrbox}{\mytempbox}\begin{minipage}{\columnwidth}% - \begin{sphinxlightbox}\sphinxstrong{#1} } - {\end{sphinxlightbox}\end{minipage}\end{lrbox}% + \definecolor{sphinxnoteBgColor}{RGB}{221,233,239} + \renewenvironment{sphinxnote}[1]{% + \begin{lrbox}{\mytempbox}% + \begin{minipage}{\columnwidth}% + \begin{sphinxlightbox}% + \sphinxstrong{#1}}% + {\end{sphinxlightbox}% + \end{minipage}% + \end{lrbox}% \colorbox{sphinxnoteBgColor}{\usebox{\mytempbox}}} ''', # The paper size ('letterpaper' or 'a4paper'). diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index 54c062e94..6094953ed 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -56,3 +56,5 @@ After release ------------- - Ensure all relevant GitHub issues are closed and moved to the *Done* column on the kanban project board (inc. the release checklist). + +- Email libEnsemble mailing list diff --git a/examples/job_submission_scripts/bebop_submit_slurm.sh b/examples/job_submission_scripts/bebop_submit_slurm.sh index fa2e07a18..c80c2ee07 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm.sh @@ -20,7 +20,6 @@ unset I_MPI_FABRICS export I_MPI_FABRICS_LIST=tmi,tcp export I_MPI_FALLBACK=1 - #If using in calling script (After N mins manager kills workers and timing.dat created.) export LIBE_WALLCLOCK=55 diff --git a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh index 700a8da06..58321c120 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh @@ -18,7 +18,6 @@ #Currently requires even distribution - either multiple workers per node or nodes per worker - #User to edit these variables export EXE=libE_calling_script.py export NUM_WORKERS=4 diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index a481b6098..fd582fc1a 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -163,7 +163,6 @@ cleanup() { #----------------------------------------------------------------------------------------- - #Parse Options #set -x @@ -279,7 +278,6 @@ if [[ $root_found == "false" ]]; then done fi; - if [ $CLEAN_ONLY = "true" ]; then if [ "$root_found" = true ]; then cleanup @@ -329,8 +327,6 @@ if [ $RUN_COV_TESTS = "true" ]; then #COV_LINE_PARALLEL='-m coverage run --branch --parallel-mode' fi; - - if [ "$root_found" = true ]; then #Running without subdirs - delete any leftover output and coverage data files @@ -368,7 +364,6 @@ if [ "$root_found" = true ]; then fi; cd $ROOT_DIR/ - # Run Regression Tests ----------------------------------------------------------------- if [ "$RUN_REG_TESTS" = true ]; then @@ -501,7 +496,6 @@ if [ "$root_found" = true ]; then # ********* End Loop over regression tests ********* - cd $ROOT_DIR/$REG_TEST_SUBDIR #Create Coverage Reports ---------------------------------------------- @@ -547,7 +541,6 @@ if [ "$root_found" = true ]; then fi; fi; - #All reg tests - summary ---------------------------------------------- if [ "$code" -eq "0" ]; then echo @@ -585,7 +578,6 @@ if [ "$root_found" = true ]; then fi; #$RUN_REG_TESTS - # Run Code standards Tests ----------------------------------------- cd $ROOT_DIR if [ "$RUN_PEP_TESTS" = true ]; then diff --git a/libensemble/tests/scaling_tests/forces/readme.md b/libensemble/tests/scaling_tests/forces/readme.md index 2ab01d893..7cd166e32 100644 --- a/libensemble/tests/scaling_tests/forces/readme.md +++ b/libensemble/tests/scaling_tests/forces/readme.md @@ -5,7 +5,6 @@ Naive Electostatics Code Test This is designed only as an artificial, highly conifurable test code for a libEnsemble sim func. - ### Forces Mini-App A system of charged particles is set up and simulated over a number of time-steps. @@ -21,7 +20,6 @@ To run forces as a standalone executable on N procs: mpirun -np N ./forces.x - ### Running with libEnsemble. A random sample of seeds is taken and used as imput to the sim func (forces miniapp). @@ -45,7 +43,6 @@ To remove output before the next run: ./cleanup.sh - ### Running with Balsam To run with balsam, set `USE_MPI = True` and `USE_BALSAM = True` in `run_libe_forces.py`. @@ -61,7 +58,6 @@ While the key output files will be copied back to the run dir at completion. Als the log in `

/log` if there are any issues. To run on batch systems, see the example scripts such as `theta_submit_balsam.sh`. - ### Using batch scripts The scripts are set up assuming a conda environment. To use script directly you will need to replace the following templated values: diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh index c0d437cf0..12531f5a5 100755 --- a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh +++ b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh @@ -77,7 +77,6 @@ RANKS_PER_NODE=64 # All jobs OUT_FILES_TO_RETURN="*.out *.txt *.log" - balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $LIBE_WALLCLOCK --num-nodes $NUM_NODES --ranks-per-node $RANKS_PER_NODE --url-out="local:/$THIS_DIR" --stage-out-files="${OUT_FILES_TO_RETURN}" --url-in="local:/$THIS_DIR/*" --yes From 34726dcd39902bb5188a2de3d70c52680a0ed43f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Sun, 1 Dec 2019 20:05:14 -0600 Subject: [PATCH 628/644] Update rst_formatting_guidelines --- docs/rst_formatting_guidelines | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/rst_formatting_guidelines b/docs/rst_formatting_guidelines index bedb5707f..22ecd7a3d 100644 --- a/docs/rst_formatting_guidelines +++ b/docs/rst_formatting_guidelines @@ -14,8 +14,3 @@ Put breakout code snippets into a box using two colons :: No italics on text of links Use italics for emphasis - - -* Unfortunately, the Sphinx crossrefs (e.g., :doc:`ref`) are different then -* hrefs, and they are italicized in the pdf. I assume there is a way to change -* this formatting, but I couldn't find it. From dabca2e2632a81a319bb29ad21847154fe420492 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Sun, 1 Dec 2019 20:25:55 -0600 Subject: [PATCH 629/644] Trying this out --- docs/FAQ.rst | 3 +- docs/data_structures/calc_status.rst | 10 ++- docs/data_structures/work_dict.rst | 7 +- .../release_platforms/rel_spack.rst | 10 +-- .../release_management/release_process.rst | 12 ++-- docs/job_controller/jc_index.rst | 4 +- docs/job_controller/job_controller.rst | 14 ++-- docs/job_controller/mpi_controller.rst | 8 ++- docs/rst_formatting_guidelines | 1 - docs/tutorials/local_sine_tutorial.rst | 12 ++-- .../balsam_tests/readme.balsam_tests.txt | 12 +++- libensemble/tests/balsam_tests/readme.rst | 64 +++++++++++++------ libensemble/tests/controller_tests/readme.txt | 9 ++- .../standalone_tests/comms_test/readme.txt | 13 +++- .../kill_test/log.autotest.txt | 4 +- postproc_scripts/readme.rst | 28 ++++++-- 16 files changed, 152 insertions(+), 59 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index cf27eac86..d8408e690 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -145,7 +145,8 @@ There are several ways to address this nuisance, but all involve trial and error An easy (but insecure) solution is temporarily disabling the Firewall through System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming connections" rule can be attempted for the offending -Job Controller executable. We've had limited success running ``sudo codesign --force --deep --sign - /path/to/application.app`` +Job Controller executable. We've had limited success running +``sudo codesign --force --deep --sign - /path/to/application.app`` on our Job Controller executables, then confirming the next alerts for the executable and ``mpiexec.hydra``. diff --git a/docs/data_structures/calc_status.rst b/docs/data_structures/calc_status.rst index 2091f1246..565ca9b93 100644 --- a/docs/data_structures/calc_status.rst +++ b/docs/data_structures/calc_status.rst @@ -3,7 +3,15 @@ calc_status =========== -The ``calc_status`` is an integer attribute with named (enumerated) values and a corresponding description that can be used in :ref:`sim_f` or :ref:`gen_f` functions to capture the status of a calcuation. This is returned to the manager and printed to the ``libE_stats.txt`` file. Only the status values ``FINISHED_PERSISTENT_SIM_TAG`` and ``FINISHED_PERSISTENT_GEN_TAG`` are currently used by the manager, but others can still provide a useful summary in libE_stats.txt. The user determines the status of the calculation, as it could include multiple application runs. It can be added as a third return variable in sim_f or gen_f functions. +The ``calc_status`` is an integer attribute with named (enumerated) values and +a corresponding description that can be used in :ref:`sim_f` or +:ref:`gen_f` functions to capture the status of a calcuation. This +is returned to the manager and printed to the ``libE_stats.txt`` file. Only the +status values ``FINISHED_PERSISTENT_SIM_TAG`` and +``FINISHED_PERSISTENT_GEN_TAG`` are currently used by the manager, but others +can still provide a useful summary in libE_stats.txt. The user determines the +status of the calculation, as it could include multiple application runs. It +can be added as a third return variable in sim_f or gen_f functions. The calc_status codes are in the ``libensemble.message_numbers`` module. Example of ``calc_status`` used along with :ref:`job controller` in sim_f: diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 14730b2e5..376992370 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -20,8 +20,11 @@ Dictionary with integer keys ``i`` and dictionary values to be given to worker ` 'persistent' [bool]: True if worker 'i' will enter persistent mode .. seealso:: - For allocation functions giving Work dictionaries using persistent workers, see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. - For a use case where the allocation and generator functions combine to do simulation evaluations with different resources (blocking some workers), see `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. + For allocation functions giving Work dictionaries using persistent workers, + see `start_only_persistent.py`_ or `start_persistent_local_opt_gens.py`_. + For a use case where the allocation and generator functions combine to do + simulation evaluations with different resources (blocking some workers), see + `test_6-hump_camel_with_different_nodes_uniform_sample.py`_. .. _start_only_persistent.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_only_persistent.py .. _start_persistent_local_opt_gens.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/alloc_funcs/start_persistent_local_opt_gens.py diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst index c2f941461..598159b61 100644 --- a/docs/dev_guide/release_management/release_platforms/rel_spack.rst +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -13,9 +13,11 @@ This assumes you have already: Details on how to create forks can be found at: https://help.github.com/articles/fork-a-repo -You now have a configuration like shown in answer at: https://stackoverflow.com/questions/6286571/are-git-forks-actually-git-clones +You now have a configuration like shown in answer at: +https://stackoverflow.com/questions/6286571/are-git-forks-actually-git-clones -Upstream, in this case, is the official Spack repository on GitHub. Origin is your fork on GitHub and Local Machine is your local clone (from your fork). +Upstream, in this case, is the official Spack repository on GitHub. Origin is +your fork on GitHub and Local Machine is your local clone (from your fork). Make sure ``SPACK_ROOT`` is set and spack binary is in your path:: @@ -37,8 +39,8 @@ To set upstream repo:: Now to update (the main develop branch) --------------------------------------- -You will now update your local machine from the upstream repo (if in doubt - make a copy of local repo -in your filestystem before doing the following). +You will now update your local machine from the upstream repo (if in doubt - +make a copy of local repo in your filestystem before doing the following). Check upstream remote is present:: diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index 6094953ed..71fb215b4 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -13,9 +13,11 @@ Before release - A release branch should be taken off develop (or develop pulls controlled). -- Release notes for this version are added to the documentation with release date, including a list of supported (tested) platforms. +- Release notes for this version are added to the documentation with release + date, including a list of supported (tested) platforms. -- Version number is updated wherever it appears (in ``setup.py``, ``libensemble/__init__.py``, ``README.rst`` and twice in ``docs/conf.py``) +- Version number is updated wherever it appears: + (in ``setup.py``, ``libensemble/__init__.py``, ``README.rst`` and twice in ``docs/conf.py``) - Check year is correct in ``README.rst`` under *Citing libEnsemble* and in ``docs/conf.py``. @@ -31,7 +33,8 @@ Before release - Documentation must build and display correctly wherever hosted (currently readthedocs.com). -- Pull request from either develop or release branch to master requesting reviewer/s (including at least one other administrator). +- Pull request from either develop or release branch to master requesting + reviewer/s (including at least one other administrator). - Reviewer will check tests have passed and approve merge. @@ -55,6 +58,7 @@ An administrator will take the following steps. After release ------------- -- Ensure all relevant GitHub issues are closed and moved to the *Done* column on the kanban project board (inc. the release checklist). +- Ensure all relevant GitHub issues are closed and moved to the *Done* column + on the kanban project board (inc. the release checklist). - Email libEnsemble mailing list diff --git a/docs/job_controller/jc_index.rst b/docs/job_controller/jc_index.rst index 16bed80a8..28482b2da 100644 --- a/docs/job_controller/jc_index.rst +++ b/docs/job_controller/jc_index.rst @@ -3,7 +3,9 @@ Job Controller ============== -The job controller can be used within the simulator (and potentially generator) functions to provide a simple, portable interface for running and managing user jobs. +The job controller can be used within the simulator (and potentially generator) +functions to provide a simple, portable interface for running and managing user +jobs. .. toctree:: :maxdepth: 2 diff --git a/docs/job_controller/job_controller.rst b/docs/job_controller/job_controller.rst index 2e6b59e50..113544498 100644 --- a/docs/job_controller/job_controller.rst +++ b/docs/job_controller/job_controller.rst @@ -18,9 +18,10 @@ See the controller APIs for optional arguments. Job Class --------- -Jobs are created and returned though the job_controller launch function. Jobs can be polled and -killed with the respective poll and kill functions. Job information can be queried through the job attributes -below and the query functions. Note that the job attributes are only updated when they are +Jobs are created and returned though the job_controller launch function. Jobs +can be polled and killed with the respective poll and kill functions. Job +information can be queried through the job attributes below and the query +functions. Note that the job attributes are only updated when they are polled/killed (or through other job or job controller functions). .. autoclass:: Job @@ -32,9 +33,12 @@ polled/killed (or through other job or job controller functions). Job Attributes -------------- -Following is a list of job status and configuration attributes that can be retrieved from a job. +Following is a list of job status and configuration attributes that can be +retrieved from a job. -:NOTE: These should not be set directly. Jobs are launched by the job controller and job information can be queired through the job attributes below and the query functions. +:NOTE: These should not be set directly. Jobs are launched by the job + controller and job information can be queired through the job attributes + below and the query functions. Job Status attributes include: diff --git a/docs/job_controller/mpi_controller.rst b/docs/job_controller/mpi_controller.rst index 70385d42b..fb1dd8670 100644 --- a/docs/job_controller/mpi_controller.rst +++ b/docs/job_controller/mpi_controller.rst @@ -22,7 +22,9 @@ See the controller API below for optional arguments. Class specific attributes ------------------------- -These attributes can be set directly to alter behaviour of the MPI job controller. However, they should be used with caution, as they may not be implemented in other job controllers. +These attributes can be set directly to alter behaviour of the MPI job +controller. However, they should be used with caution, as they may not be +implemented in other job controllers. :max_launch_attempts: (int) Maximum number of launch attempts for a given job. *Default: 5*. :fail_time: (int) *Only if wait_on_run is set.* Maximum run-time to failure in seconds that results in re-launch. *Default: 2*. @@ -33,4 +35,6 @@ Example. To increase resilience against launch failures:: jobctrl.max_launch_attempts = 10 jobctrl.fail_time = 5 -Note that an the re-try delay on launches starts at 5 seconds and increments by 5 seconds for each retry. So the 4th re-try will wait for 20 seconds before re-launching. +Note that an the re-try delay on launches starts at 5 seconds and increments by +5 seconds for each retry. So the 4th re-try will wait for 20 seconds before +re-launching. diff --git a/docs/rst_formatting_guidelines b/docs/rst_formatting_guidelines index bedb5707f..862ae4556 100644 --- a/docs/rst_formatting_guidelines +++ b/docs/rst_formatting_guidelines @@ -1,4 +1,3 @@ - Put variables (e.g., gen_f or H) in between double ticks ``stuff here`` (no quotes) Put short code in between double ticks ``stuff here`` (no quotes) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 30a8beed6..a812a20bd 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -8,10 +8,14 @@ Multiprocessing. The foundation of writing libEnsemble routines is accounting for four components: - 1. The generator function :ref:`gen_f`, which produces values for simulations. - 2. The simulator function :ref:`sim_f`, which performs simulations based on values from ``gen_f``. - 3. The allocation function :ref:`alloc_f`, which decides which of the previous two functions should be called, when. - 4. The calling script, which defines parameters and information about these functions and the libEnsemble task, then begins execution. + 1. The generator function :ref:`gen_f`, which produces values + for simulations. + 2. The simulator function :ref:`sim_f`, which performs + simulations based on values from ``gen_f``. + 3. The allocation function :ref:`alloc_f`, which decides which + of the previous two functions should be called, when. + 4. The calling script, which defines parameters and information about these + functions and the libEnsemble task, then begins execution. libEnsemble initializes a *manager* process and as many *worker* processes as the user requests. The manager coordinates data-transfer between workers and assigns diff --git a/libensemble/tests/balsam_tests/readme.balsam_tests.txt b/libensemble/tests/balsam_tests/readme.balsam_tests.txt index fd19a9616..7e5b8b342 100644 --- a/libensemble/tests/balsam_tests/readme.balsam_tests.txt +++ b/libensemble/tests/balsam_tests/readme.balsam_tests.txt @@ -1,12 +1,18 @@ List of balsam test jobs for libensemble. test_balsam_1__runjobs.py: -Launches parallel parent job. Each task iterates over a loop launching sub-jobs and waiting for completion. Does 3 iterations by default. +Launches parallel parent job. Each task iterates over a loop launching sub-jobs +and waiting for completion. Does 3 iterations by default. test_balsam_2__workerkill.py: As first test but each top-level task kills its middle job. test_balsam_3__managerkill.py: -Process 0 sends out a kill for jobs which include sim_id_1 in their name. This will kill any such jobs in the database at the time of the kill. Note that if a job has not yet been added to database at the time of the kill, it will still run. +Process 0 sends out a kill for jobs which include sim_id_1 in their name. This +will kill any such jobs in the database at the time of the kill. Note that if a +job has not yet been added to database at the time of the kill, it will still +run. -Note that test3 exploits the fact that balsam is built on a django database, and all django functionality for manipulating the database can easily be exposed. +Note that test3 exploits the fact that balsam is built on a django database, +and all django functionality for manipulating the database can easily be +exposed. diff --git a/libensemble/tests/balsam_tests/readme.rst b/libensemble/tests/balsam_tests/readme.rst index b809af293..4d60d668a 100644 --- a/libensemble/tests/balsam_tests/readme.rst +++ b/libensemble/tests/balsam_tests/readme.rst @@ -4,10 +4,14 @@ Using balsam Note: To set up balsam - see instructions on setting up in balsam docs. -Currently this can be found in the docs/ subdirectory of the hpc-edge-service repo. The documentation is in Sphinx format. +Currently this can be found in the docs/ subdirectory of the hpc-edge-service +repo. The documentation is in Sphinx format. Theta: -If the instructions are followed to set up a conda environment called balsam, then the script env_setup_theta.sh, under the balsam tests dir, can be sourced in future logins for theta. Or modified for other platforms. In some platforms you may need to only activate the balsam conda environment. +If the instructions are followed to set up a conda environment called balsam, +then the script env_setup_theta.sh, under the balsam tests dir, can be sourced +in future logins for theta. Or modified for other platforms. In some platforms +you may need to only activate the balsam conda environment. ----------------------------------------- Quickstart - Balsam tests for libEnsemble @@ -24,7 +28,8 @@ Go to balsam_tests dir run setup script: cd libensemble/code/tests/balsam_tests ./setup_balsam_tests.py -This will register the applications and jobs in the database. If work_dir and sim_dir paths are ok in script, it can be run from anywhere. +This will register the applications and jobs in the database. If work_dir and +sim_dir paths are ok in script, it can be run from anywhere. **ii) Launch the jobs** @@ -34,11 +39,14 @@ In an interactive session (or a batch script) run the launcher: balsam launcher --consume-all -Note: You will need at least 2 nodes, one for parent job and one for user apps, if you are on a machine where the schedular does not split nodes. However, it is recommended that 5 nodes are used for good concurrency. +Note: You will need at least 2 nodes, one for parent job and one for user apps, +if you are on a machine where the schedular does not split nodes. However, it +is recommended that 5 nodes are used for good concurrency. **iii) Output and reset** -Output will go into test output dirs inside the balsam_tests dir - they will be created on first run. +Output will go into test output dirs inside the balsam_tests dir - they will be +created on first run. To quickly reset the tests to run again use the reset (python) script: @@ -55,7 +63,8 @@ General Usage 1. Register applications ------------------------ -You must register with balsam the parent application (eg. libEnsemble) and any user application (eg. sim funcs/gen funcs). +You must register with balsam the parent application (eg. libEnsemble) and any +user application (eg. sim funcs/gen funcs). This only need be done once - unless eg. name of application changes. Example (as used in tests) - run from within balsam_tests dir: @@ -76,7 +85,8 @@ Register user application that will be called inside tests: balsam app --name helloworld --exec ../../examples/sim_funcs/helloworld.py --desc "Run helloworld user app" -Note: The --exec arg is the location of the script or exe, and can be absolute or relative path (the balsam app command will convert to fullpath). +Note: The --exec arg is the location of the script or exe, and can be absolute +or relative path (the balsam app command will convert to fullpath). To list apps: @@ -93,7 +103,8 @@ To clean: 2 Register job/s ---------------- -This is the job you intend to run. It will reference an application you have set up. +This is the job you intend to run. It will reference an application you have +set up. Eg. Set up job for test_balsam_1: @@ -107,14 +118,25 @@ Where WORK_DIR is set to output dir for job. --wall-min 1 --num-nodes 1 --ranks-per-node 4 --url-out="local:$WORK_DIR" --stage-out-files="job_test_balsam_1__runjobs*" -A working directory is set up when the job is run - by default under the balsam space eg: hpc-edge-service/data/balsamjobs/ Under this directory a workflow directory is created (eg. libe_workflow in above case). From there, files to keep are staged out as specified by directory in --url-out (use local: for file directory). The files to stage out are specified by --stage-out-files. +A working directory is set up when the job is run - by default under the balsam +space eg: hpc-edge-service/data/balsamjobs/ Under this directory a workflow +directory is created (eg. libe_workflow in above case). From there, files to +keep are staged out as specified by directory in --url-out (use local: for file +directory). The files to stage out are specified by --stage-out-files. A log will also be created when run under hpc-edge-service/log/ -The standard output will go to file .out. So in above case this will be job_balsam1.out which will be staged out to $WORKDIR +The standard output will go to file .out. So in above case this will +be job_balsam1.out which will be staged out to $WORKDIR -In this case 4 ranks per node and 1 node are selected. This is for running on the parent application (e.g., libEnsemble). This does not constrain the running of sub-apps (eg. helloworld), which will use the full allocation available. +In this case 4 ranks per node and 1 node are selected. This is for running on +the parent application (e.g., libEnsemble). This does not constrain the running +of sub-apps (eg. helloworld), which will use the full allocation available. -Note that the user jobs (launched in a libEnsemble job) are registered from within the code. For staging out files, the output directory needs to somehow be accessible to the code. For the tests here, this is simply the directory of the test scripts (accessed via the __file__ variable in python). Search for dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) +Note that the user jobs (launched in a libEnsemble job) are registered from +within the code. For staging out files, the output directory needs to somehow +be accessible to the code. For the tests here, this is simply the directory of +the test scripts (accessed via the __file__ variable in python). Search for +dag.add_job in test scripts (eg. test_balsam_1__runjobs.py) To list jobs: @@ -150,9 +172,11 @@ For other launcher options: A script to reset the tests is available: reset_balsam_tests.py -This script can be modified easily. However, to reset from the command line - without removing and re-adding jobs you can do the following. +This script can be modified easily. However, to reset from the command line - +without removing and re-adding jobs you can do the following. -Note: After running tests the balsam job database will contain something like the following (job_ids abbreviated for space): +Note: After running tests the balsam job database will contain something like +the following (job_ids abbreviated for space): .. code-block:: bash @@ -186,7 +210,8 @@ To remove only the generated jobs you can just use a sub-string of the job name ----------------------------------------------------------------------------------------------------------------------- 29add031-8e7c-... | job_balsam1 | libe_workflow | test_balsam_1 | [01-30-2018 18:57:47 JOB_FINISHED] -To run again - change status attribute to READY (you need to specify job_id - an abbreviation is ok) eg: +To run again - change status attribute to READY (you need to specify job_id - +an abbreviation is ok) For example: .. code-block:: bash @@ -203,11 +228,14 @@ Interactive sessions can be launched as: qsub -A -n 5 -q debug-flat-quad -t 60 -I -This would be a 60 minute interactive session with 5 nodes. You must have a project code. +This would be a 60 minute interactive session with 5 nodes. You must have a +project code. -You will need to load the conda env in the interactive session - or source the script env_setup_theta.sh. +You will need to load the conda env in the interactive session - or source the +script env_setup_theta.sh. -At time of writing theta does not log you out of interactive sessions. But jobs launched after time is up will not work. +At time of writing theta does not log you out of interactive sessions. But jobs +launched after time is up will not work. To see time remaining: diff --git a/libensemble/tests/controller_tests/readme.txt b/libensemble/tests/controller_tests/readme.txt index ef7459e8d..a043af374 100644 --- a/libensemble/tests/controller_tests/readme.txt +++ b/libensemble/tests/controller_tests/readme.txt @@ -24,7 +24,9 @@ For running with Balsam 1. You must install Balsam (https://balsam.alcf.anl.gov/) -2. For each time you start a new session: In this directory (controller_tests) initialise database and start the database server by sourcing script set.balsam.database.sh. You can modify the database location in the script. +2. For each time you start a new session: In this directory (controller_tests) +initialise database and start the database server by sourcing script +set.balsam.database.sh. You can modify the database location in the script. > . set.balsam.database.sh @@ -46,4 +48,7 @@ and: > balsam launcher --consume-all -Note: On systems like Theta only one job can be launched per node. You will need at least 2 nodes. The first will run the parent script. The test test_jobcontroller_multi.py launches 3 sub-jobs which Balsam will launch as nodes become available. To run all 3 concurrently would require 4 nodes. +Note: On systems like Theta only one job can be launched per node. You will +need at least 2 nodes. The first will run the parent script. The test +test_jobcontroller_multi.py launches 3 sub-jobs which Balsam will launch as +nodes become available. To run all 3 concurrently would require 4 nodes. diff --git a/libensemble/tests/standalone_tests/comms_test/readme.txt b/libensemble/tests/standalone_tests/comms_test/readme.txt index 872667ae2..381009c89 100644 --- a/libensemble/tests/standalone_tests/comms_test/readme.txt +++ b/libensemble/tests/standalone_tests/comms_test/readme.txt @@ -1,13 +1,20 @@ Comms test ========== -This is a simplified standalone comms tests which is a very basic model of the communications in libensemble. +This is a simplified standalone comms tests which is a very basic model of the +communications in libensemble. The current test, only models the commuications from workers to manager. -The test is easily configurable for size of data, rounds of communication, and can be run on any number of processors. The data is initailsed on each worker using the processor ID. The manager checks the correct values are received. +The test is easily configurable for size of data, rounds of communication, and +can be run on any number of processors. The data is initailsed on each worker +using the processor ID. The manager checks the correct values are received. -The data is a dictionary containing a scalar and numpy array. The size of the numpy array is given by . The default is one million values. The variable determines the number of messages sent from each worker. Two lines can be uncommented in the manager loop which gives the message count and size of the received message in bytes. +The data is a dictionary containing a scalar and numpy array. The size of the +numpy array is given by . The default is one million values. The +variable determines the number of messages sent from each worker. Two +lines can be uncommented in the manager loop which gives the message count and +size of the received message in bytes. The test can also be used for benchmarking. diff --git a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt index baeb7d0ab..5dc9c548f 100644 --- a/libensemble/tests/standalone_tests/kill_test/log.autotest.txt +++ b/libensemble/tests/standalone_tests/kill_test/log.autotest.txt @@ -3,7 +3,9 @@ Kill tests Launching MPI jobs which write at regular intervals to an output file (see sleep_and_print.c). -This test launches a job, then kills after a few seconds, and then monitors file to see if output continues. If the first job is succesfully killed, a second is launched and the test repeated. +This test launches a job, then kills after a few seconds, and then monitors +file to see if output continues. If the first job is succesfully killed, a +second is launched and the test repeated. Instructions ------------ diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index 94d2d9ae1..3616bc8da 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -2,22 +2,36 @@ Timing analysis scripts ======================= -Note that all plotting scripts produce a file rather than opening a plot interactively. +Note that all plotting scripts produce a file rather than opening a plot +interactively. -The following scripts must be run in the directory with the **libE_stats.txt** file. They extract and plot information from that file. +The following scripts must be run in the directory with the **libE_stats.txt** +file. They extract and plot information from that file. -* **plot_libe_calcs_util_v_time.py**: Extract worker utilization v time plot (with one second sampling). Shows number of workers running user sim or gen functions over time. +* **plot_libe_calcs_util_v_time.py**: Extract worker utilization v time plot + (with one second sampling). Shows number of workers running user sim or gen + functions over time. -* **plot_libe_runs_util_v_time.py**: Extract launched job utilization v time plot (with one second sampling). Shows number of workers with active jobs, launched via the job controller, over time. +* **plot_libe_runs_util_v_time.py**: Extract launched job utilization v time + plot (with one second sampling). Shows number of workers with active jobs, + launched via the job controller, over time. -* **plot_libe_histogram.py**: Create histogram showing the number of completed/killed/failed user calculations binned by run-time. +* **plot_libe_histogram.py**: Create histogram showing the number of + completed/killed/failed user calculations binned by run-time. ======================== Results analysis scripts ======================== -* **print_npy.py**: Prints to screen from a given ``*.npy`` file containing a numpy structured array. Use ``done`` to only print the lines containing *returned* points. Example:: +* **print_npy.py**: Prints to screen from a given ``*.npy`` file containing a + numpy structured array. Use ``done`` to only print the lines containing + *returned* points. Example:: ./print_npy.py run_libe_forces_results_History_length=1000_evals=8.npy done -* **compare_npy.py**: Compares either two provided ``*.npy`` files or one provided ``*.npy`` file with an expected results file (by default located at ../expected.npy). A tolerance is given on floating point results and NANs are compared as equal. Variable fields (such as those containing a time) are ignored. These fields may need to be modified depending on user's history array. +* **compare_npy.py**: Compares either two provided ``*.npy`` files or one + provided ``*.npy`` file with an expected results file (by default located at + ../expected.npy). A tolerance is given on floating point results and NANs are + compared as equal. Variable fields (such as those containing a time) are + ignored. These fields may need to be modified depending on user's history + array. From faf58001a081d532a005af098f847ef395f9d763 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 2 Dec 2019 10:11:55 -0600 Subject: [PATCH 630/644] Again updating formatting guidelines --- docs/rst_formatting_guidelines | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/rst_formatting_guidelines b/docs/rst_formatting_guidelines index 22ecd7a3d..cc4b8c4a9 100644 --- a/docs/rst_formatting_guidelines +++ b/docs/rst_formatting_guidelines @@ -1,4 +1,3 @@ - Put variables (e.g., gen_f or H) in between double ticks ``stuff here`` (no quotes) Put short code in between double ticks ``stuff here`` (no quotes) @@ -7,7 +6,7 @@ Put referenced file names in between double ticks ``stuff here`` (no quotes) Put environment variables in between double ticks ``stuff here`` (no quotes) -Put warnings line: **WARNING:** +Use .. note:: and .. warning:: directives Put breakout code snippets into a box using two colons :: From 23d441163d8f0191b411c3ba260993ed6e6a07da Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 2 Dec 2019 12:15:37 -0600 Subject: [PATCH 631/644] First round of grammar adjustments --- docs/history_output.rst | 11 +++-- docs/job_controller/overview.rst | 65 +++++++++++++++--------------- docs/overview_usecases.rst | 32 +++++++-------- docs/platforms/platforms_index.rst | 14 +++---- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/docs/history_output.rst b/docs/history_output.rst index 64cff7b55..3c4a57e92 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -1,12 +1,11 @@ The History Array ~~~~~~~~~~~~~~~~~ libEnsemble uses a NumPy structured array :ref:`H` to -store output from ``gen_f`` and corresponding ``sim_f`` output. Similarly, -``gen_f`` and ``sim_f`` are expected to return output in NumPy structured -arrays. The names of the fields to be given as input to ``gen_f`` and ``sim_f`` -must be an output from ``gen_f`` or ``sim_f``. In addition to the fields output -from ``sim_f`` and ``gen_f``, the final history returned from libEnsemble will -include the following fields: +store corresponding output from each ``gen_f`` and ``sim_f``. Similarly, +``gen_f`` and ``sim_f`` are expected to return output as NumPy structured +arrays. The names of the input fields for ``gen_f`` and ``sim_f`` +must be output from ``gen_f`` or ``sim_f``. In addition to the user-function output fields, +the final history from libEnsemble will includes the following: * ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an associated ``sim_id``. The generator can assign this, but users must be diff --git a/docs/job_controller/overview.rst b/docs/job_controller/overview.rst index fa3c63748..c36e5bc97 100644 --- a/docs/job_controller/overview.rst +++ b/docs/job_controller/overview.rst @@ -1,38 +1,37 @@ Job Controller Overview ======================= -Many users' will wish to launch an application to the system from a :ref:`sim_f` -(or :ref:`gen_f`), running on a worker. - -An MPI job, for example, could be initialized with a subprocess call to ``mpirun``, or -an alternative launcher such as ``aprun`` or ``jsrun``. The sim_f may then monitor this job, -check output, and possibly kill the job. The word ``job`` is used here to represent -a launch of an application to the system, where the system could be a supercomputer, -cluster, or any other provision of compute resources. - -In order to remove the burden of system interaction from the user, and enable sim_f -scripts that are portable between systems, a job_controller interface is provided by -libEnsemble. The job_controller provides the key functions: ``launch()``, ``poll()`` and -``kill()``. libEnsemble auto-detects a number of system criteria, such as the MPI launcher, -along with correct mechanisms for polling and killing jobs, on supported systems. It also -contains built in resilience, such as re-launching jobs that fail due to system factors. -User scripts that employ the job_controller interface will be portable between supported -systems. Job attributes can be queried to determine status after each poll. Functions are -also provided to access and interrogate files in the job's working directory. - -The Job Controller module can be used to submit -and manage jobs using a portable interface. Various back-end mechanisms may be -used to implement this interface on the system, including a proxy launcher and -job management system, such as Balsam. Currently, these job_controllers launch -at the application level within an existing resource pool. However, submissions -to a batch schedular may be supported in the future. - -At the top-level calling script, a job_controller is created and the executable -gen or sim applications are registered to it (these are applications that will -be runnable jobs). If an alternative job_controller, such as Balsam, is to be -used, then these can be created as in the example. Once in the user-side worker -code (sim/gen func), an MPI based job_controller can be retrieved without any -need to specify the type. +Users who wish to launch jobs to a system from a :ref:`sim_f`(or :ref:`gen_f`), +running on a worker have several options. + +Typically, an MPI job could be initialized with a subprocess call to +``mpirun`` or an alternative launcher such as ``aprun`` or ``jsrun``. The ``sim_f`` +may then monitor this job, check output, and possibly kill the job. We use "job" +to represent an application launch to the system, which may be a supercomputer, +cluster, or other provision of compute resources. + +A **job_controller** interface is provided by libEnsemble to remove the burden of +system interaction from the user and ease writing portable ``sim_f`` scripts. +The job_controller provides the key functions: ``launch()``, ``poll()`` and +``kill()``. Job attributes can be queried to determine status after each poll. +To implement these functions, libEnsemble auto-detects system criteria such as +the MPI launcher and mechanisms to poll and kill jobs on supported systems. +libEnsemble's job_controller is resilient, and can re-launch jobs that fail due +to system factors. + +Functions are also provided to access and interrogate files in the job's working directory. + +Various back-end mechanisms may be used by the job_controller to best interact +with each system, including proxy launchers or job management systems like +Balsam_. Currently, these job_controllers launch at the application level within +an existing resource pool. However, submissions to a batch scheduler may be +supported in the future. + +In a calling script, a job_controller object is created and the executable +generator or simulation applications are registered to it for launch. If an +alternative job_controller, such as Balsam, is to be used, then these can be +created as in the example. Once in the user-side worker code (sim/gen func), +an MPI based job_controller can be retrieved without any need to specify the type. **Example usage (code runnable with or without a Balsam backend):** @@ -92,3 +91,5 @@ For a more realistic example see: - libensemble/tests/scaling_tests/forces/ which launches the forces.x application as an MPI job. + +.. _Balsam: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 47a552a75..f96831e58 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -5,20 +5,18 @@ Overview ~~~~~~~~ .. begin_overview_rst_tag -libEnsemble is a Python library to coordinate the concurrent evaluation of -dynamic ensembles of calculations. libEnsemble uses a manager to allocate work to -multiple workers. A libEnsemble worker is the smallest indivisible unit to -perform some calculation. The work performed by libEnsemble is governed by -three routines: +libEnsemble is a Python library for coordinating evaluation of dynamic ensembles +of calculations in parallel. libEnsemble uses a manager process to allocate work to +multiple worker processes. A libEnsemble worker is the smallest indivisible unit +that can perform calculations. libEnsemble's work is governed by three routines: * :ref:`gen_f`: Generates inputs to ``sim_f``. -* :ref:`sim_f`: Evaluates a simulation or other evaluation at output from ``gen_f``. +* :ref:`sim_f`: Evaluates a simulation or other evaluation based on output from ``gen_f``. * :ref:`alloc_f`: Decides whether ``sim_f`` or ``gen_f`` should be called (and with what input/resources) as workers become available. -Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts -be found in the ``examples/`` directory. To enable portability, a -:doc:`job_controller` -interface is supplied for users to launch and monitor jobs in their +Example ``sim_f``, ``gen_f``, ``alloc_f``, and calling scripts can be found in +the ``examples/`` directory. To enable portability, a :doc:`job_controller` +interface is supplied for users to launch and monitor external scripts in their user-provided ``sim_f`` and ``gen_f`` routines. The default ``alloc_f`` tells each available worker to call ``sim_f`` with the @@ -35,23 +33,23 @@ to support) and plan to have examples of: * A user is looking to optimize a simulation calculation. The simulation may already be using parallel resources, but not a large fraction of some computer. libEnsemble can coordinate the concurrent evaluation of the - simulation ``sim_f`` at various parameter values and ``gen_f`` would return - candidate parameter values (possibly after each ``sim_f`` output). + simulation ``sim_f`` at various parameter values based on candidate parameter + values from ``gen_f`` (possibly after each ``sim_f`` output). -* A user has a ``gen_f`` that produces different meshes to be used within a - ``sim_f``. Given the ``sim_f`` output, ``gen_f`` will refine a mesh or +* A user has a ``gen_f`` that produces meshes for a + ``sim_f``. Given the ``sim_f`` output, the ``gen_f`` can refine a mesh or produce a new mesh. libEnsemble can ensure that the calculated meshes can be used by multiple simulations without requiring movement of data. * A user is attempting to sample a simulation ``sim_f`` at some parameter values, many of which will cause the simulation to fail. libEnsemble can stop unresponsive evaluations, and recover computational resources for future - evaluations. ``gen_f`` can possibly update the sampling after discovering + evaluations. The ``gen_f`` can possibly update the sampling after discovering regions where evaluations of ``sim_f`` fail. * A user has a simulation ``sim_f`` that requires calculating multiple - expensive quantities, some of which depend on other quantities. ``sim_f`` can - observe intermediate quantities in order to stop related calculations and + expensive quantities, some of which depend on other quantities. The ``sim_f`` + can observe intermediate quantities in order to stop related calculations and preempt future calculations associated with poor parameter values. * A user has a ``sim_f`` with multiple fidelities, with the higher-fidelity diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 6abb5b76b..cd27df1c8 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -2,12 +2,12 @@ Running on HPC Systems ====================== libEnsemble has been developed, supported, and tested on systems of highly varying -scales, from laptops to thousands of compute-nodes. libEnsemble's embarrassingly -parallel scaling capabilities are best exemplified on the resources available -within high-performance machines. +scales, from laptops to thousands of compute-nodes. libEnsemble's hugely parallel +scaling capabilities are most productive with the resources available within +high-performance machines. -libEnsemble's flexible architecture lends itself best to two general modes of worker -distributions across allocated compute nodes. The first mode we refer +Across allocated compute nodes, libEnsemble's flexible architecture lends itself +best to two general modes of worker distributions. The first mode we refer to as **centralized** mode, where the libEnsemble manager and worker processes are grouped on one or more nodes, but through the libEnsemble job-controller or a job-launch command can execute calculations on the other allocated nodes: @@ -32,8 +32,8 @@ other workers directly on one or more allocated nodes: Due to this factor, libEnsemble on Theta and Summit approaches centralized mode differently. On these machines, libEnsemble is run centralized on either a -compute-node with the support of Balsam_ or on a frontend server called a MOM -(Machine-Oriented Mini-server) node. +compute-node with the support of Balsam_, a frontend MOM +(Machine-Oriented Mini-server) node on Theta, or a launch node on Summit. Read more about configuring and launching libEnsemble on some HPC systems: From 5ac1945079c6fa8f03433246a65c37086eee4246 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 2 Dec 2019 12:24:02 -0600 Subject: [PATCH 632/644] second round of grammar adjustments --- docs/data_structures/worker_array.rst | 7 ++++--- docs/history_output.rst | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index 338096d3b..aa4cc4c7c 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -12,9 +12,10 @@ worker array 'blocked' [int]: Is the worker's resources blocked by another calculation -Since workers can be an a variety of states, the worker array ``W`` is contains -information about each workers state. This can allow an allocation functions to -determine what work should be performed. +Since workers can be an a variety of states, the worker array ``W`` contains +information about each worker's state. This can allow allocation functions to +determine what work should be performed based on worker-statuses. + We take the following convention: ========================================= ======= ============ ======= diff --git a/docs/history_output.rst b/docs/history_output.rst index 3c4a57e92..50c61a6a7 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -9,7 +9,7 @@ the final history from libEnsemble will includes the following: * ``sim_id`` [int]: Each unit of work output from ``gen_f`` must have an associated ``sim_id``. The generator can assign this, but users must be - careful to ensure points are added in order. For example, ``if alloc_f`` + careful to ensure points are added in order. For example, if ``alloc_f`` allows for two ``gen_f`` instances to be running simultaneously, ``alloc_f`` should ensure that both don’t generate points with the same ``sim_id``. @@ -19,7 +19,7 @@ the final history from libEnsemble will includes the following: * ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` output given to a worker? -* ``sim_worker`` [int]: libEnsemble worker that it was given to be evaluated. +* ``sim_worker`` [int]: libEnsemble worker that output was given to for evaluation * ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` @@ -43,15 +43,15 @@ where ``sim_count`` is the number of points evaluated. Other libEnsemble files produced by default are: -* ``libE_stats.txt``: This contains a one-line summary of all user - calculations. Each calculation summary is sent by workers to the manager and - printed as the run progresses. +* ``libE_stats.txt``: This contains one-line summaries for each user + calculation. Each summary is sent by workers to the manager and + logged as the run progresses. -* ``ensemble.log``: This is the logging output from libEnsemble. The default - logging is at INFO level. To gain additional diagnostics logging level can be - set to DEBUG. If this file is not removed, multiple runs will append output. - Messages at or above level MANAGER_WARNING are also copied to stderr to alert - the user promptly. For more info, see :doc:`Logging`. +* ``ensemble.log``: This contains logging output from libEnsemble. The default + logging is INFO. To gain additional diagnostics, the logging level can be + set to DEBUG. If this file is not removed, multiple runs will append output. + Messages at or above MANAGER_WARNING are also copied to stderr to alert + the user promptly. For more info, see :doc:`Logging`. Output Analysis ^^^^^^^^^^^^^^^ From 7e958b57a84692ca6091dc35db552772c4cd0548 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 2 Dec 2019 12:46:18 -0600 Subject: [PATCH 633/644] small-ish adjustments --- docs/data_structures/worker_array.rst | 2 +- docs/history_output.rst | 2 +- docs/job_controller/overview.rst | 14 +++++++------- docs/overview_usecases.rst | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index aa4cc4c7c..84d4963ae 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -14,7 +14,7 @@ worker array Since workers can be an a variety of states, the worker array ``W`` contains information about each worker's state. This can allow allocation functions to -determine what work should be performed based on worker-statuses. +determine what work should be performed based on each worker's state. We take the following convention: diff --git a/docs/history_output.rst b/docs/history_output.rst index 50c61a6a7..9035d62c3 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -48,7 +48,7 @@ Other libEnsemble files produced by default are: logged as the run progresses. * ``ensemble.log``: This contains logging output from libEnsemble. The default - logging is INFO. To gain additional diagnostics, the logging level can be + logging level is INFO. To gain additional diagnostics, the logging level can be set to DEBUG. If this file is not removed, multiple runs will append output. Messages at or above MANAGER_WARNING are also copied to stderr to alert the user promptly. For more info, see :doc:`Logging`. diff --git a/docs/job_controller/overview.rst b/docs/job_controller/overview.rst index c36e5bc97..0254809d1 100644 --- a/docs/job_controller/overview.rst +++ b/docs/job_controller/overview.rst @@ -11,11 +11,11 @@ to represent an application launch to the system, which may be a supercomputer, cluster, or other provision of compute resources. A **job_controller** interface is provided by libEnsemble to remove the burden of -system interaction from the user and ease writing portable ``sim_f`` scripts. -The job_controller provides the key functions: ``launch()``, ``poll()`` and -``kill()``. Job attributes can be queried to determine status after each poll. -To implement these functions, libEnsemble auto-detects system criteria such as -the MPI launcher and mechanisms to poll and kill jobs on supported systems. +system interaction from the user and ease writing portable user scripts that +launch applications. The job_controller provides the key functions: ``launch()``, +``poll()`` and ``kill()``. Job attributes can be queried to determine status after +each poll. To implement these functions, libEnsemble auto-detects system criteria +such as the MPI launcher and mechanisms to poll and kill jobs on supported systems. libEnsemble's job_controller is resilient, and can re-launch jobs that fail due to system factors. @@ -29,8 +29,8 @@ supported in the future. In a calling script, a job_controller object is created and the executable generator or simulation applications are registered to it for launch. If an -alternative job_controller, such as Balsam, is to be used, then these can be -created as in the example. Once in the user-side worker code (sim/gen func), +alternative job_controller like Balsam will be used, then the applications can be +registered like in the example below. Once in the user-side worker code (sim/gen func), an MPI based job_controller can be retrieved without any need to specify the type. **Example usage (code runnable with or without a Balsam backend):** diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index f96831e58..019cacb0f 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -5,7 +5,7 @@ Overview ~~~~~~~~ .. begin_overview_rst_tag -libEnsemble is a Python library for coordinating evaluation of dynamic ensembles +libEnsemble is a Python library for coordinating the evaluation of dynamic ensembles of calculations in parallel. libEnsemble uses a manager process to allocate work to multiple worker processes. A libEnsemble worker is the smallest indivisible unit that can perform calculations. libEnsemble's work is governed by three routines: From 4be063cfd89f435c10fcb5b8a421d8d0080b772c Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 2 Dec 2019 12:54:59 -0600 Subject: [PATCH 634/644] remove redundancy for worker state --- docs/data_structures/worker_array.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index 84d4963ae..f0240494c 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -12,9 +12,9 @@ worker array 'blocked' [int]: Is the worker's resources blocked by another calculation -Since workers can be an a variety of states, the worker array ``W`` contains -information about each worker's state. This can allow allocation functions to -determine what work should be performed based on each worker's state. +The worker array ``W`` contains information about each worker's state. This is +useful information for allocation functions determining what work should be +performed next. We take the following convention: From fcf6f49f31ead8de761bffb8f16ad1bb1fb44560 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 2 Dec 2019 19:41:00 -0600 Subject: [PATCH 635/644] Update platform guides --- docs/platforms/cori.rst | 97 +++++++++++++++++++++++----------------- docs/platforms/theta.rst | 2 + 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index f666f515b..5c925d650 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -6,40 +6,41 @@ Cori_ is a Cray XC40 located at NERSC, featuring both Intel Haswell and Knights Landing compute nodes. It uses the SLURM schedular to submit jobs from login nodes to run on the compute nodes. -Configuring Python ------------------- +Cori does not allow more than one MPI application per compute node. -Begin by loading the Python 3 Anaconda_ module:: - module load python +Configuring Python and installation +----------------------------------- + +Begin by loading the Python 3 Anaconda_ module:: -Create a Conda_ virtual environment in which to install libEnsemble and all -dependencies:: + module load python/3.7-anaconda-2019.07 - conda config --add channels intel - conda create --name my_env intelpython3_core python=3 - source activate my_env +In many cases this may provide all the dependent packages you need (including +mpi4py). Then libEnsemble can be installed locally:: -Installing libEnsemble and Dependencies ---------------------------------------- + export PYTHONNOUSERSITE=0 + pip install libensemble --user -You should have an indication that the virtual environment is activated. -Install mpi4py_ and libEnsemble in this environment, making sure to reference -the Cray compiler wrappers. Your prompt should be similar to the -following block: +Alternatively, you can create your own Conda_ environment in which to install +libEnsemble and all dependencies. If using ``mpi4py``, installation will need +to be done using the `specific instructions from NERSC`_. libEnsemble can then +be pip installed into the environment:: .. code-block:: console - (my_env) user@cori07:~$ CC=cc MPICC=cc pip install mpi4py --no-binary mpi4py (my_env) user@cori07:~$ pip install libensemble +If highly parallel runs experience long start-up delays consider the NERSC +documentation on `scaling Python`_. + Job Submission -------------- Cori uses Slurm_ for job submission and management. The two commands you'll likely use the most to run jobs are ``srun`` and ``sbatch`` for running interactively and batch, respectively. libEnsemble runs on the compute nodes -only Cori using either ``multi-processing`` or ``mpi4py``. +on Cori using either ``multi-processing`` or ``mpi4py``. Interactive Runs ^^^^^^^^^^^^^^^^ @@ -50,7 +51,7 @@ You can allocate four Knights Landing nodes for thirty minutes through the follo With your nodes allocated, queue your job to start with four MPI ranks:: - srun -n 4 python calling.py + srun --ntasks 4 --nodes=1 python calling.py This line launches libEnsemble with a manager and **three** workers to one allocated compute node, with three nodes available for the workers to launch @@ -58,22 +59,15 @@ user applications with the job-controller or a job-launch command. This is an example of running in :doc:`centralized` mode and, if using the :doc:`job_controller<../job_controller/mpi_controller>`, it should -be intiated with ``central_mode=True`` - -.. note:: - When performing a :doc:`distributed` MPI libEnsemble run, - specify one more MPI process than the number of allocated nodes. - The Manager and first worker run together on a node. +be intiated with ``central_mode=True``. libEnsemble must be run in central mode +on Cori as jobs cannot share nodes. -.. note:: - You will need to re-activate your conda virtual environment and reload your - modules! Configuring this routine to occur automatically is recommended. Batch Runs ^^^^^^^^^^ Batch scripts specify run-settings using ``#SBATCH`` statements. A simple example -for a libEnsemble use-case running in :doc:`distributed` MPI +for a libEnsemble use-case running in :doc:`centralized` MPI mode on KNL nodes resembles the following: .. code-block:: bash @@ -81,7 +75,7 @@ mode on KNL nodes resembles the following: #!/bin/bash #SBATCH -J myjob - #SBATCH -N 4 + #SBATCH -N 5 #SBATCH -q debug #SBATCH -A myproject #SBATCH -o myjob.out @@ -89,26 +83,45 @@ mode on KNL nodes resembles the following: #SBATCH -t 00:15:00 #SBATCH -C knl - # These four lines construct a machinefile for the job controller and slurm - srun hostname | sort -u > node_list - head -n 1 node_list > machinefile.$SLURM_JOBID - cat node_list >> machinefile.$SLURM_JOBID - export SLURM_HOSTFILE=machinefile.$SLURM_JOBID - - srun --ntasks 5 python calling_script.py + # Run the libE (manager and 4 workers) on one node + # leaving 4 nodes for worker launched applications. + srun --ntasks 5 --nodes=1 python calling_script.py With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble on Cori becomes:: sbatch myscript.sh -Example submission scripts are also given in the examples_ directory. +If you wish to run in multi-processing (local) mode instead of using mpi4py, +and your calling script uses the :doc:`parse_args()<../utilities>` function, +then the run line in the above script would be:: + + python calling_script.py --comms local --nworkers 4 + +As a larger example, the following script would launch libEnsemble in MPI mode +with one manager and 128 workers, where each worker will have two nodes for the +user application. libEnsemble could be run on more than one node, but here the +``overcommit`` option to srun is used on one node. + +.. code-block:: bash + :linenos: + + #!/bin/bash + #SBATCH -J my_bigjob + #SBATCH -N 257 + #SBATCH -q regular + #SBATCH -A myproject + #SBATCH -o myjob.out + #SBATCH -e myjob.error + #SBATCH -t 01:00:00 + #SBATCH -C knl -Debugging Strategies --------------------- + # Run the libE (manager and 128 workers) on one node + # leaving 256 nodes for worker launched applications. + srun --overcommit --ntasks 129 --nodes=1 python calling_script.py + +Example submission scripts are also given in the examples_ directory. -View the status of your submitted jobs with ``squeue`` and cancel jobs with -``scancel [Job ID]``. Additional Information ---------------------- @@ -123,3 +136,5 @@ See the NERSC Cori docs here_ for more information about Cori. .. _here: https://docs.nersc.gov/jobs/ .. _options: https://slurm.schedmd.com/srun.html .. _examples: https://github.com/Libensemble/libensemble/tree/develop/examples/job_submission_scripts +.. _specific instructions from NERSC: https://docs.nersc.gov/programming/high-level-environments/python/mpi4py/ +.. _scaling Python: https://docs.nersc.gov/programming/high-level-environments/python/scaling-up \ No newline at end of file diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index 2d1ee4c7d..e3e1d912a 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -9,6 +9,8 @@ Theta features three tiers of nodes: login, MOM (Machine-Oriented Mini-server), and compute nodes. Users on login nodes submit batch jobs to the MOM nodes. MOM nodes execute user batch-scripts to run on the compute nodes via ``aprun``. +Theta does not allow more than one MPI application per compute node. + Configuring Python ------------------ From 47941ef90e81d25ee1041dcf242547b73c19514a Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 2 Dec 2019 19:43:25 -0600 Subject: [PATCH 636/644] Add space to docs --- docs/job_controller/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/job_controller/overview.rst b/docs/job_controller/overview.rst index 0254809d1..9b88ddd24 100644 --- a/docs/job_controller/overview.rst +++ b/docs/job_controller/overview.rst @@ -1,7 +1,7 @@ Job Controller Overview ======================= -Users who wish to launch jobs to a system from a :ref:`sim_f`(or :ref:`gen_f`), +Users who wish to launch jobs to a system from a :ref:`sim_f` (or :ref:`gen_f`), running on a worker have several options. Typically, an MPI job could be initialized with a subprocess call to From 5a72660beab30208d5c0cc820439879cbc9214af Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 2 Dec 2019 19:55:30 -0600 Subject: [PATCH 637/644] Fix typos in Cori doc --- docs/platforms/cori.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index 5c925d650..a8e4966ad 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -25,7 +25,7 @@ mpi4py). Then libEnsemble can be installed locally:: Alternatively, you can create your own Conda_ environment in which to install libEnsemble and all dependencies. If using ``mpi4py``, installation will need to be done using the `specific instructions from NERSC`_. libEnsemble can then -be pip installed into the environment:: +be pip installed into the environment. .. code-block:: console @@ -83,7 +83,7 @@ mode on KNL nodes resembles the following: #SBATCH -t 00:15:00 #SBATCH -C knl - # Run the libE (manager and 4 workers) on one node + # Run libEnsemble (manager and 4 workers) on one node # leaving 4 nodes for worker launched applications. srun --ntasks 5 --nodes=1 python calling_script.py @@ -116,7 +116,7 @@ user application. libEnsemble could be run on more than one node, but here the #SBATCH -t 01:00:00 #SBATCH -C knl - # Run the libE (manager and 128 workers) on one node + # Run libEnsemble (manager and 128 workers) on one node # leaving 256 nodes for worker launched applications. srun --overcommit --ntasks 129 --nodes=1 python calling_script.py From d4c76dcb5d165b8ad8f7ff8f38fd4f3535b339ae Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 3 Dec 2019 00:58:47 -0600 Subject: [PATCH 638/644] Adding Kaushik to the persistent aposmm docstring --- libensemble/gen_funcs/persistent_aposmm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index fea4c2f7b..094fcee62 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -3,6 +3,9 @@ Parallel Optimization Solver for finding Multiple Minima (APOSMM) method described in detail in the paper `https://doi.org/10.1007/s12532-017-0131-4 `_ + +This implementation of APOSMM was developed by Kaushik Kulkarni and Jeffrey +Larson in the summer of 2019. """ __all__ = ['initialize_APOSMM', 'decide_where_to_start_localopt', 'update_history_dist'] From 2815becf107823c98daef875ee0b560281c4542c Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 3 Dec 2019 21:50:31 -0600 Subject: [PATCH 639/644] Add cori example batch script and tidy others --- docs/platforms/example_scripts.rst | 19 ++++--- .../bebop_submit_slurm_central.sh | 24 +++++++++ .../bebop_submit_slurm_centralmode.sh | 51 ------------------- ...slurm.sh => bebop_submit_slurm_distrib.sh} | 51 ++++++++++--------- .../job_submission_scripts/cori_submit.sh | 24 +++++++++ .../theta_submit_balsam.sh | 51 +++++++++++++------ 6 files changed, 122 insertions(+), 98 deletions(-) create mode 100644 examples/job_submission_scripts/bebop_submit_slurm_central.sh delete mode 100644 examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh rename examples/job_submission_scripts/{bebop_submit_slurm.sh => bebop_submit_slurm_distrib.sh} (58%) create mode 100644 examples/job_submission_scripts/cori_submit.sh diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 4af97bc74..c492b4793 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -8,19 +8,26 @@ information about the respective systems and configuration. Bebop - Central mode -------------------- -.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh - :caption: /examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh +.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_central.sh + :caption: /examples/job_submission_scripts/bebop_submit_slurm_central.sh :language: bash Bebop - Distributed mode ------------------------ -.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm.sh - :caption: /examples/job_submission_scripts/bebop_submit_slurm.sh +.. literalinclude:: ../../examples/job_submission_scripts/bebop_submit_slurm_distrib.sh + :caption: /examples/job_submission_scripts/bebop_submit_slurm_distrib.sh :language: bash -Blues ------ +Cori - Central mode +------------------- + +.. literalinclude:: ../../examples/job_submission_scripts/cori_submit.sh + :caption: /examples/job_submission_scripts/cori_submit.sh + :language: bash + +Blues (Blue Gene Q) - Distributed mode +-------------------------------------- .. literalinclude:: ../../examples/job_submission_scripts/blues_script.pbs :caption: /examples/job_submission_scripts/blues_script.pbs diff --git a/examples/job_submission_scripts/bebop_submit_slurm_central.sh b/examples/job_submission_scripts/bebop_submit_slurm_central.sh new file mode 100644 index 000000000..adc536ab8 --- /dev/null +++ b/examples/job_submission_scripts/bebop_submit_slurm_central.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH -J libE_test_central +#SBATCH -N 5 +#SBATCH -p knlall +#SBATCH -A +#SBATCH -o tlib.%j.%N.out +#SBATCH -e tlib.%j.%N.error +#SBATCH -t 01:00:00 + +# Launch script for running in central mode with mpi4py. +# libEnsemble will run on a dedicated node (or nodes). +# The remaining nodes in the allocation will be dedicated to worker launched apps. +# Use job_controller with auto-resources=True and central_mode=True. + +# User to edit these variables +export EXE=libE_calling_script.py +export NUM_WORKERS=4 +export I_MPI_FABRICS=shm:tmi + +# Overcommit will allow ntasks up to the no. of contexts on one node (eg. 320 on Bebop) +srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE + +# To use local mode instead of mpi4py (with utils.parse_args()) +# python calling_script.py --comms local --nworkers $NUM_WORKERS diff --git a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh b/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh deleted file mode 100644 index 58321c120..000000000 --- a/examples/job_submission_scripts/bebop_submit_slurm_centralmode.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -#SBATCH -J libE_test_central -#SBATCH -N 5 -#SBATCH -p knlall -##SBATCH -A -#SBATCH -o tlib.%j.%N.out -#SBATCH -e tlib.%j.%N.error -#SBATCH -t 01:00:00 - -#Launch script for running in central mode. -#LibEnsemble will run on a dedicated node (or nodes). -#The remaining nodes in the allocation will be dedicated to the jobs launched by the workers. - -#Requirements for running: -# Must use job_controller with auto-resources=True and central_mode=True. -# Note: Requires a schedular having an environment variable giving a global nodelist in a supported format (eg. SLURM/COBALT) -# Otherwise a worker_list file will be required. - -#Currently requires even distribution - either multiple workers per node or nodes per worker - -#User to edit these variables -export EXE=libE_calling_script.py -export NUM_WORKERS=4 - -export I_MPI_FABRICS=shm:tmi - -#If using in calling script (After N mins manager kills workers and timing.dat created.) -export LIBE_WALLCLOCK=55 - -#--------------------------------------------------------------------------------------------- -#Test -echo -e "Slurm job ID: $SLURM_JOBID" - -#cd $PBS_O_WORKDIR -cd $SLURM_SUBMIT_DIR - -# A little useful information for the log file... -echo -e "Master process running on: $HOSTNAME" -echo -e "Directory is: $PWD" - -#This will work for the number of contexts that will fit on one node (eg. 320 on Bebop) - increase libE nodes for more. -cmd="srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE $LIBE_WALLCLOCK" - -echo The command is: $cmd -echo End PBS script information. -echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished - -$cmd - -# Print the date again -- when finished -echo Finished at: `date` diff --git a/examples/job_submission_scripts/bebop_submit_slurm.sh b/examples/job_submission_scripts/bebop_submit_slurm_distrib.sh similarity index 58% rename from examples/job_submission_scripts/bebop_submit_slurm.sh rename to examples/job_submission_scripts/bebop_submit_slurm_distrib.sh index c80c2ee07..5ef013ab2 100644 --- a/examples/job_submission_scripts/bebop_submit_slurm.sh +++ b/examples/job_submission_scripts/bebop_submit_slurm_distrib.sh @@ -2,29 +2,32 @@ #SBATCH -J libE_test #SBATCH -N 4 #SBATCH -p knlall -##SBATCH -A +#SBATCH -A #SBATCH -o tlib.%j.%N.out #SBATCH -e tlib.%j.%N.error #SBATCH -t 01:00:00 -#Launch script that evenly spreads workers and adds manager to the first node. -#Requires even distribution - either multiple workers per node or nodes per worker -#Now option for manager to have a dedicated node. +# Launch script that runs in distributed mode with mpi4py. +# Workers are evenly spread over nodes and manager added to the first node. +# Requires even distribution - either multiple workers per node or nodes per worker +# Option for manager to have a dedicated node. +# Use of job controller will ensure workers co-locate jobs with workers -#User to edit these variables +# User to edit these variables export EXE=libE_calling_script.py export NUM_WORKERS=4 -export MANAGER_NODE=false #true = Manager has a dedicated node (use one extra node for SBATCH -N) +export MANAGER_NODE=false # true = Manager has a dedicated node (assign one extra) +# As libE shares nodes with user applications allow fallback if contexts overrun. unset I_MPI_FABRICS export I_MPI_FABRICS_LIST=tmi,tcp export I_MPI_FALLBACK=1 -#If using in calling script (After N mins manager kills workers and timing.dat created.) +# If using in calling script (After N mins manager kills workers and exits cleanly) export LIBE_WALLCLOCK=55 -#--------------------------------------------------------------------------------------------- - +#----------------------------------------------------------------------------- +# Work out distribution if [[ $MANAGER_NODE = "true" ]]; then WORKER_NODES=$(($SLURM_NNODES-1)) else @@ -38,49 +41,47 @@ else SUB_NODE_WORKERS=false NODES_PER_WORKER=$(($WORKER_NODES/$NUM_WORKERS)) fi; +#----------------------------------------------------------------------------- -#--------------------------------------------------------------------------------------------- -#Test -echo -e "Slurm job ID: $SLURM_JOBID" - -#cd $PBS_O_WORKDIR -cd $SLURM_SUBMIT_DIR - -# A little useful information for the log file... +# A little useful information echo -e "Master process running on: $HOSTNAME" echo -e "Directory is: $PWD" -#Generate a node list with 1 node per line: +# Generate a node list with 1 node per line: srun hostname | sort -u > node_list -#Generate list of nodes for workers +# Generate list of nodes for workers if [[ $MANAGER_NODE = "true" ]]; then tail -n +2 node_list > worker_list else cp node_list worker_list fi -#Add manager node to machinefile +# Add manager node to machinefile head -n 1 node_list > machinefile.$SLURM_JOBID -#Add worker nodes to machinefile +# Add worker nodes to machinefile if [[ $SUB_NODE_WORKERS = "true" ]]; then - awk -v repeat=$WORKERS_PER_NODE '{for(i=0;i>machinefile.$SLURM_JOBID + awk -v repeat=$WORKERS_PER_NODE '{for(i=0;i>machinefile.$SLURM_JOBID else - awk -v patt="$NODES_PER_WORKER" 'NR % patt == 1' worker_list >> machinefile.$SLURM_JOBID + awk -v patt="$NODES_PER_WORKER" 'NR % patt == 1' worker_list \ + >> machinefile.$SLURM_JOBID fi; # Put in a timestamp echo Starting executation at: `date` -#To use srun +# To use srun export SLURM_HOSTFILE=machinefile.$SLURM_JOBID + +# The "arbitrary" flag should ensure SLURM_HOSTFILE is picked up # cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE" cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE $LIBE_WALLCLOCK" echo The command is: $cmd echo End PBS script information. -echo All further output is from the process being run and not the pbs script.\n\n $cmd # Print the date again -- when finished +echo All further output is from the process being run and not the script.\n\n $cmd $cmd diff --git a/examples/job_submission_scripts/cori_submit.sh b/examples/job_submission_scripts/cori_submit.sh new file mode 100644 index 000000000..3b0e7d2d0 --- /dev/null +++ b/examples/job_submission_scripts/cori_submit.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH -J libE_test_central +#SBATCH -N 5 +#SBATCH -q debug +#SBATCH -A +#SBATCH -o tlib.%j.%N.out +#SBATCH -e tlib.%j.%N.error +#SBATCH -t 01:00:00 +#SBATCH -C knl + +# Launch script for running in central mode with mpi4py. +# libEnsemble will run on a dedicated node (or nodes). +# The remaining nodes in the allocation will be dedicated to worker launched apps. +# Use job_controller with auto-resources=True and central_mode=True. + +# User to edit these variables +export EXE=libE_calling_script.py +export NUM_WORKERS=4 + +# Overcommit will allow ntasks up to the no. of contexts on one node (eg. 320 on Bebop) +srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE + +# To use local mode instead of mpi4py (with utils.parse_args()) +# python calling_script.py --comms local --nworkers $NUM_WORKERS diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index 1677b118a..41c383c64 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -6,15 +6,17 @@ ##COBALT -q default # For large jobs >=128 nodes ##COBALT -A -# Script to launch libEnsemble using Balsam within Conda. Conda environment must be set up. - -# Requires Balsam is installed and a database initialized (this can be the default database). +# Script to launch libEnsemble using Balsam. +# Assumes Conda environment is set up. +# Requires Balsam is installed and a database initialized. # To be run with central job management # - Manager and workers run on one node (or a dedicated set of nodes). # - Workers submit jobs to the rest of the nodes in the pool. -# Constaint: - As set up - only uses one node (up to 63 workers) for libE. To use more, modifiy "balsam job" line to use hyper-threading and/or more than one node. +# Constaint: - As set up - only uses one node (up to 63 workers) for libE. +# To use more, modifiy "balsam job" line to use hyper-threading +# and/or more than one node for libE. # Name of calling script export EXE=libE_calling_script.py @@ -22,21 +24,30 @@ export EXE=libE_calling_script.py # Number of workers. export NUM_WORKERS=4 -# Wallclock for libE job in minutes (supplied to Balsam - make at least several mins smaller than wallclock for this submission to ensure job is launched) +# Balsam wall-clock in minutes - make few mins smaller than batch wallclock export BALSAM_WALLCLOCK=25 -# Name of working directory where Balsam places running jobs/output (inside the database directory) +# Name of working directory within database where Balsam places running jobs/output export WORKFLOW_NAME=libe_workflow -#Tell libE manager to stop workers, and exit after this time. Script must be set up to receive as argument. +# Wall-clock in mins for libE (allow clean shutdown). +# Script must be set up to receive as argument. export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) -# libEnsemble calling script arguments -# export SCRIPT_ARGS='' # No args. All defined in calling script -# export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS # If calling script is using utils.parse_args() -export SCRIPT_ARGS="$LIBE_WALLCLOCK --comms mpi --nworkers $NUM_WORKERS" # If calling script takes wall-clock as positional arg. -# Name of Conda environment (Need to have set up: https://balsam.readthedocs.io/en/latest/userguide/getting-started.html) +# libEnsemble calling script arguments (some alternatives shown) + +# No args. All defined in calling script +# export SCRIPT_ARGS='' + +# If calling script is using utils.parse_args() +# export SCRIPT_ARGS="--comms mpi --nworkers $NUM_WORKERS + +# If calling script takes wall-clock as positional arg and uses utils.parse_args() +export SCRIPT_ARGS="$LIBE_WALLCLOCK --comms mpi --nworkers $NUM_WORKERS" + + +# Name of Conda environment export CONDA_ENV_NAME= # Name of database @@ -58,17 +69,19 @@ module unload trackdeps module unload darshan module unload xalt +# Activate Balsam database . balsamactivate $DBASE_NAME # Currently need atleast one DB connection per worker (for postgres). if [[ $NUM_WORKERS -gt 128 ]] then #Add a margin - echo -e "max_connections=$(($NUM_WORKERS+10)) #Appended by submission script" >> $BALSAM_DB_PATH/balsamdb/postgresql.conf + echo -e "max_connections=$(($NUM_WORKERS+10)) #Appended by submission script" \ + >> $BALSAM_DB_PATH/balsamdb/postgresql.conf fi wait -# Make sure no existing apps/jobs +# Make sure no existing apps/jobs registered to database balsam rm apps --all --force balsam rm jobs --all --force wait @@ -81,7 +94,12 @@ SCRIPT_BASENAME=${EXE%.*} balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME" # Running libE on one node - one manager and upto 63 workers -balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $BALSAM_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes +balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME \ + --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS \ + --wall-time-minutes $BALSAM_WALLCLOCK \ + --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) \ + --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" \ + --url-in="local:/$THIS_DIR/*" --yes # Hyper-thread libE (note this will not affect HT status of user calcs - only libE itself) # Running 255 workers and one manager on one libE node. @@ -91,7 +109,8 @@ balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $ # Running 127 workers and one manager - launch script on 129 nodes (if one node per worker) # balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS --wall-time-minutes $BALSAM_WALLCLOCK --num-nodes 2 --ranks-per-node 64 --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes -#Run job +# Run job balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1 +# Deactivate Balsam database . balsamdeactivate From dedcee4c2cf5ebdc3a96a46d186aca3082b0da17 Mon Sep 17 00:00:00 2001 From: shudson Date: Tue, 3 Dec 2019 21:55:31 -0600 Subject: [PATCH 640/644] Set release date for v0.6.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 32bf21497..f5193f036 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ Release Notes Release 0.6.0 ------------- -:Date: November 27, 2019 +:Date: December 4, 2019 API changes: From af5def375dfd9aa2693327b09d8d9c1ce9fba17b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Wed, 4 Dec 2019 11:05:10 -0600 Subject: [PATCH 641/644] Whitespace --- docs/FAQ.rst | 2 +- docs/platforms/cori.rst | 3 --- examples/job_submission_scripts/theta_submit_balsam.sh | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index d8408e690..28edff017 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -145,7 +145,7 @@ There are several ways to address this nuisance, but all involve trial and error An easy (but insecure) solution is temporarily disabling the Firewall through System Preferences -> Security & Privacy -> Firewall -> Turn Off Firewall. Alternatively, adding a Firewall "Allow incoming connections" rule can be attempted for the offending -Job Controller executable. We've had limited success running +Job Controller executable. We've had limited success running ``sudo codesign --force --deep --sign - /path/to/application.app`` on our Job Controller executables, then confirming the next alerts for the executable and ``mpiexec.hydra``. diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index a8e4966ad..2e85a5ecb 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -8,7 +8,6 @@ jobs from login nodes to run on the compute nodes. Cori does not allow more than one MPI application per compute node. - Configuring Python and installation ----------------------------------- @@ -62,7 +61,6 @@ if using the :doc:`job_controller<../job_controller/mpi_controller>`, it should be intiated with ``central_mode=True``. libEnsemble must be run in central mode on Cori as jobs cannot share nodes. - Batch Runs ^^^^^^^^^^ @@ -122,7 +120,6 @@ user application. libEnsemble could be run on more than one node, but here the Example submission scripts are also given in the examples_ directory. - Additional Information ---------------------- diff --git a/examples/job_submission_scripts/theta_submit_balsam.sh b/examples/job_submission_scripts/theta_submit_balsam.sh index 41c383c64..6d66ec7bf 100644 --- a/examples/job_submission_scripts/theta_submit_balsam.sh +++ b/examples/job_submission_scripts/theta_submit_balsam.sh @@ -34,7 +34,6 @@ export WORKFLOW_NAME=libe_workflow # Script must be set up to receive as argument. export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) - # libEnsemble calling script arguments (some alternatives shown) # No args. All defined in calling script @@ -46,7 +45,6 @@ export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3)) # If calling script takes wall-clock as positional arg and uses utils.parse_args() export SCRIPT_ARGS="$LIBE_WALLCLOCK --comms mpi --nworkers $NUM_WORKERS" - # Name of Conda environment export CONDA_ENV_NAME= From 44a05434b44caa1b4ff47577fd35ec6defb2068c Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Dec 2019 12:15:11 -0600 Subject: [PATCH 642/644] fix tutorial formatting --- docs/tutorials/local_sine_tutorial.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index a812a20bd..30a8beed6 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -8,14 +8,10 @@ Multiprocessing. The foundation of writing libEnsemble routines is accounting for four components: - 1. The generator function :ref:`gen_f`, which produces values - for simulations. - 2. The simulator function :ref:`sim_f`, which performs - simulations based on values from ``gen_f``. - 3. The allocation function :ref:`alloc_f`, which decides which - of the previous two functions should be called, when. - 4. The calling script, which defines parameters and information about these - functions and the libEnsemble task, then begins execution. + 1. The generator function :ref:`gen_f`, which produces values for simulations. + 2. The simulator function :ref:`sim_f`, which performs simulations based on values from ``gen_f``. + 3. The allocation function :ref:`alloc_f`, which decides which of the previous two functions should be called, when. + 4. The calling script, which defines parameters and information about these functions and the libEnsemble task, then begins execution. libEnsemble initializes a *manager* process and as many *worker* processes as the user requests. The manager coordinates data-transfer between workers and assigns From b4f99b275b775b61b4c407f42adf99ee056bfc16 Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 4 Dec 2019 12:30:10 -0600 Subject: [PATCH 643/644] Minor fixes in docs --- docs/known_issues.rst | 2 +- examples/job_submission_scripts/summit_submit_mproc.sh | 2 +- examples/job_submission_scripts/theta_submit_mproc.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/known_issues.rst b/docs/known_issues.rst index caf187767..af45b1505 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -9,7 +9,7 @@ may occur when using libEnsemble. controller). * Local comms mode (multiprocessing) may fail if MPI is initialized before forking processors. This is thought to be responsible for issues combining - with PETSc. + multiprocessing with PETSc on some platforms. * Remote detection of logical cores via LSB_HOSTS (e.g., Summit) returns number of physical cores as SMT info not available. * TCP mode does not support: diff --git a/examples/job_submission_scripts/summit_submit_mproc.sh b/examples/job_submission_scripts/summit_submit_mproc.sh index 5a83ff106..18ada1640 100644 --- a/examples/job_submission_scripts/summit_submit_mproc.sh +++ b/examples/job_submission_scripts/summit_submit_mproc.sh @@ -10,7 +10,7 @@ # To be run with central job management # - Manager and workers run on launch node. -# - Workers submit jobs to the nodes in the job available. +# - Workers submit jobs to the compute nodes in the allocation. # Name of calling script- export EXE=libE_calling_script.py diff --git a/examples/job_submission_scripts/theta_submit_mproc.sh b/examples/job_submission_scripts/theta_submit_mproc.sh index 394091c72..3405179f5 100755 --- a/examples/job_submission_scripts/theta_submit_mproc.sh +++ b/examples/job_submission_scripts/theta_submit_mproc.sh @@ -11,7 +11,7 @@ # To be run with central job management # - Manager and workers run on launch node. -# - Workers submit jobs to the compute nodes in the node allocation available. +# - Workers submit jobs to the compute nodes in the allocation. # Name of calling script export EXE=libE_calling_script.py From 9a3d36a3eda9f70f975e6d6a83cba6ec52d06f4d Mon Sep 17 00:00:00 2001 From: shudson Date: Wed, 4 Dec 2019 12:38:20 -0600 Subject: [PATCH 644/644] Update user guide link in README --- README.rst | 2 +- docs/introduction_latex.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ca282940f..0a91c0363 100644 --- a/README.rst +++ b/README.rst @@ -241,5 +241,5 @@ Resources .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest +.. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index d82c1cac0..6135a6dbb 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -43,5 +43,5 @@ Quickstart Guide .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Travis CI: https://travis-ci.org/Libensemble/libensemble -.. _user guide: https://libensemble.readthedocs.io/en/latest +.. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info \ No newline at end of file