Libensemble · jmlarson1 · Mar 15, 2022 · Mar 10, 2022 · Mar 10, 2022 · Mar 10, 2022
diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst
@@ -3,7 +3,7 @@
 exit_criteria
 =============
 
-Exit criteria for libEnsemble::
+The following criteria (or termination tests) can be used to tell libEnsemble when to stop a given run::
 
     exit_criteria: [dict]:
 
@@ -13,7 +13,7 @@ Exit criteria for libEnsemble::
             Stop when this many new points have been evaluated by sim_f
         'gen_max' [int]:
             Stop when this many new points have been generated by gen_f
-        'elapsed_wallclock_time' [float]:
+        'wallclock_max' [float]:
             Stop when this time (since the manager has been initialized) has elapsed
         'stop_val' [(str, float)]:
             Stop when H[str] < float for the given (str, float pair)

diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst
@@ -264,7 +264,7 @@ other 127 nodes for launched applications. libEnsemble is run with MPI on 128 pr
     # Name of working directory where Balsam places running jobs/output
     export WORKFLOW_NAME=libe_workflow
 
-    # If user script takes ``elapsed_wallclock_time`` argument.
+    # If user script takes ``wallclock_max`` argument.
     # export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3))
     export SCRIPT_ARGS=""
 

diff --git a/libensemble/manager.py b/libensemble/manager.py
@@ -131,13 +131,13 @@ def filter_nans(array):
 
 
 _WALLCLOCK_MSG_ALL_RETURNED = """
-Termination due to elapsed_wallclock_time has occurred.
+Termination due to wallclock_max has occurred.
 All completed work has been returned.
 Posting kill messages for all workers.
 """
 
 _WALLCLOCK_MSG_ACTIVE = """
-Termination due to elapsed_wallclock_time has occurred.
+Termination due to wallclock_max has occurred.
 Some issued work has not been returned.
 Posting kill messages for all workers.
 """
@@ -174,7 +174,7 @@ def __init__(self, hist, libE_specs, alloc_specs,
         self.W = np.zeros(len(self.wcomms), dtype=Manager.worker_dtype)
         self.W['worker_id'] = np.arange(len(self.wcomms)) + 1
         self.term_tests = \
-            [(2, 'elapsed_wallclock_time', self.term_test_wallclock),
+            [(2, 'wallclock_max', self.term_test_wallclock),
              (1, 'sim_max', self.term_test_sim_max),
              (1, 'gen_max', self.term_test_gen_max),
              (1, 'stop_val', self.term_test_stop_val)]

diff --git a/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py b/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py
@@ -47,7 +47,7 @@
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
 # Tell libEnsemble when to stop
-exit_criteria = {'sim_max': 100, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 100, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info,

diff --git a/libensemble/tests/regression_tests/test_active_persistent_worker_abort.py b/libensemble/tests/regression_tests/test_active_persistent_worker_abort.py
@@ -63,7 +63,7 @@
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
 # Set sim_max small so persistent worker is quickly terminated
-exit_criteria = {'sim_max': 10, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 10, 'wallclock_max': 300}
 
 if nworkers < 2:
     sys.exit("Cannot run with a persistent worker if only one worker -- aborting...")

diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py
@@ -45,7 +45,7 @@ def six_hump_camel_err(H, persis_info, sim_specs, _):
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'elapsed_wallclock_time': 10}
+exit_criteria = {'wallclock_max': 10}
 
 libE_specs['abort_on_exception'] = False
 

diff --git a/libensemble/tests/regression_tests/test_cancel_in_alloc.py b/libensemble/tests/regression_tests/test_cancel_in_alloc.py
@@ -58,7 +58,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {"sim_max": 10, "elapsed_wallclock_time": 300}
+exit_criteria = {"sim_max": 10, "wallclock_max": 300}
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/libensemble/tests/regression_tests/test_comms.py b/libensemble/tests/regression_tests/test_comms.py
@@ -50,7 +50,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'sim_max': sim_max, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': sim_max, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs)

diff --git a/libensemble/tests/regression_tests/test_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_elapsed_time_abort.py
@@ -53,7 +53,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'elapsed_wallclock_time': 1}
+exit_criteria = {'elapsed_wallclock_time': 1}  # Intentionally using deprecated term. Use wallclock_max instead.
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/libensemble/tests/regression_tests/test_executor_hworld_timeout.py b/libensemble/tests/regression_tests/test_executor_hworld_timeout.py
@@ -81,7 +81,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'elapsed_wallclock_time': 30}
+exit_criteria = {'wallclock_max': 30}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs)

diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py
@@ -48,7 +48,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'sim_max': 2 * num_pts, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 2 * num_pts, 'wallclock_max': 300}
 
 if libE_specs['comms'] == 'tcp':
     # Can't use the same interface for manager and worker if we want

diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py
@@ -58,7 +58,7 @@
 val = gen_specs['user']['subbatch_size'] * gen_specs['user']['num_subbatches'] * gen_specs['user']['num_batches']
 exit_criteria = {
     'sim_max': val,
-    'elapsed_wallclock_time': 300,
+    'wallclock_max': 300,
 }
 
 alloc_specs = {'out': [], 'alloc_f': alloc_f}

diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py
@@ -83,7 +83,7 @@
 # Tell libEnsemble when to stop (stop_val key must be in H)
 exit_criteria = {
     'sim_max': 1000,
-    'elapsed_wallclock_time': 100,
+    'wallclock_max': 100,
     'stop_val': ('f', 3000),
 }
 # end_exit_criteria_rst_tag

diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py
@@ -66,8 +66,8 @@
 
 alloc_specs = {'alloc_f': alloc_f}
 
-# Setting a very high sim_max value and a short elapsed_wallclock_time so timeout will occur
-exit_criteria = {'sim_max': 50000, 'elapsed_wallclock_time': 5}
+# Setting a very high sim_max value and a short wallclock_max so timeout will occur
+exit_criteria = {'sim_max': 50000, 'wallclock_max': 5}
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 

diff --git a/libensemble/tests/regression_tests/test_persistent_independent.py b/libensemble/tests/regression_tests/test_persistent_independent.py
@@ -95,11 +95,11 @@
 # i==0 is full run, i==1 is early termination
 for i in range(2):
     if i == 0:
-        exit_criteria = {'elapsed_wallclock_time': 600, 'sim_max': 1000000}
+        exit_criteria = {'wallclock_max': 600, 'sim_max': 1000000}
         if is_manager:
             print('=== Testing full independent optimize ===', flush=True)
     else:
-        exit_criteria = {'elapsed_wallclock_time': 600, 'sim_max': 10}
+        exit_criteria = {'wallclock_max': 600, 'sim_max': 10}
         if is_manager:
             print('=== Testing independent optimize w/ stoppage ===', flush=True)
 

diff --git a/libensemble/tests/regression_tests/test_persistent_n_agent.py b/libensemble/tests/regression_tests/test_persistent_n_agent.py
@@ -74,7 +74,7 @@
     persis_info['gen_params'] = {}
 
     if prob_id <= 4:
-        exit_criteria = {'elapsed_wallclock_time': 600}
+        exit_criteria = {'wallclock_max': 600}
     else:
         exit_criteria = {'sim_max': 50}
 

diff --git a/libensemble/tests/regression_tests/test_persistent_pds.py b/libensemble/tests/regression_tests/test_persistent_pds.py
@@ -72,7 +72,7 @@
     persis_info['gen_params'] = {}
 
     if prob_id <= 4:
-        exit_criteria = {'elapsed_wallclock_time': 600}
+        exit_criteria = {'wallclock_max': 600}
     else:
         exit_criteria = {'sim_max': 100}
 

diff --git a/libensemble/tests/regression_tests/test_persistent_prox_slide.py b/libensemble/tests/regression_tests/test_persistent_prox_slide.py
@@ -57,9 +57,9 @@
     persis_info['gen_params'] = {}
 
     if prob_id < 3:
-        exit_criteria = {'elapsed_wallclock_time': 600}
+        exit_criteria = {'wallclock_max': 600}
     else:
-        exit_criteria = {'elapsed_wallclock_time': 600, 'sim_max': 1}
+        exit_criteria = {'wallclock_max': 600, 'sim_max': 1}
 
     libE_specs['safe_mode'] = False
 

diff --git a/libensemble/tests/regression_tests/test_persistent_sampling_CUDA_variable_resources.py b/libensemble/tests/regression_tests/test_persistent_sampling_CUDA_variable_resources.py
@@ -65,7 +65,7 @@
 }
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py
@@ -63,7 +63,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)

diff --git a/libensemble/tests/regression_tests/test_persistent_tasmanian.py b/libensemble/tests/regression_tests/test_persistent_tasmanian.py
@@ -83,7 +83,7 @@ def tasmanian_init_localp():
     if run != 1:
         # note that using 'setAnisotropicRefinement' without 'gen_max' will create an infinite loop
         # other stopping criteria could be used with 'setSurplusRefinement' or no refinement
-        exit_criteria = {'elapsed_wallclock_time': 10}
+        exit_criteria = {'wallclock_max': 10}
     elif run == 1:
         exit_criteria = {'gen_max': 100}  # This will test persistent_tasmanian stopping early.
 

diff --git a/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py b/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py
@@ -85,7 +85,7 @@ def sim_f(H, persis_info, sim_specs, _):
     }
 )
 exit_criteria_arr = []
-exit_criteria_arr.append({'elapsed_wallclock_time': 3})
+exit_criteria_arr.append({'wallclock_max': 3})
 exit_criteria_arr.append({'gen_max': 100})
 
 # Test over all possible parameter combinations.

diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py
@@ -65,7 +65,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'gen_max': 50, 'elapsed_wallclock_time': 300}
+exit_criteria = {'gen_max': 50, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)

diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py
@@ -55,7 +55,7 @@
 for i in persis_info:
     persis_info[i]['get_grad'] = True
 
-exit_criteria = {'gen_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'gen_max': 40, 'wallclock_max': 300}
 
 libE_specs['kill_canceled_sims'] = False
 # Perform the run

diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py
@@ -58,7 +58,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'gen_max': 100, 'elapsed_wallclock_time': 300}
+exit_criteria = {'gen_max': 100, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)

diff --git a/libensemble/tests/regression_tests/test_runlines_adaptive_workers.py b/libensemble/tests/regression_tests/test_runlines_adaptive_workers.py
@@ -81,7 +81,7 @@
 }  # Name of file containing a node-list
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/libensemble/tests/regression_tests/test_runlines_adaptive_workers_persistent.py b/libensemble/tests/regression_tests/test_runlines_adaptive_workers_persistent.py
@@ -85,7 +85,7 @@
 }  # Name of file containing a node-list
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/...e/tests/regression_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py b/...e/tests/regression_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py
@@ -89,7 +89,7 @@
 }  # Name of file containing a node-list
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(

diff --git a/libensemble/tests/regression_tests/test_uniform_sampling.py b/libensemble/tests/regression_tests/test_uniform_sampling.py
@@ -47,7 +47,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'gen_max': 501, 'elapsed_wallclock_time': 300}
+exit_criteria = {'gen_max': 501, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs)

diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py
@@ -84,7 +84,7 @@ def create_H0(persis_info, gen_specs, sim_max):
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 sim_max = 500
-exit_criteria = {'sim_max': sim_max, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': sim_max, 'wallclock_max': 300}
 
 aspec1 = {
     'alloc_f': gswf,

diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py
@@ -78,7 +78,7 @@
 persis_info = add_unique_random_streams(persis_info, nworkers + 1)
 persis_info_safe = deepcopy(persis_info)
 
-exit_criteria = {'sim_max': budget, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': budget, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)

diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_then_persistent_localopt_runs.py b/libensemble/tests/regression_tests/test_uniform_sampling_then_persistent_localopt_runs.py
@@ -60,7 +60,7 @@
 
 persis_info = add_unique_random_streams({}, nworkers + 1)
 
-exit_criteria = {'sim_max': 1000, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 1000, 'wallclock_max': 300}
 
 # Perform the run
 H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs)

diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_with_variable_resources.py b/libensemble/tests/regression_tests/test_uniform_sampling_with_variable_resources.py
@@ -79,7 +79,7 @@
 }
 
 
-exit_criteria = {'sim_max': 40, 'elapsed_wallclock_time': 300}
+exit_criteria = {'sim_max': 40, 'wallclock_max': 300}
 
 for prob_id in range(2):
     if prob_id == 0:

diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py
@@ -48,7 +48,7 @@
 libE_specs['save_H_and_persis_on_abort'] = False
 
 # Tell libEnsemble when to stop
-exit_criteria = {'elapsed_wallclock_time': 10}
+exit_criteria = {'wallclock_max': 10}
 
 # Perform the run
 return_flag = 1

diff --git a/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh b/libensemble/tests/scaling_tests/forces/theta_submit_balsam.sh
@@ -28,7 +28,7 @@ export LIBE_WALLCLOCK=25
 # Name of working directory where Balsam places running jobs/output (inside the database directory)
 export WORKFLOW_NAME=libe_workflow
 
-# If user script takes ``elapsed_wallclock_time`` argument.
+# If user script takes ``wallclock_max`` argument.
 # export SCRIPT_ARGS=$(($LIBE_WALLCLOCK-3))
 export SCRIPT_ARGS=""
 

diff --git a/libensemble/tests/unit_tests/setup.py b/libensemble/tests/unit_tests/setup.py
@@ -27,7 +27,7 @@ def make_criteria_and_specs_0(simx=10, n=1):
 def make_criteria_and_specs_1(simx=10):
     sim_specs = {'sim_f': np.linalg.norm, 'in': ['x'], 'out': [('g', float)], }
     gen_specs = {'gen_f': np.random.uniform, 'in': [], 'out': [('x', float), ('priority', float)], 'user': {}}
-    exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5}
+    exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'wallclock_max': 0.5}
 
     return sim_specs, gen_specs, exit_criteria
 
@@ -38,7 +38,7 @@ def make_criteria_and_specs_1A(simx=10):
                  'in': [],
                  'out': [('x', float), ('priority', float), ('sim_id', int)],
                  'user': {}}
-    exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'elapsed_wallclock_time': 0.5}
+    exit_criteria = {'sim_max': simx, 'stop_val': ('g', -1), 'wallclock_max': 0.5}
 
     return sim_specs, gen_specs, exit_criteria
 

diff --git a/libensemble/tools/check_inputs.py b/libensemble/tools/check_inputs.py
@@ -1,8 +1,11 @@
 import os
 import numpy as np
+import logging
 from libensemble.tools.fields_keys import libE_fields, allowed_gen_spec_keys, \
     allowed_sim_spec_keys, allowed_alloc_spec_keys, allowed_libE_spec_keys
 
+logger = logging.getLogger(__name__)
+
 
 def _check_consistent_field(name, field0, field1):
     """Checks that new field (field1) is compatible with an old field (field0)."""
@@ -97,9 +100,17 @@ def check_exit_criteria(exit_criteria, sim_specs, gen_specs):
 
     assert len(exit_criteria) > 0, "Must have some exit criterion"
 
+    if 'elapsed_wallclock_time' in exit_criteria:
+        logger.warning(
+            "exit_criteria['elapsed_wallclock_time'] is depricated.'\n"
+            + "This will break in the future. Use exit_criteria['wallclock_max']"
+        )
+
+        exit_criteria['wallclock_max'] = exit_criteria.pop('elapsed_wallclock_time')
+
     # Ensure termination criteria are valid
     valid_term_fields = ['sim_max', 'gen_max',
-                         'elapsed_wallclock_time', 'stop_val']
+                         'wallclock_max', 'stop_val']
     assert all([term_field in valid_term_fields for term_field in exit_criteria]), \
         "Valid termination options: " + str(valid_term_fields)