From 392a1d296d3248372a25ce686bb52c8aa816f2f6 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 21 Mar 2022 11:54:02 -0500 Subject: [PATCH 01/33] Proposing more consistent/intutitive naming of libE_fields --- libensemble/tools/fields_keys.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 17da49313..458ec7337 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -1,21 +1,22 @@ """ -Below are the fields used within libEnsemble +Below are the fields used within libEnsemble (All times are since epoch) """ -libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that (first) generated the entry - ('gen_time', float), # Time (since epoch) entry (first) was entered into H from a gen - ('last_gen_time', float), # Time (since epoch) entry was last requested by a gen - ('given', bool), # True if entry has been given for sim eval - ('given_time', float), # Time (since epoch) that the entry was (first) given to be evaluated - ('last_given_time', float), # Time (since epoch) that the entry was last given to be evaluated - ('last_given_back_time', float), # Time (since epoch) that the entry was last given back to the gen - ('returned', bool), # True if entry has been returned from sim eval - ('returned_time', float), # Time entry was (last) returned from sim eval - ('sim_worker', int), # Worker that did (or is doing) the sim eval - ('cancel_requested', bool), # True if cancellation of this entry is requested - ('kill_sent', bool), # True if a kill signal has been sent to worker - ('given_back', bool) # True if entry has been given back to the gen after evaluation. +libE_fields = [('sim_id', int), # Unique id of entry in H that was generated + ('gen_worker', int), # Worker that (first) generated the entry + ('gen_start_time', float), # Time gen_worker was first called that produced this entry + ('gen_end_time', float), # Time entry was (first) entered into H after being requested by a gen + ('sim_worker', int), # Worker that did (or is doing) the sim eval + ('sim_given', bool), # True if entry has been given for sim eval + ('sim_returned', bool), # True if entry has been returned from sim eval + ('sim_given_time', float), # Time entry was (first) given to be evaluated + ('sim_returned_time', float), # Time entry was (last) returned from sim eval + ('gen_informed', bool) # True if gen has been informed about the results of this evaluation + ('gen_informed_time', float), # Time gen was (last) informed about the results of evaluating this point + ('cancel_requested', bool), # True if cancellation of this entry is requested + ('kill_sent', bool), # True if a kill signal has been sent to worker + ('last_gen_end_time', float), # Time entry was last requested by a gen + ('last_sim_given_time', float), # Time entry was last given to be evaluated ] # end_libE_fields_rst_tag From 8ce77b5636df1a58305398436759557dee62b345 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 21 Mar 2022 12:11:10 -0500 Subject: [PATCH 02/33] Consistent terminology --- libensemble/tools/fields_keys.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 458ec7337..5e26c6028 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -3,19 +3,19 @@ """ libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that (first) generated the entry + ('gen_worker', int), # Worker that (first) generated this entry ('gen_start_time', float), # Time gen_worker was first called that produced this entry - ('gen_end_time', float), # Time entry was (first) entered into H after being requested by a gen - ('sim_worker', int), # Worker that did (or is doing) the sim eval + ('gen_end_time', float), # Time any gen (first) requested this entry + ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry ('sim_given', bool), # True if entry has been given for sim eval - ('sim_returned', bool), # True if entry has been returned from sim eval ('sim_given_time', float), # Time entry was (first) given to be evaluated - ('sim_returned_time', float), # Time entry was (last) returned from sim eval - ('gen_informed', bool) # True if gen has been informed about the results of this evaluation - ('gen_informed_time', float), # Time gen was (last) informed about the results of evaluating this point - ('cancel_requested', bool), # True if cancellation of this entry is requested - ('kill_sent', bool), # True if a kill signal has been sent to worker - ('last_gen_end_time', float), # Time entry was last requested by a gen + ('sim_completed', bool), # True if entry's sim eval completed + ('sim_completed_time', float), # Time entry's (last) sim eval completed + ('gen_informed', bool) # True if gen_worker was informed about the sim eval of this entry + ('gen_informed_time', float), # Time gen was (last) informed about the sim eval from this entry + ('cancel_requested', bool), # True if cancellation requested for this entry + ('kill_sent', bool), # True if a kill signal has been sent to worker for this entry + ('last_gen_end_time', float), # Time gen_worker last requested this entry ('last_sim_given_time', float), # Time entry was last given to be evaluated ] # end_libE_fields_rst_tag From e91e70e79b1d76bd0b3d717b801b4f4be46f24a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Mar 2022 15:40:32 -0500 Subject: [PATCH 03/33] Further alignment/improvement in libE fields --- libensemble/tools/fields_keys.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 5e26c6028..e7d48e066 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -1,22 +1,24 @@ """ -Below are the fields used within libEnsemble (All times are since epoch) +Below are the fields used within libEnsemble +Note: + - All times are since epoch + - All times are from the manager's perspective (i.e., times may differ + slightly from those recorded by the worker in the libE_stats file.) """ libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that (first) generated this entry - ('gen_start_time', float), # Time gen_worker was first called that produced this entry - ('gen_end_time', float), # Time any gen (first) requested this entry + ('gen_worker', int), # Worker that generated this entry + ('gen_start_time', float), # Time gen_worker was initiated that produced this entry + ('gen_end_time', float), # Time gen_worker requested this entry ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry - ('sim_given', bool), # True if entry has been given for sim eval - ('sim_given_time', float), # Time entry was (first) given to be evaluated - ('sim_completed', bool), # True if entry's sim eval completed - ('sim_completed_time', float), # Time entry's (last) sim eval completed + ('sim_start', bool), # True if entry has been given for sim eval + ('sim_start_time', float), # Time entry was given to be evaluated + ('sim_end', bool), # True if entry's sim eval completed + ('sim_end_time', float), # Time entry's sim eval completed ('gen_informed', bool) # True if gen_worker was informed about the sim eval of this entry - ('gen_informed_time', float), # Time gen was (last) informed about the sim eval from this entry + ('gen_informed_time', float), # Time gen was informed about the sim eval from this entry ('cancel_requested', bool), # True if cancellation requested for this entry ('kill_sent', bool), # True if a kill signal has been sent to worker for this entry - ('last_gen_end_time', float), # Time gen_worker last requested this entry - ('last_sim_given_time', float), # Time entry was last given to be evaluated ] # end_libE_fields_rst_tag From 7857ed4e7a1336e09539716dc4e2f80faebf7e15 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Mar 2022 15:59:57 -0500 Subject: [PATCH 04/33] Whitespace --- libensemble/tools/fields_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index e7d48e066..7b64683a6 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -1,6 +1,6 @@ """ -Below are the fields used within libEnsemble -Note: +Below are the fields used within libEnsemble +Note: - All times are since epoch - All times are from the manager's perspective (i.e., times may differ slightly from those recorded by the worker in the libE_stats file.) From 2f7da3a83d01b7c342ff6aa63b37b36967095e57 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Mar 2022 16:05:38 -0500 Subject: [PATCH 05/33] Forgotten comma --- libensemble/tools/fields_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 7b64683a6..4febf108a 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -15,7 +15,7 @@ ('sim_start_time', float), # Time entry was given to be evaluated ('sim_end', bool), # True if entry's sim eval completed ('sim_end_time', float), # Time entry's sim eval completed - ('gen_informed', bool) # True if gen_worker was informed about the sim eval of this entry + ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry ('gen_informed_time', float), # Time gen was informed about the sim eval from this entry ('cancel_requested', bool), # True if cancellation requested for this entry ('kill_sent', bool), # True if a kill signal has been sent to worker for this entry From 410432cfb9a59c65c5ef5f05aa650395bd3c3907 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Tue, 22 Mar 2022 20:17:41 -0500 Subject: [PATCH 06/33] Further re-wording --- libensemble/tools/fields_keys.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 4febf108a..82974923e 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -6,19 +6,19 @@ slightly from those recorded by the worker in the libE_stats file.) """ -libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that generated this entry - ('gen_start_time', float), # Time gen_worker was initiated that produced this entry - ('gen_end_time', float), # Time gen_worker requested this entry - ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry - ('sim_start', bool), # True if entry has been given for sim eval - ('sim_start_time', float), # Time entry was given to be evaluated - ('sim_end', bool), # True if entry's sim eval completed - ('sim_end_time', float), # Time entry's sim eval completed - ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry - ('gen_informed_time', float), # Time gen was informed about the sim eval from this entry - ('cancel_requested', bool), # True if cancellation requested for this entry - ('kill_sent', bool), # True if a kill signal has been sent to worker for this entry +libE_fields = [('sim_id', int), # Unique id of a generated entry in H + ('gen_worker', int), # Worker that generated this entry + ('gen_start_time', float), # Time gen_worker was initiated that produced this entry + ('gen_end_time', float), # Time gen_worker requested this entry + ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry + ('sim_start', bool), # True if entry was given to sim_worker for sim eval + ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval + ('sim_end', bool), # True if entry's sim eval completed + ('sim_end_time', float), # Time entry's sim eval completed + ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry + ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry + ('cancel_requested', bool), # True if cancellation was requested for this entry + ('kill_sent', bool), # True if a kill signal was sent to worker for this entry ] # end_libE_fields_rst_tag From 177c41f84c62ee6ba6cea3a30a320699d84e5883 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 24 Mar 2022 13:22:12 -0500 Subject: [PATCH 07/33] Reverting to old field temporarily --- libensemble/tools/fields_keys.py | 46 ++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 82974923e..5cbe97e3c 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -1,27 +1,39 @@ """ Below are the fields used within libEnsemble -Note: - - All times are since epoch - - All times are from the manager's perspective (i.e., times may differ - slightly from those recorded by the worker in the libE_stats file.) """ -libE_fields = [('sim_id', int), # Unique id of a generated entry in H - ('gen_worker', int), # Worker that generated this entry - ('gen_start_time', float), # Time gen_worker was initiated that produced this entry - ('gen_end_time', float), # Time gen_worker requested this entry - ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry - ('sim_start', bool), # True if entry was given to sim_worker for sim eval - ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval - ('sim_end', bool), # True if entry's sim eval completed - ('sim_end_time', float), # Time entry's sim eval completed - ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry - ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry - ('cancel_requested', bool), # True if cancellation was requested for this entry - ('kill_sent', bool), # True if a kill signal was sent to worker for this entry +libE_fields = [('sim_id', int), # Unique id of entry in H that was generated + ('gen_worker', int), # Worker that (first) generated the entry + ('gen_end_time', float), # Time (since epoch) entry (first) was entered into H from a gen + ('last_gen_time', float), # Time (since epoch) entry was last requested by a gen + ('given', bool), # True if entry has been given for sim eval + ('given_time', float), # Time (since epoch) that the entry was (first) given to be evaluated + ('last_given_time', float), # Time (since epoch) that the entry was last given to be evaluated + ('last_given_back_time', float), # Time (since epoch) that the entry was last given back to the gen + ('returned', bool), # True if entry has been returned from sim eval + ('returned_time', float), # Time entry was (last) returned from sim eval + ('sim_worker', int), # Worker that did (or is doing) the sim eval + ('cancel_requested', bool), # True if cancellation of this entry is requested + ('kill_sent', bool), # True if a kill signal has been sent to worker + ('given_back', bool) # True if entry has been given back to the gen after evaluation. ] # end_libE_fields_rst_tag +# libE_fields = [('sim_id', int), # Unique id of a generated entry in H +# ('gen_worker', int), # Worker that generated this entry +# ('gen_start_time', float), # Time gen_worker was initiated that produced this entry +# ('gen_end_time', float), # Time gen_worker requested this entry +# ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry +# ('sim_start', bool), # True if entry was given to sim_worker for sim eval +# ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval +# ('sim_end', bool), # True if entry's sim eval completed +# ('sim_end_time', float), # Time entry's sim eval completed +# ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry +# ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry +# ('cancel_requested', bool), # True if cancellation was requested for this entry +# ('kill_sent', bool), # True if a kill signal was sent to worker for this entry +# ] + protected_libE_fields = ['gen_worker', 'gen_time', 'given', From bf6125179c91068c9846d82b03d0c16944c48816 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 24 Mar 2022 13:54:28 -0500 Subject: [PATCH 08/33] Reverting to old field temporarily --- libensemble/tools/fields_keys.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 5cbe97e3c..4b91d5247 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -2,9 +2,10 @@ Below are the fields used within libEnsemble """ + libE_fields = [('sim_id', int), # Unique id of entry in H that was generated ('gen_worker', int), # Worker that (first) generated the entry - ('gen_end_time', float), # Time (since epoch) entry (first) was entered into H from a gen + ('gen_time', float), # Time (since epoch) entry (first) was entered into H from a gen ('last_gen_time', float), # Time (since epoch) entry was last requested by a gen ('given', bool), # True if entry has been given for sim eval ('given_time', float), # Time (since epoch) that the entry was (first) given to be evaluated From 5bc94eb53dcffefaa70f0590c895ce793f7332a5 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 24 Mar 2022 14:36:25 -0500 Subject: [PATCH 09/33] Removing various 'last_' fields --- docs/data_structures/history_array.rst | 16 +-- .../alloc_funcs/give_sim_work_first.py | 2 +- libensemble/history.py | 16 +-- .../test_allocation_funcs_and_support.py | 16 +-- libensemble/tests/unit_tests/test_history.py | 100 +++++++++--------- libensemble/tools/fields_keys.py | 4 +- 6 files changed, 68 insertions(+), 86 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 5f824d5b1..d33b84805 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -85,24 +85,16 @@ The full list of these reserved fields is given below. * ``given`` [bool]: True if this ``gen_f`` output been given to a libEnsemble worker to be evaluated by a ``sim_f``. -* ``given_time`` [float]: Time this entry - was *first* given to a worker to be evaluated by a ``sim_f``. - -* ``last_given_time`` [float]: Time this entry - was *last* given to a worker to be evaluated by a ``sim_f``. +* ``given_time`` [float]: Time this entry was given to a worker to be evaluated by a ``sim_f``. * ``returned`` [bool]: True if this entry has been evaluated by a ``sim_f``. -* ``returned_time`` [float]: Time this entry was *last* returned from a ``sim_f``. +* ``returned_time`` [float]: Time this entry was returned from a ``sim_f``. * ``given_back`` [bool]: True if this ``gen_f`` output been given back to a ``gen_f`` worker after being returned from evaluation. -.. * ``given_back_time`` [float]: Time this entry -.. was *first* given back to a ``gen_f`` worker. - -* ``last_given_back_time`` [float]: Time this entry - was *last* given back to a ``gen_f`` worker. +* ``given_back_time`` [float]: Time this entry was given back to a ``gen_f`` worker. * ``sim_worker`` [int]: libEnsemble worker that performed the ``sim_f`` evaluation. @@ -110,8 +102,6 @@ The full list of these reserved fields is given below. * ``gen_time`` [float]: Time this entry was put into ``H`` by the manager. -* ``last_gen_time`` [float]: Time this entry was last requested by a ``gen_f``. - * ``kill_sent`` [bool]: True if a kill signal has been sent to the worker evaluating this entry. Other than ``'sim_id'`` and ``cancel_requested``, reserved fields cannot be diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 2f5a3b8f5..6d2f7f5b3 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -34,7 +34,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long rows = np.where(np.logical_and.reduce((H['given'], ~H['returned'], ~H['cancel_requested'])))[0] - inds = time.time() - H['last_given_time'][rows] > user['cancel_sims_time'] + inds = time.time() - H['given_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: H[row]['cancel_requested'] = True diff --git a/libensemble/history.py b/libensemble/history.py index dc53dcd71..3f591f849 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -77,8 +77,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): H['sim_id'][-L:] = -1 H['given_time'][-L:] = np.inf - H['last_given_time'][-L:] = np.inf - H['last_given_back_time'][-L:] = np.inf + H['given_back_time'][-L:] = np.inf self.H = H self.using_H0 = len(H0) > 0 @@ -136,12 +135,10 @@ def update_history_x_out(self, q_inds, sim_worker): Worker ID """ q_inds = np.atleast_1d(q_inds) - first_given_inds = ~self.H['given'][q_inds] t = time.time() self.H['given'][q_inds] = True - self.H['given_time'][q_inds[first_given_inds]] = t - self.H['last_given_time'][q_inds] = t + self.H['given_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker self.given_count += len(q_inds) @@ -164,7 +161,7 @@ def update_history_to_gen(self, q_inds): "Giving entries in H0 back to gen. Marking entries in H0 as 'given_back' if 'returned'.") self.given_back_warned = True - self.H['last_given_back_time'][q_inds] = t + self.H['given_back_time'][q_inds] = t self.given_back_count += len(q_inds) def update_history_x_in(self, gen_worker, D, safe_mode): @@ -215,9 +212,7 @@ def update_history_x_in(self, gen_worker, D, safe_mode): self.H[field][update_inds] = D[field] first_gen_inds = update_inds[self.H['gen_time'][update_inds] == 0] - t = time.time() - self.H['gen_time'][first_gen_inds] = t - self.H['last_gen_time'][update_inds] = t + self.H['gen_time'][first_gen_inds] = time.time() self.H['gen_worker'][first_gen_inds] = gen_worker self.index += num_new @@ -234,8 +229,7 @@ def grow_H(self, k): H_1 = np.zeros(k, dtype=self.H.dtype) H_1['sim_id'] = -1 H_1['given_time'] = np.inf - H_1['last_given_time'] = np.inf - H_1['last_given_back_time'] = np.inf + H_1['given_back_time'] = np.inf self.H = np.append(self.H, H_1) # Could be arguments here to return different truncations eg. all done, given etc... diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 03d8927be..d005dfaf3 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -18,15 +18,15 @@ dtype=[('worker_id', ' Date: Thu, 24 Mar 2022 14:40:06 -0500 Subject: [PATCH 10/33] Whitespace --- libensemble/tests/unit_tests/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/unit_tests/test_history.py b/libensemble/tests/unit_tests/test_history.py index d633d5406..29351ae6f 100644 --- a/libensemble/tests/unit_tests/test_history.py +++ b/libensemble/tests/unit_tests/test_history.py @@ -62,7 +62,7 @@ dtype=[('gen_worker', ' Date: Thu, 24 Mar 2022 14:55:02 -0500 Subject: [PATCH 11/33] putting back given_back_time --- libensemble/tests/unit_tests/test_history.py | 92 ++++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/libensemble/tests/unit_tests/test_history.py b/libensemble/tests/unit_tests/test_history.py index 29351ae6f..96f2ac396 100644 --- a/libensemble/tests/unit_tests/test_history.py +++ b/libensemble/tests/unit_tests/test_history.py @@ -13,73 +13,73 @@ # - compare selected values # - compare from npy file - stored -wrs_H0 = np.array([(False, 0., 0, 0., 1, True, 1, True, [0., 0., 0.], True, 0.1, 1.1, False, False, False), - (False, 0., 0, 0., 1, True, 2, True, [0., 0., 0.], True, 0.2, 1.2, False, False, False), - (False, 0., 0, 0., 1, True, 3, True, [0., 0., 0.], True, 0.3, 1.3, False, False, False)], +wrs_H0 = np.array([(False, 0., 0, 0., 1, True, 1, True, [0., 0., 0.], True, 0.1, 1.1, False, False, False, inf), + (False, 0., 0, 0., 1, True, 2, True, [0., 0., 0.], True, 0.2, 1.2, False, False, False, inf), + (False, 0., 0, 0., 1, True, 3, True, [0., 0., 0.], True, 0.3, 1.3, False, False, False, inf)], dtype=[('local_pt', '?'), ('priority', ' Date: Thu, 24 Mar 2022 15:42:26 -0500 Subject: [PATCH 12/33] Renaming gen_time to gen_end_time --- docs/data_structures/history_array.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 2 +- libensemble/history.py | 4 ++-- .../test_persistent_sim_uniform_sampling.py | 2 +- .../test_persistent_uniform_gen_decides_stop.py | 8 ++++---- .../regression_tests/test_persistent_uniform_sampling.py | 2 +- .../test_persistent_uniform_sampling_adv.py | 2 +- .../test_persistent_uniform_sampling_async.py | 6 +++--- .../tests/unit_tests/test_allocation_funcs_and_support.py | 2 +- libensemble/tests/unit_tests/test_history.py | 4 ++-- libensemble/tools/fields_keys.py | 4 ++-- postproc_scripts/compare_npy.py | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index d33b84805..9b18fe3ef 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -100,7 +100,7 @@ The full list of these reserved fields is given below. * ``gen_worker`` [int]: libEnsemble worker that generated this entry. -* ``gen_time`` [float]: Time this entry was put into ``H`` by the manager. +* ``gen_end_time`` [float]: Time this entry was put into ``H`` by the manager. * ``kill_sent`` [bool]: True if a kill signal has been sent to the worker evaluating this entry. diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index ae0a0642b..81d37509c 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'given_time', 'gen_worker', 'sim_worker', 'given', 'returned', 'x', 'allocated', 'sim_id', 'gen_time'] + ['y', 'given_time', 'gen_worker', 'sim_worker', 'given', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/history.py b/libensemble/history.py index 3f591f849..f7b2e2062 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -211,8 +211,8 @@ def update_history_x_in(self, gen_worker, D, safe_mode): assert field not in protected_libE_fields, "The field '" + field + "' is protected" self.H[field][update_inds] = D[field] - first_gen_inds = update_inds[self.H['gen_time'][update_inds] == 0] - self.H['gen_time'][first_gen_inds] = time.time() + first_gen_inds = update_inds[self.H['gen_end_time'][update_inds] == 0] + self.H['gen_end_time'][first_gen_inds] = time.time() self.H['gen_worker'][first_gen_inds] = gen_worker self.index += num_new diff --git a/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py index f392eb3c8..a8d6743b7 100644 --- a/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py @@ -69,7 +69,7 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_time'])) == 8 + assert len(np.unique(H['gen_end_time'])) == 8 assert not any((H['f'] == 0)) # Should overwrite the last value (in fact last (nworker-1) values) with f(1,1) = 3.23333333 assert not np.isclose(H['f'][0], 3.23333333) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py index 6d8590619..fb375b4db 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py @@ -71,11 +71,11 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - [_, counts] = np.unique(H['gen_time'], return_counts=True) + [_, counts] = np.unique(H['gen_end_time'], return_counts=True) print('Num. points in each gen iteration:', counts) - assert counts[0] == nworkers, "The first gen_time should be common among initial_batch_size number of points" - assert counts[1] == nworkers, "The second gen_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_times but there should be for the async case" + assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" + assert counts[1] == nworkers, "The second gen_end_time should be common among initial_batch_size number of points" + assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_end_times but there should be for the async case" gen_workers = np.unique(H['gen_worker']) print('Generators that issued points', gen_workers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py index 50dd41972..0b252baa6 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py @@ -62,6 +62,6 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_time'])) == 2 + assert len(np.unique(H['gen_end_time'])) == 2 save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py index 03f25f668..8794d2501 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py @@ -70,7 +70,7 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_time'])) == 1, "Everything should have been generated in one batch" + assert len(np.unique(H['gen_end_time'])) == 1, "Everything should have been generated in one batch" if prob_id == 1: assert np.all(H['x'][0:sim_max] == -1.23), "The persistent gen should have set these at shutdown" diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py index 2626f15dd..33ea7a9e9 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py @@ -64,9 +64,9 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - [_, counts] = np.unique(H['gen_time'], return_counts=True) + [_, counts] = np.unique(H['gen_end_time'], return_counts=True) print('Num. points in each gen iteration:', counts) - assert counts[0] == nworkers, "The first gen_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_times but there should be for the async case" + assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" + assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_end_times but there should be for the async case" save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index d005dfaf3..4a768028b 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -24,7 +24,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], dtype=[('given', '?'), ('given_back', '?'), ('gen_worker', ' Date: Thu, 24 Mar 2022 15:48:57 -0500 Subject: [PATCH 13/33] Renaming given to sim_start --- docs/tutorials/calib_cancel_tutorial.rst | 4 ++-- docs/tutorials/local_sine_tutorial.rst | 2 +- .../alloc_funcs/give_sim_work_first.py | 4 ++-- .../alloc_funcs/inverse_bayes_allocf.py | 2 +- .../alloc_funcs/start_fd_persistent.py | 2 +- .../alloc_funcs/start_only_persistent.py | 4 ++-- .../alloc_funcs/start_persistent_consensus.py | 2 +- .../start_persistent_local_opt_gens.py | 2 +- libensemble/history.py | 14 ++++++------- libensemble/manager.py | 2 +- .../test_evaluate_existing_sample.py | 4 ++-- .../test_evaluate_mixed_sample.py | 4 ++-- .../test_persistent_aposmm_with_grad.py | 4 ++-- .../test_uniform_sampling_cancel.py | 10 +++++----- .../test_allocation_funcs_and_support.py | 10 +++++----- libensemble/tests/unit_tests/test_history.py | 20 +++++++++---------- .../tests/unit_tests/test_libE_main.py | 2 +- .../tests/unit_tests/test_manager_main.py | 2 +- libensemble/tools/alloc_support.py | 6 +++--- libensemble/tools/check_inputs.py | 2 +- libensemble/tools/fields_keys.py | 4 ++-- 21 files changed, 53 insertions(+), 53 deletions(-) diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst index 613d2c351..7b4d910a4 100644 --- a/docs/tutorials/calib_cancel_tutorial.rst +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -186,7 +186,7 @@ prepared for irregular sending /receiving of data. .. Between routines to call the allocation function and distribute allocated work .. to each Worker, the Manager selects points from the History array that are: .. -.. 1) Marked as ``'given'`` by the allocation function +.. 1) Marked as ``'sim_start'`` by the allocation function .. 2) Marked with ``'cancel_requested'`` by the generator .. 3) *Not* been marked as ``'returned'`` by the Manager .. 4) *Not* been marked with ``'kill_sent'`` by the Manager @@ -198,7 +198,7 @@ prepared for irregular sending /receiving of data. .. function calls, any points in the Manager's History array that have ``'cancel_requested'`` .. as ``True`` are not allocated:: .. -.. task_avail = ~H['given'] & ~H['cancel_requested'] +.. task_avail = ~H['sim_start'] & ~H['cancel_requested'] .. .. This ``alloc_f`` also can prioritize allocating points that have .. higher ``'priority'`` values from the ``gen_f`` values in the local History array:: diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 81d37509c..36096af03 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'given_time', 'gen_worker', 'sim_worker', 'given', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'given_time', 'gen_worker', 'sim_worker', 'sim_start', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 6d2f7f5b3..2eb4916e8 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -33,7 +33,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long - rows = np.where(np.logical_and.reduce((H['given'], ~H['returned'], ~H['cancel_requested'])))[0] + rows = np.where(np.logical_and.reduce((H['sim_start'], ~H['returned'], ~H['cancel_requested'])))[0] inds = time.time() - H['given_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: @@ -52,7 +52,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li gen_count = support.count_gens() Work = {} - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(): if np.any(points_to_evaluate): diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index a5aeb0407..fa36e8fb2 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -45,7 +45,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe Work[wid] = support.gen_work(wid, ['like'], inds_to_send_back, persis_info.get(wid), persistent=True) - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(persistent=False): if np.any(points_to_evaluate): diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index 011afabd3..ff7e42674 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -50,7 +50,7 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE Work[wid] = support.gen_work(wid, gen_specs['persis_in'], inds_to_send, persis_info.get(wid), persistent=True) - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(persistent=False): if np.any(points_to_evaluate): # perform sim evaluations (if they exist in History). diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 6155642df..09ea98bf5 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -81,7 +81,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l returned_but_not_given[point_ids] = False # Now the give_sim_work_first part - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] avail_workers = support.avail_worker_ids(persistent=False, zero_resource_workers=False) for wid in avail_workers: @@ -192,7 +192,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info returned_but_not_given[point_ids] = False # Now the give_sim_work_first part - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] avail_workers = list( set(support.avail_worker_ids(persistent=False, zero_resource_workers=False)) | set(support.avail_worker_ids(persistent=EVAL_SIM_TAG, zero_resource_workers=False))) diff --git a/libensemble/alloc_funcs/start_persistent_consensus.py b/libensemble/alloc_funcs/start_persistent_consensus.py index 663722c31..ed867d51a 100644 --- a/libensemble/alloc_funcs/start_persistent_consensus.py +++ b/libensemble/alloc_funcs/start_persistent_consensus.py @@ -267,7 +267,7 @@ def start_consensus_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, per # skip points that are not sim work or are already done while persis_info['next_to_give'] < len(H) and \ - (H[persis_info['next_to_give']]['given'] or + (H[persis_info['next_to_give']]['sim_start'] or H[persis_info['next_to_give']]['consensus_pt'] or H[persis_info['next_to_give']]['cancel_requested']): diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index fe5f64c95..189fbfc10 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -33,7 +33,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per support = AllocSupport(W, manage_resources, persis_info, sched_opts) Work = {} gen_count = support.count_persis_gens() - points_to_evaluate = ~H['given'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] # If a persistent localopt run has just finished, use run_order to update H # and then remove other information from persis_info diff --git a/libensemble/history.py b/libensemble/history.py index f7b2e2062..b12608e48 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -63,13 +63,13 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): # for ind, val in np.ndenumerate(H0[field]): # Works if H0[field] has arbitrary dimension but is slow # H[field][ind] = val - if 'given' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'given' and 'returned'") - H['given'][:len(H0)] = 1 + if 'sim_start' not in fields: + logger.manager_warning("Marking entries in H0 as having been 'sim_start' and 'returned'") + H['sim_start'][:len(H0)] = 1 H['returned'][:len(H0)] = 1 elif 'returned' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'returned' if 'given'") - H['returned'][:len(H0)] = H0['given'] + logger.manager_warning("Marking entries in H0 as having been 'returned' if 'sim_start'") + H['returned'][:len(H0)] = H0['sim_start'] if 'sim_id' not in fields: logger.manager_warning("Assigning sim_ids to entries in H0") @@ -84,7 +84,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.index = len(H0) self.grow_count = 0 - self.given_count = np.sum(H['given']) + self.given_count = np.sum(H['sim_start']) self.returned_count = np.sum(H['returned']) self.given_back_count = np.sum(H['given_back']) self.given_back_warned = False @@ -137,7 +137,7 @@ def update_history_x_out(self, q_inds, sim_worker): q_inds = np.atleast_1d(q_inds) t = time.time() - self.H['given'][q_inds] = True + self.H['sim_start'][q_inds] = True self.H['given_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker diff --git a/libensemble/manager.py b/libensemble/manager.py index 12653491b..d1da0fa2a 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -463,7 +463,7 @@ def _handle_msg_from_worker(self, persis_info, w): def _kill_cancelled_sims(self): """Send kill signals to any sims marked as cancel_requested""" if self.kill_canceled_sims: - kill_sim = self.hist.H['given'] & self.hist.H['cancel_requested'] \ + kill_sim = self.hist.H['sim_start'] & self.hist.H['cancel_requested'] \ & ~self.hist.H['returned'] & ~self.hist.H['kill_sent'] # Note that a return is still expected when running sims are killed diff --git a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py index d2591f858..c43c131f7 100644 --- a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py @@ -35,12 +35,12 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('sim_id', int), ('given', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('sim_id', int), ('sim_start', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) H0['sim_id'] = range(n_samp) -H0['given'] = False +H0['sim_start'] = False alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 9cbe623af..8bb2d9d3d 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -35,7 +35,7 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('given', bool), ('returned', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_start', bool), ('returned', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) @@ -43,7 +43,7 @@ for i in range(500): H0['f'][i] = borehole_func(H0['x'][i]) -H0['given'][:500] = True +H0['sim_start'][:500] = True H0['returned'][:500] = True alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index c1b8d4d17..401f06828 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -95,7 +95,7 @@ ('returned', bool), ('f', float), ('given_back', bool), - ('given', bool), + ('sim_start', bool), ] H0 = np.zeros(sample_size, dtype=H0_dtype) @@ -104,7 +104,7 @@ H0['x'] = np.round(minima, 1) H0['x_on_cube'] = (H0['x'] - gen_specs['user']['lb']) / (gen_specs['user']['ub'] - gen_specs['user']['lb']) H0['sim_id'] = range(sample_size) -H0[['given', 'given_back', 'returned']] = True +H0[['sim_start', 'given_back', 'returned']] = True for i in range(sample_size): H0['f'][i] = six_hump_camel_func(H0['x'][i]) diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py index 618570f1a..88f99345a 100644 --- a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py +++ b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py @@ -41,21 +41,21 @@ def create_H0(persis_info, gen_specs, sim_max): n = len(lb) b = sim_max - H0 = np.zeros(b, dtype=[('x', float, 2), ('sim_id', int), ('given', bool), ('cancel_requested', bool)]) + H0 = np.zeros(b, dtype=[('x', float, 2), ('sim_id', int), ('sim_start', bool), ('cancel_requested', bool)]) H0['x'] = persis_info[0]['rand_stream'].uniform(lb, ub, (b, n)) H0['sim_id'] = range(b) - H0['given'] = False + H0['sim_start'] = False for i in range(b): if i % 10 == 0: H0[i]['cancel_requested'] = True # Using uniform_random_sample_cancel call - need to adjust some gen_specs though # gen_specs['out'].append(('sim_id', int)) - # gen_specs['out'].append(('given', bool)) + # gen_specs['out'].append(('sim_start', bool)) # gen_specs['user']['gen_batch_size'] = sim_max # H0, persis_info[0] = uniform_random_sample_cancel({}, persis_info[0], gen_specs, {}) # H0['sim_id'] = range(gen_specs['user']['gen_batch_size']) - # H0['given'] = False + # H0['sim_start'] = False return H0 @@ -149,7 +149,7 @@ def create_H0(persis_info, gen_specs, sim_max): if is_manager: assert flag == 0 assert np.all(H['cancel_requested'][::10]), 'Some values should be cancelled but are not' - assert np.all(~H['given'][::10]), 'Some values are given that should not have been' + assert np.all(~H['sim_start'][::10]), 'Some values are given that should not have been' tol = 0.1 for m in minima: assert np.min(np.sum((H['x'] - m) ** 2, 1)) < tol diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 4a768028b..22d587d35 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -23,7 +23,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-2.11, -1.63], 0., 2, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], - dtype=[('given', '?'), ('given_back', '?'), ('gen_worker', ' Date: Thu, 24 Mar 2022 15:49:57 -0500 Subject: [PATCH 14/33] Renaming given_time to sim_start_time --- docs/tutorials/local_sine_tutorial.rst | 2 +- libensemble/alloc_funcs/give_sim_work_first.py | 2 +- .../alloc_funcs/start_persistent_local_opt_gens.py | 2 +- libensemble/history.py | 6 +++--- .../unit_tests/test_allocation_funcs_and_support.py | 2 +- libensemble/tests/unit_tests/test_history.py | 10 +++++----- libensemble/tests/unit_tests/test_libE_main.py | 2 +- libensemble/tests/unit_tests/test_manager_main.py | 2 +- libensemble/tests/unit_tests/tofix__test_worker.py | 2 +- libensemble/tools/fields_keys.py | 4 ++-- postproc_scripts/compare_npy.py | 2 +- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 36096af03..1658ffd77 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'given_time', 'gen_worker', 'sim_worker', 'sim_start', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'sim_start_time', 'gen_worker', 'sim_worker', 'sim_start', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 2eb4916e8..3ac8d2dd7 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -34,7 +34,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long rows = np.where(np.logical_and.reduce((H['sim_start'], ~H['returned'], ~H['cancel_requested'])))[0] - inds = time.time() - H['given_time'][rows] > user['cancel_sims_time'] + inds = time.time() - H['sim_start_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: H[row]['cancel_requested'] = True diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 189fbfc10..83e86d2b1 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -51,7 +51,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): gen_inds = (H['gen_worker'] == wid) if support.all_returned(H, gen_inds): - last_time_pos = np.argmax(H['given_time'][gen_inds]) + last_time_pos = np.argmax(H['sim_start_time'][gen_inds]) last_ind = np.nonzero(gen_inds)[0][last_time_pos] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], last_ind, persis_info[wid], persistent=True) persis_info[wid]['run_order'].append(last_ind) diff --git a/libensemble/history.py b/libensemble/history.py index b12608e48..6e0ad51a7 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -76,7 +76,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): H['sim_id'][:len(H0)] = np.arange(0, len(H0)) H['sim_id'][-L:] = -1 - H['given_time'][-L:] = np.inf + H['sim_start_time'][-L:] = np.inf H['given_back_time'][-L:] = np.inf self.H = H @@ -138,7 +138,7 @@ def update_history_x_out(self, q_inds, sim_worker): t = time.time() self.H['sim_start'][q_inds] = True - self.H['given_time'][q_inds] = t + self.H['sim_start_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker self.given_count += len(q_inds) @@ -228,7 +228,7 @@ def grow_H(self, k): """ H_1 = np.zeros(k, dtype=self.H.dtype) H_1['sim_id'] = -1 - H_1['given_time'] = np.inf + H_1['sim_start_time'] = np.inf H_1['given_back_time'] = np.inf self.H = np.append(self.H, H_1) diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 22d587d35..7d2408f8c 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -26,7 +26,7 @@ dtype=[('sim_start', '?'), ('given_back', '?'), ('gen_worker', ' Date: Thu, 24 Mar 2022 15:52:31 -0500 Subject: [PATCH 15/33] Renaming returned to sim_end --- docs/tutorials/calib_cancel_tutorial.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 2 +- .../alloc_funcs/fast_alloc_and_pausing.py | 6 ++-- .../alloc_funcs/fast_alloc_to_aposmm.py | 2 +- .../alloc_funcs/give_sim_work_first.py | 2 +- .../alloc_funcs/persistent_aposmm_alloc.py | 4 +-- .../alloc_funcs/start_fd_persistent.py | 2 +- .../alloc_funcs/start_only_persistent.py | 8 ++--- .../alloc_funcs/start_persistent_consensus.py | 2 +- .../start_persistent_local_opt_gens.py | 2 +- .../gen_funcs/aposmm_localopt_support.py | 4 +-- libensemble/gen_funcs/old_aposmm.py | 14 ++++---- libensemble/gen_funcs/persistent_aposmm.py | 12 +++---- libensemble/history.py | 22 ++++++------ libensemble/manager.py | 6 ++-- .../test_nan_func_old_aposmm.py | 2 +- .../deprecated_tests/test_old_aposmm_logic.py | 12 +++---- .../test_old_aposmm_one_residual_at_a_time.py | 2 +- .../test_old_aposmm_pounders.py | 2 +- .../test_old_aposmm_pounders_splitcomm.py | 2 +- .../test_old_aposmm_pounders_subcomm.py | 2 +- .../test_old_aposmm_sim_dirs.py | 2 +- .../test_old_aposmm_with_gradients.py | 2 +- .../test_evaluate_existing_sample.py | 2 +- .../test_evaluate_mixed_sample.py | 6 ++-- .../test_persistent_aposmm_dfols.py | 2 +- .../test_persistent_aposmm_scipy.py | 2 +- .../test_persistent_aposmm_with_grad.py | 6 ++-- .../test_persistent_surmise_calib.py | 4 +-- .../test_persistent_surmise_killsims.py | 4 +-- .../tests/scaling_tests/warpx/plot_results.py | 2 +- .../test_allocation_funcs_and_support.py | 4 +-- libensemble/tests/unit_tests/test_history.py | 34 +++++++++---------- .../tests/unit_tests/test_libE_main.py | 12 +++---- .../tests/unit_tests/test_manager_main.py | 2 +- .../unit_tests/test_persistent_aposmm.py | 12 +++---- libensemble/tools/alloc_support.py | 2 +- libensemble/tools/check_inputs.py | 4 +-- libensemble/tools/fields_keys.py | 4 +-- libensemble/tools/tools.py | 2 +- postproc_scripts/print_npy.py | 4 +-- postproc_scripts/readme.rst | 2 +- 42 files changed, 113 insertions(+), 113 deletions(-) diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst index 7b4d910a4..01faa64c1 100644 --- a/docs/tutorials/calib_cancel_tutorial.rst +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -188,7 +188,7 @@ prepared for irregular sending /receiving of data. .. .. 1) Marked as ``'sim_start'`` by the allocation function .. 2) Marked with ``'cancel_requested'`` by the generator -.. 3) *Not* been marked as ``'returned'`` by the Manager +.. 3) *Not* been marked as ``'sim_end'`` by the Manager .. 4) *Not* been marked with ``'kill_sent'`` by the Manager .. .. If any points match these characteristics, the Workers that are processing these diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 1658ffd77..30be14c57 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'sim_start_time', 'gen_worker', 'sim_worker', 'sim_start', 'returned', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'sim_start_time', 'gen_worker', 'sim_worker', 'sim_start', 'sim_end', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 7fbe1018d..7b7927608 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -78,7 +78,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'local_pt' in H.dtype.names and H['local_pt'][a1][0]: persis_info['local_pt_ids'].add(pt_id) - if np.all(H['returned'][a1]): + if np.all(H['sim_end'][a1]): persis_info['complete'].add(pt_id) values = gen_specs['user']['combine_component_func'](H['f_i'][a1]) persis_info['best_complete_val'] = min(persis_info['best_complete_val'], values) @@ -124,13 +124,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if len(H): # Don't give gen instances in batch mode if points are unfinished if (alloc_specs['user'].get('batch_mode') - and not all(np.logical_or(H['returned'][last_size:], H['paused'][last_size:]))): + and not all(np.logical_or(H['sim_end'][last_size:], H['paused'][last_size:]))): break # Don't call APOSMM if there are runs going but none need advancing if len(persis_info[lw]['run_order']): runs_needing_to_advance = np.zeros(len(persis_info[lw]['run_order']), dtype=bool) for run, inds in enumerate(persis_info[lw]['run_order'].values()): - runs_needing_to_advance[run] = np.all(H['returned'][inds]) + runs_needing_to_advance[run] = np.all(H['sim_end'][inds]) if not np.any(runs_needing_to_advance): break diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index a0dd71898..01a50c32b 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -55,7 +55,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if len(persis_info[lw]['run_order']): runs_needing_to_advance = np.zeros(len(persis_info[lw]['run_order']), dtype=bool) for run, inds in enumerate(persis_info[lw]['run_order'].values()): - runs_needing_to_advance[run] = H['returned'][inds[-1]] + runs_needing_to_advance[run] = H['sim_end'][inds[-1]] if not np.any(runs_needing_to_advance): break diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 3ac8d2dd7..b19ae556e 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -33,7 +33,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long - rows = np.where(np.logical_and.reduce((H['sim_start'], ~H['returned'], ~H['cancel_requested'])))[0] + rows = np.where(np.logical_and.reduce((H['sim_start'], ~H['sim_end'], ~H['cancel_requested'])))[0] inds = time.time() - H['sim_start_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index a51bd8d02..a19755d42 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -44,11 +44,11 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # If any persistent worker's calculated values have returned, give them back. for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): - if (persis_info.get('sample_done') or sum(H['returned']) >= init_sample_size + persis_info['samples_in_H0']): + if (persis_info.get('sample_done') or sum(H['sim_end']) >= init_sample_size + persis_info['samples_in_H0']): # Don't return if the initial sample is not complete persis_info['sample_done'] = True - returned_but_not_given = np.logical_and(H['returned'], ~H['given_back']) + returned_but_not_given = np.logical_and(H['sim_end'], ~H['given_back']) if np.any(returned_but_not_given): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index ff7e42674..613705c5d 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -42,7 +42,7 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE inds_to_send = np.array([], dtype=int) for x_ind in range(gen_specs['user']['n']): for f_ind in range(gen_specs['user']['p']): - inds = np.logical_and.reduce((H_tmp['x_ind'] == x_ind, H_tmp['f_ind'] == f_ind, H_tmp['returned'])) + inds = np.logical_and.reduce((H_tmp['x_ind'] == x_ind, H_tmp['f_ind'] == f_ind, H_tmp['sim_end'])) if sum(inds) == gen_specs['user']['nf']: inds_to_send = np.append(inds_to_send, H_tmp['sim_id'][inds]) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 09ea98bf5..786a1b8ec 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -63,7 +63,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l Work = {} # Asynchronous return to generator - async_return = user.get('async_return', False) and sum(H['returned']) >= init_sample_size + async_return = user.get('async_return', False) and sum(H['sim_end']) >= init_sample_size if gen_count < persis_info.get('num_gens_started', 0): # When a persistent worker is done, trigger a shutdown (returning exit condition of 1) @@ -72,7 +72,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['returned'], ~H['given_back'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['given_back'], gen_inds)) if np.any(returned_but_not_given): if async_return or support.all_returned(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] @@ -174,7 +174,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info Work = {} # Asynchronous return to generator - async_return = user.get('async_return', False) and sum(H['returned']) >= init_sample_size + async_return = user.get('async_return', False) and sum(H['sim_end']) >= init_sample_size if gen_count < persis_info.get('num_gens_started', 0): # When a persistent gen worker is done, trigger a shutdown (returning exit condition of 1) @@ -183,7 +183,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['returned'], ~H['given_back'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['given_back'], gen_inds)) if np.any(returned_but_not_given): if async_return or support.all_returned(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] diff --git a/libensemble/alloc_funcs/start_persistent_consensus.py b/libensemble/alloc_funcs/start_persistent_consensus.py index ed867d51a..7b945d505 100644 --- a/libensemble/alloc_funcs/start_persistent_consensus.py +++ b/libensemble/alloc_funcs/start_persistent_consensus.py @@ -105,7 +105,7 @@ def start_consensus_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, per [l_H_id, r_H_id] = persis_info[wid].get('curr_H_ids') num_sims_req = r_H_id - l_H_id - num_fin_sims = np.sum(H['returned'][l_H_id:r_H_id]) + num_fin_sims = np.sum(H['sim_end'][l_H_id:r_H_id]) completed_all_sims_for_gen_i = num_fin_sims == num_sims_req diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 83e86d2b1..5b10e9cf8 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -58,7 +58,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for wid in support.avail_worker_ids(persistent=False): # Find candidates to start local opt runs if a sample has been evaluated - if np.any(np.logical_and(~H['local_pt'], H['returned'], ~H['cancel_requested'])): + if np.any(np.logical_and(~H['local_pt'], H['sim_end'], ~H['cancel_requested'])): n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs) update_history_dist(H, n, gen_specs['user'], c_flag=False) starting_inds = decide_where_to_start_localopt(H, r_k, mu, nu) diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index 71fd58097..cd750d771 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -563,10 +563,10 @@ def simulate_recv_from_manager(local_H, gen_specs): # output as if the calculations were performed externally by libEnsemble. user = gen_specs['user']['standalone'] - if np.sum(local_H['returned']) >= user['eval_max']: + if np.sum(local_H['sim_end']) >= user['eval_max']: return STOP_TAG, {}, {} - H_rows = np.where(~local_H['returned'])[0] + H_rows = np.where(~local_H['sim_end'])[0] H_fields = [i[0] for i in gen_specs['out']] Work = {'libE_info': {'H_rows': H_rows}, 'H_fields': H_fields} diff --git a/libensemble/gen_funcs/old_aposmm.py b/libensemble/gen_funcs/old_aposmm.py index fa6d9c0df..18a9eca4d 100644 --- a/libensemble/gen_funcs/old_aposmm.py +++ b/libensemble/gen_funcs/old_aposmm.py @@ -243,7 +243,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): # Find next point in any uncompleted run using persis_info['run_order'] for run in active_runs: - if not np.all(H['returned'][persis_info['run_order'][run]]): + if not np.all(H['sim_end'][persis_info['run_order'][run]]): continue # Can't advance a run if all points aren't returned. x_opt, exit_code, persis_info, sorted_run_inds, x_new = advance_local_run(H, gen_specs['user'], c_flag, run, persis_info) @@ -399,9 +399,9 @@ def update_history_dist(H, n, gen_specs, c_flag): H['f'][inds] = np.inf H['f'][np.where(inds)[0][0]] = gen_specs['user']['combine_component_func'](H['f_i'][inds]) - p = np.logical_and.reduce((H['returned'], H['obj_component'] == 0, ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_end'], H['obj_component'] == 0, ~np.isnan(H['f']))) else: - p = np.logical_and.reduce((H['returned'], ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_end'], ~np.isnan(H['f']))) for new_ind in new_inds: # Loop over new returned points and update their distances @@ -783,7 +783,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): if nu > 0: test_2_through_5 = np.logical_and.reduce(( - H['returned'] == 1, # have a returned function value + H['sim_end'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -795,7 +795,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): )) else: test_2_through_5 = np.logical_and.reduce(( - H['returned'] == 1, # have a returned function value + H['sim_end'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -915,12 +915,12 @@ def initialize_APOSMM(H, gen_specs): if c_flag: # Get the pt_id for non-nan, returned points - pt_ids = H['pt_id'][np.logical_and(H['returned'], ~np.isnan(H['f_i']))] + pt_ids = H['pt_id'][np.logical_and(H['sim_end'], ~np.isnan(H['f_i']))] _, counts = np.unique(pt_ids, return_counts=True) n_s = np.sum(counts == user_specs['components']) else: # Number of returned sampled points (excluding nans) - n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['returned']))) + n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['sim_end']))) # Rather than build up a large output, we will just make changes in the # given H, and then send back the rows corresponding to updated H entries. diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index cdc93c853..54128afe0 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -261,7 +261,7 @@ def update_local_H_after_receiving(local_H, n, n_s, user_specs, Work, calc_in, f for name in calc_in.dtype.names: local_H[name][Work['libE_info']['H_rows']] = calc_in[name] - local_H['returned'][Work['libE_info']['H_rows']] = True + local_H['sim_end'][Work['libE_info']['H_rows']] = True n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) n_r = len(Work['libE_info']['H_rows']) @@ -323,7 +323,7 @@ def update_history_dist(H, n): new_inds = np.where(~H['known_to_aposmm'])[0] - p = np.logical_and.reduce((H['returned'], ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_end'], ~np.isnan(H['f']))) for new_ind in new_inds: # Loop over new returned points and update their distances @@ -491,7 +491,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): if nu > 0: test_2_through_5 = np.logical_and.reduce(( - H['returned'] == 1, # have a returned function value + H['sim_end'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -503,7 +503,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): )) else: test_2_through_5 = np.logical_and.reduce(( - H['returned'] == 1, # have a returned function value + H['sim_end'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -605,7 +605,7 @@ def initialize_APOSMM(H, user_specs, libE_info): ('local_min', bool), ('sim_id', int), ('paused', bool), - ('returned', bool), + ('sim_end', bool), ] if 'components' in user_specs: @@ -623,7 +623,7 @@ def initialize_APOSMM(H, user_specs, libE_info): assert 'f' in H.dtype.names, "Must give 'f' values to persistent_aposmm in gen_specs['in']" assert 'sim_id' in H.dtype.names, "Must give 'sim_id' to persistent_aposmm in gen_specs['in']" - assert 'returned' in H.dtype.names, "Must give 'returned' status to persistent_aposmm in gen_specs['in']" + assert 'sim_end' in H.dtype.names, "Must give 'sim_end' status to persistent_aposmm in gen_specs['in']" over_written_fields = ['dist_to_unit_bounds', 'dist_to_better_l', 'dist_to_better_s', 'ind_of_better_l', 'ind_of_better_s'] if any([i in H.dtype.names for i in over_written_fields]): diff --git a/libensemble/history.py b/libensemble/history.py index 6e0ad51a7..cee35e70e 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -64,12 +64,12 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): # H[field][ind] = val if 'sim_start' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'sim_start' and 'returned'") + logger.manager_warning("Marking entries in H0 as having been 'sim_start' and 'sim_end'") H['sim_start'][:len(H0)] = 1 - H['returned'][:len(H0)] = 1 - elif 'returned' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'returned' if 'sim_start'") - H['returned'][:len(H0)] = H0['sim_start'] + H['sim_end'][:len(H0)] = 1 + elif 'sim_end' not in fields: + logger.manager_warning("Marking entries in H0 as having been 'sim_end' if 'sim_start'") + H['sim_end'][:len(H0)] = H0['sim_start'] if 'sim_id' not in fields: logger.manager_warning("Assigning sim_ids to entries in H0") @@ -85,7 +85,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.grow_count = 0 self.given_count = np.sum(H['sim_start']) - self.returned_count = np.sum(H['returned']) + self.returned_count = np.sum(H['sim_end']) self.given_back_count = np.sum(H['given_back']) self.given_back_warned = False @@ -118,7 +118,7 @@ def update_history_f(self, D, safe_mode): else: self.H[field][ind][:H0_size] = returned_H[field][j] # Slice View - self.H['returned'][ind] = True + self.H['sim_end'][ind] = True self.H['returned_time'][ind] = time.time() self.returned_count += 1 @@ -149,16 +149,16 @@ def update_history_to_gen(self, q_inds): t = time.time() if q_inds.size > 0: - if np.all(self.H['returned'][q_inds]): + if np.all(self.H['sim_end'][q_inds]): self.H['given_back'][q_inds] = True - elif np.any(self.H['returned'][q_inds]): # sporadic returned points need updating - for ind in q_inds[self.H['returned'][q_inds]]: + elif np.any(self.H['sim_end'][q_inds]): # sporadic returned points need updating + for ind in q_inds[self.H['sim_end'][q_inds]]: self.H['given_back'][ind] = True if self.using_H0 and not self.given_back_warned: logger.manager_warning( - "Giving entries in H0 back to gen. Marking entries in H0 as 'given_back' if 'returned'.") + "Giving entries in H0 back to gen. Marking entries in H0 as 'given_back' if 'sim_end'.") self.given_back_warned = True self.H['given_back_time'][q_inds] = t diff --git a/libensemble/manager.py b/libensemble/manager.py index d1da0fa2a..2a63b37c8 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -212,7 +212,7 @@ def term_test_stop_val(self, stop_val): """Checks against stop value criterion""" key, val = stop_val H = self.hist.H - return np.any(filter_nans(H[key][H['returned']]) <= val) + return np.any(filter_nans(H[key][H['sim_end']]) <= val) def term_test(self, logged=True): """Checks termination criteria""" @@ -464,7 +464,7 @@ def _kill_cancelled_sims(self): """Send kill signals to any sims marked as cancel_requested""" if self.kill_canceled_sims: kill_sim = self.hist.H['sim_start'] & self.hist.H['cancel_requested'] \ - & ~self.hist.H['returned'] & ~self.hist.H['kill_sent'] + & ~self.hist.H['sim_end'] & ~self.hist.H['kill_sent'] # Note that a return is still expected when running sims are killed if np.any(kill_sim): @@ -491,7 +491,7 @@ def _final_receive_and_kill(self, persis_info): for w in self.W['worker_id'][self.W['persis_state'] > 0]: logger.debug("Manager sending PERSIS_STOP to worker {}".format(w)) if 'final_fields' in self.libE_specs: - rows_to_send = self.hist.trim_H()['returned'] + rows_to_send = self.hist.trim_H()['sim_end'] fields_to_send = self.libE_specs['final_fields'] H_to_send = self.hist.trim_H()[rows_to_send][fields_to_send] self.wcomms[w-1].send(PERSIS_STOP, H_to_send) diff --git a/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py b/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py index 5273872fd..5c0e3aabc 100644 --- a/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py +++ b/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py @@ -32,7 +32,7 @@ gen_out += [('x', float, n), ('x_on_cube', float, n), ('obj_component', int)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'f_i', 'returned'], + 'in': [o[0] for o in gen_out] + ['f', 'f_i', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': -2*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py b/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py index ab05c62eb..ba23f43da 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py @@ -23,7 +23,7 @@ def test_failing_localopt_method(): hist, sim_specs_0, gen_specs_0, exit_criteria_0, alloc = setup.hist_setup1() - hist.H['returned'] = 1 + hist.H['sim_end'] = 1 gen_specs_0['user']['localopt_method'] = 'BADNAME' @@ -37,7 +37,7 @@ def test_failing_localopt_method(): def test_exception_raising(): hist, sim_specs_0, gen_specs_0, exit_criteria_0, alloc = setup.hist_setup1(n=2) - hist.H['returned'] = 1 + hist.H['sim_end'] = 1 for method in ['LN_SBPLX', 'pounders', 'scipy_COBYLA']: gen_specs_0['user']['localopt_method'] = method @@ -48,10 +48,10 @@ def test_exception_raising(): def test_decide_where_to_start_localopt(): - H = np.zeros(10, dtype=gen_out + [('f', float), ('returned', bool)]) + H = np.zeros(10, dtype=gen_out + [('f', float), ('sim_end', bool)]) H['x'] = np.random.uniform(0, 1, (10, 2)) H['f'] = np.random.uniform(0, 1, 10) - H['returned'] = 1 + H['sim_end'] = 1 b = al.decide_where_to_start_localopt(H, 9, 1) assert len(b) == 0 @@ -94,10 +94,10 @@ def test_declare_opt(): def test_localopt_error_saving(): _, sim_specs_0, gen_specs_0, _, _ = setup.hist_setup1() - H = np.zeros(4, dtype=gen_out + [('f', float), ('fvec', float, 2), ('returned', bool)]) + H = np.zeros(4, dtype=gen_out + [('f', float), ('fvec', float, 2), ('sim_end', bool)]) H['x'] = np.random.uniform(0, 1, (4, 2)) H['f'] = np.random.uniform(0, 1, 4) - H['returned'] = True + H['sim_end'] = True H['local_pt'][1:] = True gen_specs_0['user']['initial_sample_size'] = 1 gen_specs_0['user']['localopt_method'] = 'scipy_COBYLA' diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py b/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py index bc8241f56..048404572 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py @@ -44,7 +44,7 @@ UB = 2*np.ones(n) LB = (-2-np.pi/10)*np.ones(n) gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f_i', 'returned'], + 'in': [o[0] for o in gen_out] + ['f_i', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': LB, diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py index 8e6d7e320..009ba2b08 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py @@ -43,7 +43,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py index 51798ea91..5381382a8 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py @@ -48,7 +48,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py index fb1e3b312..dbf6489b9 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py @@ -52,7 +52,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'returned'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py b/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py index 9fbaadb14..32fdc50d7 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py @@ -47,7 +47,7 @@ n = 2 gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'returned'], + 'in': [o[0] for o in gen_out] + ['f', 'sim_end'], 'out': gen_out, 'user': {'lb': np.array([-5, 0]), 'ub': np.array([10, 15]), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py b/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py index bc16d0812..ef90953ce 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py @@ -40,7 +40,7 @@ gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'grad', 'returned'], + 'in': [o[0] for o in gen_out] + ['f', 'grad', 'sim_end'], 'out': gen_out, 'user': {'initial_sample_size': 100, 'sample_points': np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py index c43c131f7..626abfcdc 100644 --- a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py @@ -52,6 +52,6 @@ if is_manager: assert len(H) == len(H0) assert np.array_equal(H0['x'], H['x']) - assert np.all(H['returned']) + assert np.all(H['sim_end']) print("\nlibEnsemble correctly didn't add anything to initial sample") save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 8bb2d9d3d..45bf32608 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -35,7 +35,7 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_start', bool), ('returned', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_start', bool), ('sim_end', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) @@ -44,7 +44,7 @@ H0['f'][i] = borehole_func(H0['x'][i]) H0['sim_start'][:500] = True -H0['returned'][:500] = True +H0['sim_end'][:500] = True alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} @@ -56,6 +56,6 @@ if is_manager: assert len(H) == len(H0) assert np.array_equal(H0['x'], H['x']) - assert np.all(H['returned']) + assert np.all(H['sim_end']) print("\nlibEnsemble correctly didn't add anything to initial sample") save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py index 0513b785d..e96f7bb0d 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py @@ -94,7 +94,7 @@ if is_manager: assert persis_info[1].get('run_order'), "Run_order should have been given back" assert flag == 0 - assert np.min(H['f'][H['returned']]) <= 3000, "Didn't find a value below 3000" + assert np.min(H['f'][H['sim_end']]) <= 3000, "Didn't find a value below 3000" save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py index 0af6b17d4..fad84cb5e 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py @@ -124,4 +124,4 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert np.sum(H['returned']) >= exit_criteria['sim_max'], "Run didn't finish" + assert np.sum(H['sim_end']) >= exit_criteria['sim_max'], "Run didn't finish" diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index 401f06828..fab29d5ee 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -58,7 +58,7 @@ ('local_pt', bool), ] -gen_in = ['x', 'f', 'grad', 'local_pt', 'sim_id', 'returned', 'x_on_cube', 'local_min'] +gen_in = ['x', 'f', 'grad', 'local_pt', 'sim_id', 'sim_end', 'x_on_cube', 'local_min'] gen_specs = { 'gen_f': gen_f, @@ -92,7 +92,7 @@ ('grad', float, n), ('sim_id', int), ('x_on_cube', float, n), - ('returned', bool), + ('sim_end', bool), ('f', float), ('given_back', bool), ('sim_start', bool), @@ -104,7 +104,7 @@ H0['x'] = np.round(minima, 1) H0['x_on_cube'] = (H0['x'] - gen_specs['user']['lb']) / (gen_specs['user']['ub'] - gen_specs['user']['lb']) H0['sim_id'] = range(sample_size) -H0[['sim_start', 'given_back', 'returned']] = True +H0[['sim_start', 'given_back', 'sim_end']] = True for i in range(sample_size): H0['f'][i] = six_hump_camel_func(H0['x'][i]) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py index ca4caf170..330cb1a94 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -77,7 +77,7 @@ gen_specs = { 'gen_f': gen_f, - 'persis_in': [o[0] for o in gen_out] + ['f', 'returned', 'sim_id'], + 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_end', 'sim_id'], 'out': gen_out, 'user': { 'n_init_thetas': n_init_thetas, # Num thetas in initial batch @@ -114,6 +114,6 @@ if is_manager: print('Cancelled sims', H['sim_id'][H['cancel_requested']]) - sims_done = np.count_nonzero(H['returned']) + sims_done = np.count_nonzero(H['sim_end']) save_libE_output(H, persis_info, __file__, nworkers) assert sims_done == max_evals, 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py index e127c0d34..48ae5b25a 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -97,7 +97,7 @@ gen_specs = { 'gen_f': gen_f, - 'persis_in': [o[0] for o in gen_out] + ['f', 'returned', 'sim_id'], + 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_end', 'sim_id'], 'out': gen_out, 'user': { 'n_init_thetas': n_init_thetas, # Num thetas in initial batch @@ -131,6 +131,6 @@ if is_manager: print('Cancelled sims', H['sim_id'][H['cancel_requested']]) print('Killed sims', H['sim_id'][H['kill_sent']]) - sims_done = np.count_nonzero(H['returned']) + sims_done = np.count_nonzero(H['sim_end']) save_libE_output(H, persis_info, __file__, nworkers) assert sims_done == max_evals, 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/scaling_tests/warpx/plot_results.py b/libensemble/tests/scaling_tests/warpx/plot_results.py index 83fb9f68e..753f7abc7 100755 --- a/libensemble/tests/scaling_tests/warpx/plot_results.py +++ b/libensemble/tests/scaling_tests/warpx/plot_results.py @@ -34,7 +34,7 @@ nbatches = len(pickle_data[1]['run_order']) # Remove un-returned (i.e., not submitted) simulations -results = results[results['returned'] == 1] +results = results[results['sim_end'] == 1] # Re-organize results into a dictionary. Each key is a field in libE_output, # each value is an array with the value of this field for all WarpX simulations diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 7d2408f8c..b5ac59a04 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -24,7 +24,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], dtype=[('sim_start', '?'), ('given_back', '?'), ('gen_worker', '= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" + assert np.sum(H['sim_end']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" assert persis_info.get('run_order'), "Standalone persistent_aposmm didn't do any localopt runs" tol = 1e-3 @@ -120,7 +120,7 @@ def combined_func(x): H, persis_info, exit_code = al.aposmm(H, persis_info, gen_specs, libE_info) assert exit_code == FINISHED_PERSISTENT_GEN_TAG, "Standalone persistent_aposmm didn't exit correctly" - assert np.sum(H['returned']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" + assert np.sum(H['sim_end']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" assert persis_info.get('run_order'), "Standalone persistent_aposmm didn't do any localopt runs" diff --git a/libensemble/tools/alloc_support.py b/libensemble/tools/alloc_support.py index 7bcebbbb7..ef7bf4fdb 100644 --- a/libensemble/tools/alloc_support.py +++ b/libensemble/tools/alloc_support.py @@ -261,7 +261,7 @@ def all_returned(self, H, pt_filter=None, low_bound=None): """ H, pfilter = self._filter_points(H, pt_filter, low_bound) excluded_points = H['cancel_requested'] & ~H['sim_start'] - return np.all(H['returned'][pfilter & ~excluded_points]) + return np.all(H['sim_end'][pfilter & ~excluded_points]) def all_given_back(self, H, pt_filter=None, low_bound=None): """Returns ``True`` if all expected points have been given back to gen. diff --git a/libensemble/tools/check_inputs.py b/libensemble/tools/check_inputs.py index 1d27c533e..d7f4f5633 100644 --- a/libensemble/tools/check_inputs.py +++ b/libensemble/tools/check_inputs.py @@ -140,11 +140,11 @@ def check_H(H0, sim_specs, alloc_specs, gen_specs): format(set(fields).difference(set(Dummy_H.dtype.names))) # Prior history cannot contain unreturned points - # assert 'returned' not in fields or np.all(H0['returned']), \ + # assert 'sim_end' not in fields or np.all(H0['sim_end']), \ # "H0 contains unreturned points." # Fail if prior history contains unreturned points (or returned but not given). - assert('returned' not in fields or np.all(H0['sim_start'] == H0['returned'])), \ + assert('sim_end' not in fields or np.all(H0['sim_start'] == H0['sim_end'])), \ 'H0 contains unreturned or invalid points' # # Fail if points in prior history don't have a sim_id. diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 5b776fafd..07e95e70a 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -9,7 +9,7 @@ ('sim_start', bool), # True if entry has been given for sim eval ('sim_start_time', float), # Time (since epoch) that the entry was (first) given to be evaluated ('given_back_time', float), # Time (since epoch) that the entry was last given back to the gen - ('returned', bool), # True if entry has been returned from sim eval + ('sim_end', bool), # True if entry has been returned from sim eval ('returned_time', float), # Time entry was (last) returned from sim eval ('sim_worker', int), # Worker that did (or is doing) the sim eval ('cancel_requested', bool), # True if cancellation of this entry is requested @@ -37,7 +37,7 @@ 'gen_end_time', 'sim_start', 'sim_start_time', - 'returned', + 'sim_end', 'returned_time', 'given_back', 'sim_worker'] diff --git a/libensemble/tools/tools.py b/libensemble/tools/tools.py index ddca20946..39177aa17 100644 --- a/libensemble/tools/tools.py +++ b/libensemble/tools/tools.py @@ -103,7 +103,7 @@ def save_libE_output(H, persis_info, calling_file, nworkers, mess='Run completed script_name = os.path.splitext(os.path.basename(calling_file))[0] short_name = script_name.split("test_", 1).pop() prob_str = 'length=' + str(len(H)) \ - + '_evals=' + str(sum(H['returned'])) \ + + '_evals=' + str(sum(H['sim_end'])) \ + '_workers=' + str(nworkers) h_filename = short_name + '_history_' + prob_str diff --git a/postproc_scripts/print_npy.py b/postproc_scripts/print_npy.py index 6a82e2854..0caaca5b2 100755 --- a/postproc_scripts/print_npy.py +++ b/postproc_scripts/print_npy.py @@ -23,13 +23,13 @@ if done_only: count = 0 for line in results: - if line['returned']: + if line['sim_end']: count += 1 results_filtered = np.zeros(count, dtype=results.dtype) count = 0 for i, line in enumerate(results): - if line['returned']: + if line['sim_end']: results_filtered[count] = results[i] count += 1 diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index 022c2495c..c2f88c3d4 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -25,7 +25,7 @@ Results analysis scripts * ``print_npy.py``: Prints to screen from a given ``*.npy`` file containing a NumPy structured array. Use ``done`` to print only the lines containing - ``'returned'`` points. Example:: + ``'sim_end'`` points. Example:: ./print_npy.py run_libe_forces_results_History_length=1000_evals=8.npy done From 0616223c6134a782fc2c5d124fe9c96269062bea Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 24 Mar 2022 15:53:03 -0500 Subject: [PATCH 16/33] Renaming returned_time to sim_end_time --- libensemble/history.py | 2 +- .../tests/unit_tests/test_allocation_funcs_and_support.py | 2 +- libensemble/tests/unit_tests/test_history.py | 6 +++--- libensemble/tools/fields_keys.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libensemble/history.py b/libensemble/history.py index cee35e70e..a8c44d4bf 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -119,7 +119,7 @@ def update_history_f(self, D, safe_mode): self.H[field][ind][:H0_size] = returned_H[field][j] # Slice View self.H['sim_end'][ind] = True - self.H['returned_time'][ind] = time.time() + self.H['sim_end_time'][ind] = time.time() self.returned_count += 1 def update_history_x_out(self, q_inds, sim_worker): diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index b5ac59a04..72c8eb0c3 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -23,7 +23,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-2.11, -1.63], 0., 2, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], - dtype=[('sim_start', '?'), ('given_back', '?'), ('gen_worker', ' Date: Thu, 24 Mar 2022 15:54:26 -0500 Subject: [PATCH 17/33] Renaming given_back to gen_informed --- libensemble/alloc_funcs/persistent_aposmm_alloc.py | 2 +- libensemble/alloc_funcs/start_fd_persistent.py | 2 +- libensemble/alloc_funcs/start_only_persistent.py | 4 ++-- libensemble/history.py | 8 ++++---- .../skip_test_persistent_gp_multitask_ax.py | 2 +- .../test_persistent_aposmm_with_grad.py | 4 ++-- .../tests/scaling_tests/persistent_gp/run_example.py | 2 +- .../unit_tests/test_allocation_funcs_and_support.py | 4 ++-- libensemble/tests/unit_tests/test_history.py | 10 +++++----- libensemble/tools/alloc_support.py | 2 +- libensemble/tools/fields_keys.py | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index a19755d42..2629de42c 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -48,7 +48,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # Don't return if the initial sample is not complete persis_info['sample_done'] = True - returned_but_not_given = np.logical_and(H['sim_end'], ~H['given_back']) + returned_but_not_given = np.logical_and(H['sim_end'], ~H['gen_informed']) if np.any(returned_but_not_given): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index 613705c5d..96efee41d 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -36,7 +36,7 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE # What (x_ind, f_ind) pairs have all of the evaluation of all n_ind # values complete. - inds_not_sent_back = ~H['given_back'] + inds_not_sent_back = ~H['gen_informed'] H_tmp = H[inds_not_sent_back] inds_to_send = np.array([], dtype=int) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 786a1b8ec..35b012b08 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -72,7 +72,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['given_back'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): if async_return or support.all_returned(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] @@ -183,7 +183,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['given_back'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): if async_return or support.all_returned(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] diff --git a/libensemble/history.py b/libensemble/history.py index a8c44d4bf..13d230639 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -86,7 +86,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.given_count = np.sum(H['sim_start']) self.returned_count = np.sum(H['sim_end']) - self.given_back_count = np.sum(H['given_back']) + self.given_back_count = np.sum(H['gen_informed']) self.given_back_warned = False self.given_offset = self.given_count @@ -150,15 +150,15 @@ def update_history_to_gen(self, q_inds): if q_inds.size > 0: if np.all(self.H['sim_end'][q_inds]): - self.H['given_back'][q_inds] = True + self.H['gen_informed'][q_inds] = True elif np.any(self.H['sim_end'][q_inds]): # sporadic returned points need updating for ind in q_inds[self.H['sim_end'][q_inds]]: - self.H['given_back'][ind] = True + self.H['gen_informed'][ind] = True if self.using_H0 and not self.given_back_warned: logger.manager_warning( - "Giving entries in H0 back to gen. Marking entries in H0 as 'given_back' if 'sim_end'.") + "Giving entries in H0 back to gen. Marking entries in H0 as 'gen_informed' if 'sim_end'.") self.given_back_warned = True self.H['given_back_time'][q_inds] = t diff --git a/libensemble/tests/regression_tests/skip_test_persistent_gp_multitask_ax.py b/libensemble/tests/regression_tests/skip_test_persistent_gp_multitask_ax.py index f662ab047..0572b85d5 100644 --- a/libensemble/tests/regression_tests/skip_test_persistent_gp_multitask_ax.py +++ b/libensemble/tests/regression_tests/skip_test_persistent_gp_multitask_ax.py @@ -98,7 +98,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): alloc_specs = { 'alloc_f': only_persistent_gens, - 'out': [('given_back', bool)], + 'out': [('gen_informed', bool)], 'user': {'async_return': False}, } diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index fab29d5ee..f942d1610 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -94,7 +94,7 @@ ('x_on_cube', float, n), ('sim_end', bool), ('f', float), - ('given_back', bool), + ('gen_informed', bool), ('sim_start', bool), ] H0 = np.zeros(sample_size, dtype=H0_dtype) @@ -104,7 +104,7 @@ H0['x'] = np.round(minima, 1) H0['x_on_cube'] = (H0['x'] - gen_specs['user']['lb']) / (gen_specs['user']['ub'] - gen_specs['user']['lb']) H0['sim_id'] = range(sample_size) -H0[['sim_start', 'given_back', 'sim_end']] = True +H0[['sim_start', 'gen_informed', 'sim_end']] = True for i in range(sample_size): H0['f'][i] = six_hump_camel_func(H0['x'][i]) diff --git a/libensemble/tests/scaling_tests/persistent_gp/run_example.py b/libensemble/tests/scaling_tests/persistent_gp/run_example.py index 8647a34bb..7b919a8f6 100644 --- a/libensemble/tests/scaling_tests/persistent_gp/run_example.py +++ b/libensemble/tests/scaling_tests/persistent_gp/run_example.py @@ -80,7 +80,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): alloc_specs = { 'alloc_f': only_persistent_gens, - 'out': [('given_back', bool)], + 'out': [('gen_informed', bool)], 'user': {'async_return': False}, } diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 72c8eb0c3..0aaa28afb 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -23,7 +23,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-2.11, -1.63], 0., 2, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], - dtype=[('sim_start', '?'), ('given_back', '?'), ('gen_worker', ' Date: Thu, 24 Mar 2022 15:54:49 -0500 Subject: [PATCH 18/33] Renaming given_back_time to gen_informed_time --- libensemble/history.py | 6 +++--- libensemble/tests/unit_tests/test_history.py | 10 +++++----- libensemble/tools/fields_keys.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libensemble/history.py b/libensemble/history.py index 13d230639..40ee01d2b 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -77,7 +77,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): H['sim_id'][-L:] = -1 H['sim_start_time'][-L:] = np.inf - H['given_back_time'][-L:] = np.inf + H['gen_informed_time'][-L:] = np.inf self.H = H self.using_H0 = len(H0) > 0 @@ -161,7 +161,7 @@ def update_history_to_gen(self, q_inds): "Giving entries in H0 back to gen. Marking entries in H0 as 'gen_informed' if 'sim_end'.") self.given_back_warned = True - self.H['given_back_time'][q_inds] = t + self.H['gen_informed_time'][q_inds] = t self.given_back_count += len(q_inds) def update_history_x_in(self, gen_worker, D, safe_mode): @@ -229,7 +229,7 @@ def grow_H(self, k): H_1 = np.zeros(k, dtype=self.H.dtype) H_1['sim_id'] = -1 H_1['sim_start_time'] = np.inf - H_1['given_back_time'] = np.inf + H_1['gen_informed_time'] = np.inf self.H = np.append(self.H, H_1) # Could be arguments here to return different truncations eg. all done, given etc... diff --git a/libensemble/tests/unit_tests/test_history.py b/libensemble/tests/unit_tests/test_history.py index 79469f7bb..504d015ad 100644 --- a/libensemble/tests/unit_tests/test_history.py +++ b/libensemble/tests/unit_tests/test_history.py @@ -19,7 +19,7 @@ dtype=[('local_pt', '?'), ('priority', ' Date: Thu, 24 Mar 2022 16:06:29 -0500 Subject: [PATCH 19/33] Flake8 --- .../test_persistent_uniform_gen_decides_stop.py | 2 +- .../regression_tests/test_persistent_uniform_sampling_async.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py index fb375b4db..ad9e63290 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py @@ -75,7 +75,7 @@ print('Num. points in each gen iteration:', counts) assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" assert counts[1] == nworkers, "The second gen_end_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_end_times but there should be for the async case" + assert len(np.unique(counts)) > 1, "All gen_end_times are the same; they should be different for the async case" gen_workers = np.unique(H['gen_worker']) print('Generators that issued points', gen_workers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py index 33ea7a9e9..1c2ea2811 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py @@ -67,6 +67,6 @@ [_, counts] = np.unique(H['gen_end_time'], return_counts=True) print('Num. points in each gen iteration:', counts) assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "There is no variablitiy in the gen_end_times but there should be for the async case" + assert len(np.unique(counts)) > 1, "All gen_end_times are the same; they should be different for the async case" save_libE_output(H, persis_info, __file__, nworkers) From ec76260b9f0162de6baace0857ac7df23fa833ac Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Thu, 24 Mar 2022 16:50:34 -0500 Subject: [PATCH 20/33] A first gen_start_time version --- libensemble/history.py | 6 +++-- libensemble/manager.py | 7 ++++- libensemble/tools/fields_keys.py | 44 ++++++++++++-------------------- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/libensemble/history.py b/libensemble/history.py index 40ee01d2b..0f54d31e1 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -164,7 +164,7 @@ def update_history_to_gen(self, q_inds): self.H['gen_informed_time'][q_inds] = t self.given_back_count += len(q_inds) - def update_history_x_in(self, gen_worker, D, safe_mode): + def update_history_x_in(self, gen_worker, D, safe_mode, gen_start_time): """ Updates the history (in place) when new points have been returned from a gen @@ -179,6 +179,7 @@ def update_history_x_in(self, gen_worker, D, safe_mode): if len(D) == 0: return + t = time.time() rows_remaining = len(self.H)-self.index if 'sim_id' not in D.dtype.names: @@ -212,7 +213,8 @@ def update_history_x_in(self, gen_worker, D, safe_mode): self.H[field][update_inds] = D[field] first_gen_inds = update_inds[self.H['gen_end_time'][update_inds] == 0] - self.H['gen_end_time'][first_gen_inds] = time.time() + self.H['gen_start_time'][first_gen_inds] = gen_start_time + self.H['gen_end_time'][first_gen_inds] = t self.H['gen_worker'][first_gen_inds] = gen_worker self.index += num_new diff --git a/libensemble/manager.py b/libensemble/manager.py index 2a63b37c8..00d565268 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -30,6 +30,7 @@ import cProfile import pstats import copy +import time if tuple(np.__version__.split('.')) >= ('1', '15'): from numpy.lib.recfunctions import repack_fields @@ -150,6 +151,7 @@ class Manager: ('active', int), ('persis_state', int), ('active_recv', int), + ('gen_start_time', float), ('zero_resource_worker', bool)] def __init__(self, hist, libE_specs, alloc_specs, @@ -314,6 +316,9 @@ def _send_work_order(self, Work, w): self.wcomms[w-1].send(Work['tag'], Work) + if Work['tag'] == EVAL_GEN_TAG: + self.W[w]['gen_start_time'] = time.time() + work_rows = Work['libE_info']['H_rows'] work_name = calc_type_strings[Work['tag']] logger.debug("Manager sending {} work to worker {}. Rows {}". @@ -424,7 +429,7 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv, self.safe_mode) if calc_type == EVAL_GEN_TAG: - self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode) + self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode, self.W[w]['gen_start_time']) assert len(D_recv['calc_out']) or np.any(self.W['active']) or self.W[w-1]['persis_state'], \ "Gen must return work when is is the only thing active and not persistent." if 'libE_info' in D_recv and 'persistent' in D_recv['libE_info']: diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 03dfe8c4b..18339e938 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -3,44 +3,34 @@ """ -libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that (first) generated the entry - ('gen_end_time', float), # Time (since epoch) entry (first) was entered into H from a gen - ('sim_start', bool), # True if entry has been given for sim eval - ('sim_start_time', float), # Time (since epoch) that the entry was (first) given to be evaluated - ('gen_informed_time', float), # Time (since epoch) that the entry was last given back to the gen - ('sim_end', bool), # True if entry has been returned from sim eval - ('sim_end_time', float), # Time entry was (last) returned from sim eval - ('sim_worker', int), # Worker that did (or is doing) the sim eval - ('cancel_requested', bool), # True if cancellation of this entry is requested - ('kill_sent', bool), # True if a kill signal has been sent to worker - ('gen_informed', bool) # True if entry has been given back to the gen after evaluation. +libE_fields = [('sim_id', int), # Unique id of a generated entry in H + ('gen_worker', int), # Worker that generated this entry + ('gen_start_time', float), # Time gen_worker was initiated that produced this entry + ('gen_end_time', float), # Time gen_worker requested this entry + ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry + ('sim_start', bool), # True if entry was given to sim_worker for sim eval + ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval + ('sim_end', bool), # True if entry's sim eval completed + ('sim_end_time', float), # Time entry's sim eval completed + ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry + ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry + ('cancel_requested', bool), # True if cancellation was requested for this entry + ('kill_sent', bool), # True if a kill signal was sent to worker for this entry ] # end_libE_fields_rst_tag -# libE_fields = [('sim_id', int), # Unique id of a generated entry in H -# ('gen_worker', int), # Worker that generated this entry -# ('gen_start_time', float), # Time gen_worker was initiated that produced this entry -# ('gen_end_time', float), # Time gen_worker requested this entry -# ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry -# ('sim_start', bool), # True if entry was given to sim_worker for sim eval -# ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval -# ('sim_end', bool), # True if entry's sim eval completed -# ('sim_end_time', float), # Time entry's sim eval completed -# ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry -# ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry -# ('cancel_requested', bool), # True if cancellation was requested for this entry -# ('kill_sent', bool), # True if a kill signal was sent to worker for this entry -# ] protected_libE_fields = ['gen_worker', + 'gen_start_time', 'gen_end_time', + 'sim_worker', 'sim_start', 'sim_start_time', 'sim_end', 'sim_end_time', 'gen_informed', - 'sim_worker'] + 'gen_informed_time', + 'kill_sent'] allowed_sim_spec_keys = ['sim_f', # 'in', # From 22fff1b7dc1e3eaf316a928e6448d5c456a54034 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 28 Mar 2022 12:47:05 -0500 Subject: [PATCH 21/33] Updating rst file --- docs/data_structures/history_array.rst | 43 +++++++++++++------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 9b18fe3ef..3ebb05587 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -33,8 +33,8 @@ run with the generated ``x`` and ``theta`` values. For ease, the ``gen_f`` and ``sim_f`` are expected to return output as NumPy structured arrays. The output from the ``gen_f`` or ``sim_f`` functions can be the exact elements declared in the gen/sim_specs['out'] lists, or can be a -subset of these lists. (The manager's history array will update any fields given -back to it.) +subset of these lists. (The manager's history array will update any fields +returned to it.) The names of the input fields for ``gen_f`` and ``sim_f`` must exist in the manager's history array (i.e.,~ they must be output fields from ``gen_f`` or ``sim_f`` or @@ -57,12 +57,13 @@ The reserved boolean field ``cancel_requested`` can also be set in a user function to request that libEnsemble cancels evaluation of the point. The remaining reserved fields are protected (populated by libEnsemble), and -store information about each entry. These include boolean fields for the current -scheduling status of the point (``given`` out for evaluation, ``returned`` from -evaluation, and ``given_back`` to the generator). Timing fields give the time -(since the epoch) corresponding to each state, and when the point was generated. -Other protected fields include the worker IDs on which points were generated or -evaluated. +store information about each entry. These include boolean fields for the +current scheduling status of the point (``sim_start`` when the sim evaluation +has started out, ``sim_end`` when sim evaluation has completed, and +``gen_informed`` when the sim output has been passed back to the generator). +Timing fields give the time (since the epoch) corresponding to each state, and +when the point was generated. Other protected fields include the worker IDs on +which points were generated or evaluated. The user fields and the reserved fields together make up the final history array returned by libEnsemble. @@ -82,27 +83,27 @@ The full list of these reserved fields is given below. :*Protected fields*: -* ``given`` [bool]: True if this ``gen_f`` output been given to a libEnsemble - worker to be evaluated by a ``sim_f``. +``gen_worker`` [int]: Worker that generated this entry -* ``given_time`` [float]: Time this entry was given to a worker to be evaluated by a ``sim_f``. +``gen_start_time`` [ float]: Time gen_worker was initiated that produced this entry -* ``returned`` [bool]: True if this entry has been evaluated by a ``sim_f``. +``gen_end_time`` [ float]: Time gen_worker requested this entry -* ``returned_time`` [float]: Time this entry was returned from a ``sim_f``. +``sim_worker`` [int]: Worker that did (or is doing) the sim evaluation for this entry -* ``given_back`` [bool]: True if this ``gen_f`` output been given back to a ``gen_f`` - worker after being returned from evaluation. +``sim_start`` [bool]: True if entry was given to sim_worker for sim evaluation -* ``given_back_time`` [float]: Time this entry was given back to a ``gen_f`` worker. +``sim_start_time`` [float]: Time entry was given to sim_worker for a sim evaluation -* ``sim_worker`` [int]: libEnsemble worker that performed the ``sim_f`` evaluation. +``sim_end`` [bool]: True if entry's sim evaluation completed -* ``gen_worker`` [int]: libEnsemble worker that generated this entry. +``sim_end_time`` [float]: Time entry's sim evaluation completed -* ``gen_end_time`` [float]: Time this entry was put into ``H`` by the manager. +``gen_informed`` [bool]: True if gen_worker was informed about the sim evaluation of this entry -* ``kill_sent`` [bool]: True if a kill signal has been sent to the worker evaluating this entry. +``gen_informed_time`` [float]: Time gen_worker was informed about the sim evaluation of this entry + +``kill_sent`` [bool]: True if a kill signal was sent to worker for this entry Other than ``'sim_id'`` and ``cancel_requested``, reserved fields cannot be overwritten by user functions unless ``libE_specs['safe_mode']`` is set to ``False``. @@ -118,7 +119,7 @@ Example workflow updating history array The history array is initialized using the libEnsemble reserved field and the user-provided ``gen_specs['out']`` and ``sim_specs['out']`` entries. In the figure below, only the -reserved fields: ``sim_id``, ``given``, and ``returned`` are shown for brevity. +reserved fields: ``sim_id``, ``sim_start``, and ``sim_end`` are shown for brevity. .. figure:: ../images/history_init.png :scale: 40 From ca9077e62479c1dc01e511a08e37b3149d9d9852 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 28 Mar 2022 13:41:32 -0500 Subject: [PATCH 22/33] Aligning functions with new names --- .../alloc_funcs/fast_alloc_to_aposmm.py | 2 +- .../alloc_funcs/give_sim_work_first.py | 2 +- .../alloc_funcs/inverse_bayes_allocf.py | 2 +- libensemble/alloc_funcs/only_one_gen_alloc.py | 2 +- .../alloc_funcs/persistent_aposmm_alloc.py | 6 +- .../alloc_funcs/start_only_persistent.py | 4 +- .../start_persistent_local_opt_gens.py | 2 +- libensemble/history.py | 24 +-- libensemble/libE.py | 6 +- libensemble/manager.py | 14 +- libensemble/sim_funcs/executor_hworld.py | 22 +-- .../test_executor_hworld_pass_fail.py | 2 +- .../test_allocation_funcs_and_support.py | 140 +++++++-------- libensemble/tests/unit_tests/test_history.py | 166 +++++++++--------- .../tests/unit_tests/test_manager_main.py | 6 +- libensemble/tools/alloc_support.py | 22 +-- 16 files changed, 211 insertions(+), 211 deletions(-) diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index 01a50c32b..a55acf4b4 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -49,7 +49,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if len(H): # Don't give gen instances in batch mode if points are unfinished - if user.get('batch_mode') and not support.all_returned(pt_filter=~H['paused'], low_bound=last_size): + if user.get('batch_mode') and not support.all_sim_end(pt_filter=~H['paused'], low_bound=last_size): break # Don't call APOSMM if there are runs going but none need advancing if len(persis_info[lw]['run_order']): diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index b19ae556e..1277bcbcc 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -69,7 +69,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li break # Do not start gen instances in batch mode if workers still working - if user.get('batch_mode') and not support.all_returned(H): + if user.get('batch_mode') and not support.all_sim_end(H): break # Give gen work diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index fa36e8fb2..c98c6f0f7 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -31,7 +31,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe # if > 1 persistent generator, assign the correct work to it inds_generated_by_wid = (H['gen_worker'] == wid) - if support.all_returned(H, inds_generated_by_wid): + if support.all_sim_end(H, inds_generated_by_wid): # Has sim_f completed everything from this persistent worker? # Then give back everything in the last batch diff --git a/libensemble/alloc_funcs/only_one_gen_alloc.py b/libensemble/alloc_funcs/only_one_gen_alloc.py index ad46cabe4..079bdf16b 100644 --- a/libensemble/alloc_funcs/only_one_gen_alloc.py +++ b/libensemble/alloc_funcs/only_one_gen_alloc.py @@ -38,7 +38,7 @@ def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info, elif not support.test_any_gen() and gen_flag: - if not support.all_returned(H): + if not support.all_sim_end(H): break # Give gen work diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 2629de42c..d7aa83d71 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -31,9 +31,9 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info Work = {} if persis_info.get('first_call', True): - assert support.all_given(H), "Initial points in H have never been given." - assert support.all_returned(H), "Initial points in H have never been returned." - assert support.all_given_back(H), "Initial points in H have never been given back to gen." + assert support.all_sim_start(H), "Initial points in H have never been given." + assert support.all_sim_end(H), "Initial points in H have never been returned." + assert support.all_gen_informed(H), "Initial points in H have never been given back to gen." persis_info['samples_in_H0'] = sum(H['local_pt'] == 0) persis_info['next_to_give'] = len(H) # diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 35b012b08..8947b2908 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -74,7 +74,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l gen_inds = (H['gen_worker'] == wid) returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): - if async_return or support.all_returned(H, gen_inds): + if async_return or support.all_sim_end(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), persistent=True, active_recv=active_recv_gen) @@ -185,7 +185,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info gen_inds = (H['gen_worker'] == wid) returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): - if async_return or support.all_returned(H, gen_inds): + if async_return or support.all_sim_end(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), persistent=True, active_recv=active_recv_gen) diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 5b10e9cf8..6f7e5dbb9 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -50,7 +50,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per # returned, give them back to i. Otherwise, give nothing to wid for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): gen_inds = (H['gen_worker'] == wid) - if support.all_returned(H, gen_inds): + if support.all_sim_end(H, gen_inds): last_time_pos = np.argmax(H['sim_start_time'][gen_inds]) last_ind = np.nonzero(gen_inds)[0][last_time_pos] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], last_ind, persis_info[wid], persistent=True) diff --git a/libensemble/history.py b/libensemble/history.py index 0f54d31e1..2fcc910aa 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -28,13 +28,13 @@ class History: :ivar int index: Index where libEnsemble should start filling in H - :ivar int given_count: + :ivar int sim_start_count: Number of points given to sim functions (according to H) - :ivar int returned_count: + :ivar int sim_end_count: Number of points evaluated (according to H) - Note that index, given_count and returned_count reflect the total number of points + Note that index, sim_start_count and sim_end_count reflect the total number of points in H and therefore include those prepended to H in addition to the current run. """ @@ -84,14 +84,14 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.index = len(H0) self.grow_count = 0 - self.given_count = np.sum(H['sim_start']) - self.returned_count = np.sum(H['sim_end']) - self.given_back_count = np.sum(H['gen_informed']) + self.sim_start_count = np.sum(H['sim_start']) + self.sim_end_count = np.sum(H['sim_end']) + self.gen_informed_count = np.sum(H['gen_informed']) self.given_back_warned = False - self.given_offset = self.given_count - self.returned_offset = self.returned_count - self.given_back_offset = self.given_back_count + self.sim_start_offset = self.sim_start_count + self.sim_end_offset = self.sim_end_count + self.gen_informed_offset = self.gen_informed_count def update_history_f(self, D, safe_mode): """ @@ -120,7 +120,7 @@ def update_history_f(self, D, safe_mode): self.H['sim_end'][ind] = True self.H['sim_end_time'][ind] = time.time() - self.returned_count += 1 + self.sim_end_count += 1 def update_history_x_out(self, q_inds, sim_worker): """ @@ -141,7 +141,7 @@ def update_history_x_out(self, q_inds, sim_worker): self.H['sim_start_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker - self.given_count += len(q_inds) + self.sim_start_count += len(q_inds) def update_history_to_gen(self, q_inds): """Updates the history (in place) when points are given back to the gen""" @@ -162,7 +162,7 @@ def update_history_to_gen(self, q_inds): self.given_back_warned = True self.H['gen_informed_time'][q_inds] = t - self.given_back_count += len(q_inds) + self.gen_informed_count += len(q_inds) def update_history_x_in(self, gen_worker, D, safe_mode, gen_start_time): """ diff --git a/libensemble/libE.py b/libensemble/libE.py index 44d6db5d5..f98fe156a 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -612,9 +612,9 @@ def _dump_on_abort(hist, persis_info, save_H=True): """Dump history and persis_info on abort""" logger.error("Manager exception raised .. aborting ensemble:") logger.error("Dumping ensemble history with {} sims evaluated:". - format(hist.returned_count)) + format(hist.sim_end_count)) if save_H: - np.save('libE_history_at_abort_' + str(hist.returned_count) + '.npy', hist.trim_H()) - with open('libE_persis_info_at_abort_' + str(hist.returned_count) + '.pickle', "wb") as f: + np.save('libE_history_at_abort_' + str(hist.sim_end_count) + '.npy', hist.trim_H()) + with open('libE_persis_info_at_abort_' + str(hist.sim_end_count) + '.pickle', "wb") as f: pickle.dump(persis_info, f) diff --git a/libensemble/manager.py b/libensemble/manager.py index 00d565268..1cb370513 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -204,11 +204,11 @@ def term_test_wallclock(self, max_elapsed): def term_test_sim_max(self, sim_max): """Checks against max simulations""" - return self.hist.returned_count >= sim_max + self.hist.returned_offset + return self.hist.sim_end_count >= sim_max + self.hist.sim_end_offset def term_test_gen_max(self, gen_max): """Checks against max generator calls""" - return self.hist.index >= gen_max + self.hist.given_back_offset + return self.hist.index >= gen_max + self.hist.gen_informed_offset def term_test_stop_val(self, stop_val): """Checks against stop value criterion""" @@ -247,7 +247,7 @@ def _save_every_k(self, fname, count, k): def _save_every_k_sims(self): """Saves history every kth sim step""" self._save_every_k('libE_history_for_run_starting_{}_after_sim_{}.npy', - self.hist.returned_count, + self.hist.sim_end_count, self.libE_specs['save_every_k_sims']) def _save_every_k_gens(self): @@ -528,7 +528,7 @@ def _final_receive_and_kill(self, persis_info): def _sim_max_given(self): if 'sim_max' in self.exit_criteria: - return self.hist.given_count >= self.exit_criteria['sim_max'] + self.hist.given_offset + return self.hist.sim_start_count >= self.exit_criteria['sim_max'] + self.hist.sim_start_offset else: return False @@ -539,9 +539,9 @@ def _get_alloc_libE_info(self): 'exit_criteria': self.exit_criteria, 'elapsed_time': self.elapsed(), 'manager_kill_canceled_sims': self.kill_canceled_sims, - 'given_count': self.hist.given_count, - 'returned_count': self.hist.returned_count, - 'given_back_count': self.hist.given_back_count, + 'sim_start_count': self.hist.sim_start_count, + 'sim_end_count': self.hist.sim_end_count, + 'gen_informed_count': self.hist.gen_informed_count, 'sim_max_given': self._sim_max_given(), 'use_resource_sets': 'num_resource_sets' in self.libE_specs} diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index 4efbf1cb2..5bc29816e 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -13,7 +13,7 @@ __all__ = ['executor_hworld'] # Alt send values through X -returned_count = 0 +sim_end_count = 0 def custom_polling_loop(exctr, task, timeout_sec=5.0, delay=0.3): @@ -83,24 +83,24 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): timeout = 65.0 else: - global returned_count - returned_count += 1 + global sim_end_count + sim_end_count += 1 timeout = 6.0 launch_shc = False - print(returned_count) + print(sim_end_count) - if returned_count == 1: + if sim_end_count == 1: args_for_sim = 'sleep 1' # Should finish - elif returned_count == 2: + elif sim_end_count == 2: args_for_sim = 'sleep 1 Error' # Worker kill on error - elif returned_count == 3: + elif sim_end_count == 3: wait = True args_for_sim = 'sleep 1' # Should finish launch_shc = True - elif returned_count == 4: + elif sim_end_count == 4: args_for_sim = 'sleep 8' # Worker kill on timeout timeout = 1.0 - elif returned_count == 5: + elif sim_end_count == 5: args_for_sim = 'sleep 2 Fail' # Manager kill - if signal received else completes if USE_BALSAM: @@ -127,9 +127,9 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): else: if not ELAPSED_TIMEOUT: - if returned_count >= 2 and not USE_BALSAM: + if sim_end_count >= 2 and not USE_BALSAM: calc_status = exctr.polling_loop(task, timeout=timeout, delay=0.3, poll_manager=True) - if returned_count == 2 and task.stdout_exists() and 'Error' in task.read_stdout(): + if sim_end_count == 2 and task.stdout_exists() and 'Error' in task.read_stdout(): calc_status = WORKER_KILL_ON_ERR else: diff --git a/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py b/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py index ccc63cb3d..863950466 100644 --- a/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py +++ b/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py @@ -81,7 +81,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) -# num returned_count conditions in executor_hworld +# num sim_end_count conditions in executor_hworld exit_criteria = {'sim_max': nworkers * 5} # Perform the run diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 0aaa28afb..7137a89f9 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -227,7 +227,7 @@ def test_als_gen_work(): clear_resources() -def test_als_all_given(): +def test_als_all_sim_start(): H_some_gvn = H.copy() H_some_gvn['sim_start'] = np.array([True, False, False, True, True]) als = AllocSupport(W, True) @@ -235,47 +235,47 @@ def test_als_all_given(): myfilter1 = np.array([False, True, False, True, True]) myfilter2 = np.array([True, False, False, False, True]) - assert not als.all_given(H_some_gvn), \ - "all_given() should\'ve returned False on incomplete sim results." + assert not als.all_sim_start(H_some_gvn), \ + "all_sim_start() should\'ve returned False on incomplete sim results." - assert als.all_given(H_some_gvn, low_bound=3), \ - "all_given() should\'ve returned True with adjusted lower bound." + assert als.all_sim_start(H_some_gvn, low_bound=3), \ + "all_sim_start() should\'ve returned True with adjusted lower bound." - assert not als.all_given(H_some_gvn, pt_filter=myfilter1), \ - "all_given() should\'ve returned False based on boolean filter." + assert not als.all_sim_start(H_some_gvn, pt_filter=myfilter1), \ + "all_sim_start() should\'ve returned False based on boolean filter." - assert als.all_given(H_some_gvn, pt_filter=myfilter1, low_bound=3), \ - "all_given() should\'ve returned True with boolean filter and adjusted lower bound." + assert als.all_sim_start(H_some_gvn, pt_filter=myfilter1, low_bound=3), \ + "all_sim_start() should\'ve returned True with boolean filter and adjusted lower bound." - assert als.all_given(H_some_gvn, pt_filter=myfilter2), \ - "all_given() should\'ve returned True based on boolean filter." + assert als.all_sim_start(H_some_gvn, pt_filter=myfilter2), \ + "all_sim_start() should\'ve returned True based on boolean filter." # Now cancel a point H_some_gvn['cancel_requested'] = np.array([False, True, False, False, False]) - assert not als.all_given(H_some_gvn), \ - "all_given(H_some_gvn) should\'ve returned False on incomplete sim results." + assert not als.all_sim_start(H_some_gvn), \ + "all_sim_start(H_some_gvn) should\'ve returned False on incomplete sim results." - assert als.all_given(H_some_gvn, pt_filter=myfilter1), \ - "all_given(H_some_gvn) should\'ve returned True based on boolean filter and cancelled." + assert als.all_sim_start(H_some_gvn, pt_filter=myfilter1), \ + "all_sim_start(H_some_gvn) should\'ve returned True based on boolean filter and cancelled." # Now cancel more points H_some_gvn['cancel_requested'] = np.array([False, True, True, False, False]) - assert als.all_given(H_some_gvn), \ - "all_given(H_some_gvn) should\'ve returned True as cancelled point ignored." + assert als.all_sim_start(H_some_gvn), \ + "all_sim_start(H_some_gvn) should\'ve returned True as cancelled point ignored." # Now cancel a different point H_some_gvn['cancel_requested'] = np.array([False, False, True, False, False]) - assert not als.all_given(H_some_gvn), \ - "all_given(H_some_gvn) should\'ve returned False on incomplete sim results." + assert not als.all_sim_start(H_some_gvn), \ + "all_sim_start(H_some_gvn) should\'ve returned False on incomplete sim results." - assert als.all_given(H_some_gvn, low_bound=2), \ - "all_given(H_some_gvn) should\'ve returned True with cancelled and adjusted lower bound." + assert als.all_sim_start(H_some_gvn, low_bound=2), \ + "all_sim_start(H_some_gvn) should\'ve returned True with cancelled and adjusted lower bound." -def test_als_all_returned(): +def test_als_all_sim_end(): H_some_rtn = H.copy() H_some_rtn['sim_end'] = np.array([True, False, False, True, True]) als = AllocSupport(W, True) @@ -283,53 +283,53 @@ def test_als_all_returned(): myfilter1 = np.array([False, True, False, True, True]) myfilter2 = np.array([True, False, False, False, True]) - assert not als.all_returned(H_some_rtn), \ - "all_returned() should\'ve returned False on incomplete sim results." + assert not als.all_sim_end(H_some_rtn), \ + "all_sim_end() should\'ve returned False on incomplete sim results." - assert als.all_returned(H_some_rtn, low_bound=3), \ - "all_returned() should\'ve returned True with adjusted lower bound." + assert als.all_sim_end(H_some_rtn, low_bound=3), \ + "all_sim_end() should\'ve returned True with adjusted lower bound." - assert not als.all_returned(H_some_rtn, pt_filter=myfilter1), \ - "all_returned() should\'ve returned False based on boolean filter." + assert not als.all_sim_end(H_some_rtn, pt_filter=myfilter1), \ + "all_sim_end() should\'ve returned False based on boolean filter." - assert als.all_returned(H_some_rtn, pt_filter=myfilter1, low_bound=3), \ - "all_returned() should\'ve returned True with boolean filter and adjusted lower bound." + assert als.all_sim_end(H_some_rtn, pt_filter=myfilter1, low_bound=3), \ + "all_sim_end() should\'ve returned True with boolean filter and adjusted lower bound." - assert als.all_returned(H_some_rtn, pt_filter=myfilter2), \ - "all_returned() should\'ve returned True based on boolean filter." + assert als.all_sim_end(H_some_rtn, pt_filter=myfilter2), \ + "all_sim_end() should\'ve returned True based on boolean filter." # Now cancel a point H_some_rtn['cancel_requested'] = np.array([False, True, False, False, False]) - assert not als.all_returned(H_some_rtn), \ - "all_returned() should\'ve returned False on incomplete sim results." + assert not als.all_sim_end(H_some_rtn), \ + "all_sim_end() should\'ve returned False on incomplete sim results." - assert als.all_returned(H_some_rtn, pt_filter=myfilter1), \ - "all_returned() should\'ve returned True based on boolean filter and cancelled." + assert als.all_sim_end(H_some_rtn, pt_filter=myfilter1), \ + "all_sim_end() should\'ve returned True based on boolean filter and cancelled." # Now cancel more points H_some_rtn['cancel_requested'] = np.array([False, True, True, False, False]) - assert als.all_returned(H_some_rtn), \ - "all_returned() should\'ve returned True as cancelled point ignored." + assert als.all_sim_end(H_some_rtn), \ + "all_sim_end() should\'ve returned True as cancelled point ignored." # Now cancel a different point H_some_rtn['cancel_requested'] = np.array([False, False, True, False, False]) - assert not als.all_returned(H_some_rtn), \ - "all_returned() should\'ve returned False on incomplete sim results." + assert not als.all_sim_end(H_some_rtn), \ + "all_sim_end() should\'ve returned False on incomplete sim results." - assert als.all_returned(H_some_rtn, low_bound=2), \ - "all_returned() should\'ve returned True with cancelled and adjusted lower bound." + assert als.all_sim_end(H_some_rtn, low_bound=2), \ + "all_sim_end() should\'ve returned True with cancelled and adjusted lower bound." # But if the cancelled point is already given, we still expect it back. H_some_rtn['sim_start'] = np.array([False, False, True, False, False]) - assert not als.all_returned(H_some_rtn, low_bound=2), \ - "all_returned() should\'ve returned False with given cancelled and adjusted lower bound." + assert not als.all_sim_end(H_some_rtn, low_bound=2), \ + "all_sim_end() should\'ve returned False with given cancelled and adjusted lower bound." -def test_als_all_given_back(): +def test_als_all_gen_informed(): H_some_gvnbk = H.copy() H_some_gvnbk['gen_informed'] = np.array([True, False, False, True, True]) als = AllocSupport(W, True) @@ -337,50 +337,50 @@ def test_als_all_given_back(): myfilter1 = np.array([False, True, False, True, True]) myfilter2 = np.array([True, False, False, False, True]) - assert not als.all_given_back(H_some_gvnbk), \ - "all_given_back() should\'ve returned False on incomplete sim results." + assert not als.all_gen_informed(H_some_gvnbk), \ + "all_gen_informed() should\'ve returned False on incomplete sim results." - assert als.all_given_back(H_some_gvnbk, low_bound=3), \ - "all_given_back() should\'ve returned True with adjusted lower bound." + assert als.all_gen_informed(H_some_gvnbk, low_bound=3), \ + "all_gen_informed() should\'ve returned True with adjusted lower bound." - assert not als.all_given_back(H_some_gvnbk, pt_filter=myfilter1), \ - "all_given_back() should\'ve returned False based on boolean filter." + assert not als.all_gen_informed(H_some_gvnbk, pt_filter=myfilter1), \ + "all_gen_informed() should\'ve returned False based on boolean filter." - assert als.all_given_back(H_some_gvnbk, pt_filter=myfilter1, low_bound=3), \ - "all_given_back() should\'ve returned True with boolean filter and adjusted lower bound." + assert als.all_gen_informed(H_some_gvnbk, pt_filter=myfilter1, low_bound=3), \ + "all_gen_informed() should\'ve returned True with boolean filter and adjusted lower bound." - assert als.all_given_back(H_some_gvnbk, pt_filter=myfilter2), \ - "all_given_back() should\'ve returned True based on boolean filter." + assert als.all_gen_informed(H_some_gvnbk, pt_filter=myfilter2), \ + "all_gen_informed() should\'ve returned True based on boolean filter." # Now cancel a point H_some_gvnbk['cancel_requested'] = np.array([False, True, False, False, False]) - assert not als.all_given_back(H_some_gvnbk), \ - "all_given_back() should\'ve returned False on incomplete sim results." + assert not als.all_gen_informed(H_some_gvnbk), \ + "all_gen_informed() should\'ve returned False on incomplete sim results." - assert als.all_given_back(H_some_gvnbk, pt_filter=myfilter1), \ - "all_given_back() should\'ve returned True based on boolean filter and cancelled." + assert als.all_gen_informed(H_some_gvnbk, pt_filter=myfilter1), \ + "all_gen_informed() should\'ve returned True based on boolean filter and cancelled." # Now cancel more points H_some_gvnbk['cancel_requested'] = np.array([False, True, True, False, False]) - assert als.all_given_back(H_some_gvnbk), \ - "all_given_back() should\'ve returned True as cancelled point ignored." + assert als.all_gen_informed(H_some_gvnbk), \ + "all_gen_informed() should\'ve returned True as cancelled point ignored." # Now cancel a different point H_some_gvnbk['cancel_requested'] = np.array([False, False, True, False, False]) - assert not als.all_given_back(H_some_gvnbk), \ - "all_given_back() should\'ve returned False on incomplete sim results." + assert not als.all_gen_informed(H_some_gvnbk), \ + "all_gen_informed() should\'ve returned False on incomplete sim results." - assert als.all_given_back(H_some_gvnbk, low_bound=2), \ - "all_given_back() should\'ve returned True with cancelled and adjusted lower bound." + assert als.all_gen_informed(H_some_gvnbk, low_bound=2), \ + "all_gen_informed() should\'ve returned True with cancelled and adjusted lower bound." # But if the cancelled point is already given, we still expect it back. H_some_gvnbk['sim_start'] = np.array([False, False, True, False, False]) - assert not als.all_given_back(H_some_gvnbk, low_bound=2), \ - "all_given_back() should\'ve returned False with given cancelled and adjusted lower bound." + assert not als.all_gen_informed(H_some_gvnbk, low_bound=2), \ + "all_gen_informed() should\'ve returned False with given cancelled and adjusted lower bound." def test_als_points_by_priority(): @@ -412,7 +412,7 @@ def test_als_points_by_priority(): test_als_evaluate_gens() test_als_sim_work() test_als_gen_work() - test_als_all_given() - test_als_all_returned() - test_als_all_given_back() + test_als_all_sim_start() + test_als_all_sim_end() + test_als_all_gen_informed() test_als_points_by_priority() diff --git a/libensemble/tests/unit_tests/test_history.py b/libensemble/tests/unit_tests/test_history.py index 504d015ad..eb41adf22 100644 --- a/libensemble/tests/unit_tests/test_history.py +++ b/libensemble/tests/unit_tests/test_history.py @@ -32,39 +32,39 @@ ('sim_end_time', ' Date: Mon, 28 Mar 2022 13:44:29 -0500 Subject: [PATCH 23/33] A few more stragglers --- docs/function_guides/allocator.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/function_guides/allocator.rst b/docs/function_guides/allocator.rst index 7b3fce5e6..13cb69f0b 100644 --- a/docs/function_guides/allocator.rst +++ b/docs/function_guides/allocator.rst @@ -96,9 +96,9 @@ routine can be found in ``libE_info``, passed into the allocation function:: libE_info = {'exit_criteria': dict, # Criteria for ending routine 'elapsed_time': float, # Time elapsed since start of routine 'manager_kill_canceled_sims': bool, # True if manager is to send kills to cancelled simulations - 'given_count': int, # Total number of points given for simulation function evaluation - 'returned_count': int, # Total number of points returned from simulation function evaluations - 'given_back_count': int, # Total number of evaluated points given back to a generator function + 'sim_start_count': int, # Total number of points given for simulation function evaluation + 'sim_end_count': int, # Total number of points returned from simulation function evaluations + 'gen_informed_count': int, # Total number of evaluated points given back to a generator function 'sim_max_given': bool, # True if `sim_max` simulations have been given out to workers 'use_resource_sets': bool} # True if num_resource_sets has been explicitly set. @@ -113,7 +113,7 @@ allocation function and detect impending timeouts, then pack up cleanup work req or mark points for cancellation. The remaining values above are useful for efficient filtering of H values -(e.g. ``returned_count``), saves a filtering an entire column of H. +(e.g. ``sim_end_count``), saves a filtering an entire column of H. .. note:: An error occurs when the ``alloc_f`` returns nothing while all workers are idle From 8ccd5ed2e3cf78c32dffd747e5cee787babe69cb Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 08:02:47 -0500 Subject: [PATCH 24/33] Feedback from Angel for discussion. --- libensemble/tools/fields_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 18339e938..5c408bca2 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -8,9 +8,9 @@ ('gen_start_time', float), # Time gen_worker was initiated that produced this entry ('gen_end_time', float), # Time gen_worker requested this entry ('sim_worker', int), # Worker that did (or is doing) the sim eval for this entry - ('sim_start', bool), # True if entry was given to sim_worker for sim eval + ('sim_started', bool), # True if entry was given to sim_worker for sim eval ('sim_start_time', float), # Time entry was given to sim_worker for a sim eval - ('sim_end', bool), # True if entry's sim eval completed + ('sim_ended', bool), # True if entry's sim eval completed ('sim_end_time', float), # Time entry's sim eval completed ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry From 7b361a50ad7273f597c20f7520562d3ab021018e Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 08:04:29 -0500 Subject: [PATCH 25/33] Feedback from Angel for discussion. --- libensemble/tools/fields_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 5c408bca2..528387bb2 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -24,9 +24,9 @@ 'gen_start_time', 'gen_end_time', 'sim_worker', - 'sim_start', + 'sim_started', 'sim_start_time', - 'sim_end', + 'sim_ended', 'sim_end_time', 'gen_informed', 'gen_informed_time', From cf14986091cc1033298aba177b08256b8e88893b Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 12:51:10 -0500 Subject: [PATCH 26/33] Some sim_start to sim_started --- .../alloc_funcs/give_sim_work_first.py | 6 +- .../alloc_funcs/inverse_bayes_allocf.py | 2 +- .../alloc_funcs/start_fd_persistent.py | 2 +- .../alloc_funcs/start_only_persistent.py | 4 +- .../alloc_funcs/start_persistent_consensus.py | 2 +- .../start_persistent_local_opt_gens.py | 4 +- libensemble/history.py | 28 ++++---- libensemble/manager.py | 6 +- .../test_evaluate_existing_sample.py | 4 +- .../test_evaluate_mixed_sample.py | 4 +- .../test_persistent_aposmm_with_grad.py | 4 +- .../test_uniform_sampling_cancel.py | 10 +-- .../test_allocation_funcs_and_support.py | 12 ++-- libensemble/tests/unit_tests/test_history.py | 70 +++++++++---------- .../tests/unit_tests/test_libE_main.py | 4 +- .../tests/unit_tests/test_manager_main.py | 10 +-- .../tests/unit_tests/tofix__test_worker.py | 2 +- libensemble/tools/alloc_support.py | 6 +- libensemble/tools/check_inputs.py | 2 +- libensemble/tools/fields_keys.py | 4 +- 20 files changed, 93 insertions(+), 93 deletions(-) diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 1277bcbcc..daf800ddb 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -33,8 +33,8 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long - rows = np.where(np.logical_and.reduce((H['sim_start'], ~H['sim_end'], ~H['cancel_requested'])))[0] - inds = time.time() - H['sim_start_time'][rows] > user['cancel_sims_time'] + rows = np.where(np.logical_and.reduce((H['sim_started'], ~H['sim_end'], ~H['cancel_requested'])))[0] + inds = time.time() - H['sim_started_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: H[row]['cancel_requested'] = True @@ -52,7 +52,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li gen_count = support.count_gens() Work = {} - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(): if np.any(points_to_evaluate): diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index c98c6f0f7..bb8da0e7a 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -45,7 +45,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe Work[wid] = support.gen_work(wid, ['like'], inds_to_send_back, persis_info.get(wid), persistent=True) - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(persistent=False): if np.any(points_to_evaluate): diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index 96efee41d..ea8ccb18d 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -50,7 +50,7 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE Work[wid] = support.gen_work(wid, gen_specs['persis_in'], inds_to_send, persis_info.get(wid), persistent=True) - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] for wid in support.avail_worker_ids(persistent=False): if np.any(points_to_evaluate): # perform sim evaluations (if they exist in History). diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 8947b2908..40eaee0a2 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -81,7 +81,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l returned_but_not_given[point_ids] = False # Now the give_sim_work_first part - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] avail_workers = support.avail_worker_ids(persistent=False, zero_resource_workers=False) for wid in avail_workers: @@ -192,7 +192,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info returned_but_not_given[point_ids] = False # Now the give_sim_work_first part - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] avail_workers = list( set(support.avail_worker_ids(persistent=False, zero_resource_workers=False)) | set(support.avail_worker_ids(persistent=EVAL_SIM_TAG, zero_resource_workers=False))) diff --git a/libensemble/alloc_funcs/start_persistent_consensus.py b/libensemble/alloc_funcs/start_persistent_consensus.py index 7b945d505..65ba7b0da 100644 --- a/libensemble/alloc_funcs/start_persistent_consensus.py +++ b/libensemble/alloc_funcs/start_persistent_consensus.py @@ -267,7 +267,7 @@ def start_consensus_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, per # skip points that are not sim work or are already done while persis_info['next_to_give'] < len(H) and \ - (H[persis_info['next_to_give']]['sim_start'] or + (H[persis_info['next_to_give']]['sim_started'] or H[persis_info['next_to_give']]['consensus_pt'] or H[persis_info['next_to_give']]['cancel_requested']): diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 6f7e5dbb9..ab42aca80 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -33,7 +33,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per support = AllocSupport(W, manage_resources, persis_info, sched_opts) Work = {} gen_count = support.count_persis_gens() - points_to_evaluate = ~H['sim_start'] & ~H['cancel_requested'] + points_to_evaluate = ~H['sim_started'] & ~H['cancel_requested'] # If a persistent localopt run has just finished, use run_order to update H # and then remove other information from persis_info @@ -51,7 +51,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): gen_inds = (H['gen_worker'] == wid) if support.all_sim_end(H, gen_inds): - last_time_pos = np.argmax(H['sim_start_time'][gen_inds]) + last_time_pos = np.argmax(H['sim_started_time'][gen_inds]) last_ind = np.nonzero(gen_inds)[0][last_time_pos] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], last_ind, persis_info[wid], persistent=True) persis_info[wid]['run_order'].append(last_ind) diff --git a/libensemble/history.py b/libensemble/history.py index 2fcc910aa..15f0b0052 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -28,13 +28,13 @@ class History: :ivar int index: Index where libEnsemble should start filling in H - :ivar int sim_start_count: + :ivar int sim_started_count: Number of points given to sim functions (according to H) :ivar int sim_end_count: Number of points evaluated (according to H) - Note that index, sim_start_count and sim_end_count reflect the total number of points + Note that index, sim_started_count and sim_end_count reflect the total number of points in H and therefore include those prepended to H in addition to the current run. """ @@ -63,20 +63,20 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): # for ind, val in np.ndenumerate(H0[field]): # Works if H0[field] has arbitrary dimension but is slow # H[field][ind] = val - if 'sim_start' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'sim_start' and 'sim_end'") - H['sim_start'][:len(H0)] = 1 + if 'sim_started' not in fields: + logger.manager_warning("Marking entries in H0 as having been 'sim_started' and 'sim_end'") + H['sim_started'][:len(H0)] = 1 H['sim_end'][:len(H0)] = 1 elif 'sim_end' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'sim_end' if 'sim_start'") - H['sim_end'][:len(H0)] = H0['sim_start'] + logger.manager_warning("Marking entries in H0 as having been 'sim_end' if 'sim_started'") + H['sim_end'][:len(H0)] = H0['sim_started'] if 'sim_id' not in fields: logger.manager_warning("Assigning sim_ids to entries in H0") H['sim_id'][:len(H0)] = np.arange(0, len(H0)) H['sim_id'][-L:] = -1 - H['sim_start_time'][-L:] = np.inf + H['sim_started_time'][-L:] = np.inf H['gen_informed_time'][-L:] = np.inf self.H = H @@ -84,12 +84,12 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.index = len(H0) self.grow_count = 0 - self.sim_start_count = np.sum(H['sim_start']) + self.sim_started_count = np.sum(H['sim_started']) self.sim_end_count = np.sum(H['sim_end']) self.gen_informed_count = np.sum(H['gen_informed']) self.given_back_warned = False - self.sim_start_offset = self.sim_start_count + self.sim_started_offset = self.sim_started_count self.sim_end_offset = self.sim_end_count self.gen_informed_offset = self.gen_informed_count @@ -137,11 +137,11 @@ def update_history_x_out(self, q_inds, sim_worker): q_inds = np.atleast_1d(q_inds) t = time.time() - self.H['sim_start'][q_inds] = True - self.H['sim_start_time'][q_inds] = t + self.H['sim_started'][q_inds] = True + self.H['sim_started_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker - self.sim_start_count += len(q_inds) + self.sim_started_count += len(q_inds) def update_history_to_gen(self, q_inds): """Updates the history (in place) when points are given back to the gen""" @@ -230,7 +230,7 @@ def grow_H(self, k): """ H_1 = np.zeros(k, dtype=self.H.dtype) H_1['sim_id'] = -1 - H_1['sim_start_time'] = np.inf + H_1['sim_started_time'] = np.inf H_1['gen_informed_time'] = np.inf self.H = np.append(self.H, H_1) diff --git a/libensemble/manager.py b/libensemble/manager.py index 1cb370513..f47d76ede 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -468,7 +468,7 @@ def _handle_msg_from_worker(self, persis_info, w): def _kill_cancelled_sims(self): """Send kill signals to any sims marked as cancel_requested""" if self.kill_canceled_sims: - kill_sim = self.hist.H['sim_start'] & self.hist.H['cancel_requested'] \ + kill_sim = self.hist.H['sim_started'] & self.hist.H['cancel_requested'] \ & ~self.hist.H['sim_end'] & ~self.hist.H['kill_sent'] # Note that a return is still expected when running sims are killed @@ -528,7 +528,7 @@ def _final_receive_and_kill(self, persis_info): def _sim_max_given(self): if 'sim_max' in self.exit_criteria: - return self.hist.sim_start_count >= self.exit_criteria['sim_max'] + self.hist.sim_start_offset + return self.hist.sim_started_count >= self.exit_criteria['sim_max'] + self.hist.sim_started_offset else: return False @@ -539,7 +539,7 @@ def _get_alloc_libE_info(self): 'exit_criteria': self.exit_criteria, 'elapsed_time': self.elapsed(), 'manager_kill_canceled_sims': self.kill_canceled_sims, - 'sim_start_count': self.hist.sim_start_count, + 'sim_started_count': self.hist.sim_started_count, 'sim_end_count': self.hist.sim_end_count, 'gen_informed_count': self.hist.gen_informed_count, 'sim_max_given': self._sim_max_given(), diff --git a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py index 626abfcdc..3276611e1 100644 --- a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py @@ -35,12 +35,12 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('sim_id', int), ('sim_start', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('sim_id', int), ('sim_started', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) H0['sim_id'] = range(n_samp) -H0['sim_start'] = False +H0['sim_started'] = False alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 45bf32608..3effbc9ce 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -35,7 +35,7 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_start', bool), ('sim_end', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_started', bool), ('sim_end', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) @@ -43,7 +43,7 @@ for i in range(500): H0['f'][i] = borehole_func(H0['x'][i]) -H0['sim_start'][:500] = True +H0['sim_started'][:500] = True H0['sim_end'][:500] = True alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index f942d1610..af54f4b9c 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -95,7 +95,7 @@ ('sim_end', bool), ('f', float), ('gen_informed', bool), - ('sim_start', bool), + ('sim_started', bool), ] H0 = np.zeros(sample_size, dtype=H0_dtype) @@ -104,7 +104,7 @@ H0['x'] = np.round(minima, 1) H0['x_on_cube'] = (H0['x'] - gen_specs['user']['lb']) / (gen_specs['user']['ub'] - gen_specs['user']['lb']) H0['sim_id'] = range(sample_size) -H0[['sim_start', 'gen_informed', 'sim_end']] = True +H0[['sim_started', 'gen_informed', 'sim_end']] = True for i in range(sample_size): H0['f'][i] = six_hump_camel_func(H0['x'][i]) diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py index 88f99345a..503a632b5 100644 --- a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py +++ b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py @@ -41,21 +41,21 @@ def create_H0(persis_info, gen_specs, sim_max): n = len(lb) b = sim_max - H0 = np.zeros(b, dtype=[('x', float, 2), ('sim_id', int), ('sim_start', bool), ('cancel_requested', bool)]) + H0 = np.zeros(b, dtype=[('x', float, 2), ('sim_id', int), ('sim_started', bool), ('cancel_requested', bool)]) H0['x'] = persis_info[0]['rand_stream'].uniform(lb, ub, (b, n)) H0['sim_id'] = range(b) - H0['sim_start'] = False + H0['sim_started'] = False for i in range(b): if i % 10 == 0: H0[i]['cancel_requested'] = True # Using uniform_random_sample_cancel call - need to adjust some gen_specs though # gen_specs['out'].append(('sim_id', int)) - # gen_specs['out'].append(('sim_start', bool)) + # gen_specs['out'].append(('sim_started', bool)) # gen_specs['user']['gen_batch_size'] = sim_max # H0, persis_info[0] = uniform_random_sample_cancel({}, persis_info[0], gen_specs, {}) # H0['sim_id'] = range(gen_specs['user']['gen_batch_size']) - # H0['sim_start'] = False + # H0['sim_started'] = False return H0 @@ -149,7 +149,7 @@ def create_H0(persis_info, gen_specs, sim_max): if is_manager: assert flag == 0 assert np.all(H['cancel_requested'][::10]), 'Some values should be cancelled but are not' - assert np.all(~H['sim_start'][::10]), 'Some values are given that should not have been' + assert np.all(~H['sim_started'][::10]), 'Some values are given that should not have been' tol = 0.1 for m in minima: assert np.min(np.sum((H['x'] - m) ** 2, 1)) < tol diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 7137a89f9..f29d2f58c 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -23,10 +23,10 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-2.11, -1.63], 0., 2, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], - dtype=[('sim_start', '?'), ('gen_informed', '?'), ('gen_worker', ' Date: Mon, 4 Apr 2022 12:51:45 -0500 Subject: [PATCH 27/33] sim_start functions --- .../alloc_funcs/persistent_aposmm_alloc.py | 2 +- .../test_allocation_funcs_and_support.py | 44 +++++++++---------- libensemble/tools/alloc_support.py | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index d7aa83d71..a9b342305 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -31,7 +31,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info Work = {} if persis_info.get('first_call', True): - assert support.all_sim_start(H), "Initial points in H have never been given." + assert support.all_sim_started(H), "Initial points in H have never been given." assert support.all_sim_end(H), "Initial points in H have never been returned." assert support.all_gen_informed(H), "Initial points in H have never been given back to gen." diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index f29d2f58c..826ba64ff 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -227,7 +227,7 @@ def test_als_gen_work(): clear_resources() -def test_als_all_sim_start(): +def test_als_all_sim_started(): H_some_gvn = H.copy() H_some_gvn['sim_started'] = np.array([True, False, False, True, True]) als = AllocSupport(W, True) @@ -235,44 +235,44 @@ def test_als_all_sim_start(): myfilter1 = np.array([False, True, False, True, True]) myfilter2 = np.array([True, False, False, False, True]) - assert not als.all_sim_start(H_some_gvn), \ - "all_sim_start() should\'ve returned False on incomplete sim results." + assert not als.all_sim_started(H_some_gvn), \ + "all_sim_started() should\'ve returned False on incomplete sim results." - assert als.all_sim_start(H_some_gvn, low_bound=3), \ - "all_sim_start() should\'ve returned True with adjusted lower bound." + assert als.all_sim_started(H_some_gvn, low_bound=3), \ + "all_sim_started() should\'ve returned True with adjusted lower bound." - assert not als.all_sim_start(H_some_gvn, pt_filter=myfilter1), \ - "all_sim_start() should\'ve returned False based on boolean filter." + assert not als.all_sim_started(H_some_gvn, pt_filter=myfilter1), \ + "all_sim_started() should\'ve returned False based on boolean filter." - assert als.all_sim_start(H_some_gvn, pt_filter=myfilter1, low_bound=3), \ - "all_sim_start() should\'ve returned True with boolean filter and adjusted lower bound." + assert als.all_sim_started(H_some_gvn, pt_filter=myfilter1, low_bound=3), \ + "all_sim_started() should\'ve returned True with boolean filter and adjusted lower bound." - assert als.all_sim_start(H_some_gvn, pt_filter=myfilter2), \ - "all_sim_start() should\'ve returned True based on boolean filter." + assert als.all_sim_started(H_some_gvn, pt_filter=myfilter2), \ + "all_sim_started() should\'ve returned True based on boolean filter." # Now cancel a point H_some_gvn['cancel_requested'] = np.array([False, True, False, False, False]) - assert not als.all_sim_start(H_some_gvn), \ - "all_sim_start(H_some_gvn) should\'ve returned False on incomplete sim results." + assert not als.all_sim_started(H_some_gvn), \ + "all_sim_started(H_some_gvn) should\'ve returned False on incomplete sim results." - assert als.all_sim_start(H_some_gvn, pt_filter=myfilter1), \ - "all_sim_start(H_some_gvn) should\'ve returned True based on boolean filter and cancelled." + assert als.all_sim_started(H_some_gvn, pt_filter=myfilter1), \ + "all_sim_started(H_some_gvn) should\'ve returned True based on boolean filter and cancelled." # Now cancel more points H_some_gvn['cancel_requested'] = np.array([False, True, True, False, False]) - assert als.all_sim_start(H_some_gvn), \ - "all_sim_start(H_some_gvn) should\'ve returned True as cancelled point ignored." + assert als.all_sim_started(H_some_gvn), \ + "all_sim_started(H_some_gvn) should\'ve returned True as cancelled point ignored." # Now cancel a different point H_some_gvn['cancel_requested'] = np.array([False, False, True, False, False]) - assert not als.all_sim_start(H_some_gvn), \ - "all_sim_start(H_some_gvn) should\'ve returned False on incomplete sim results." + assert not als.all_sim_started(H_some_gvn), \ + "all_sim_started(H_some_gvn) should\'ve returned False on incomplete sim results." - assert als.all_sim_start(H_some_gvn, low_bound=2), \ - "all_sim_start(H_some_gvn) should\'ve returned True with cancelled and adjusted lower bound." + assert als.all_sim_started(H_some_gvn, low_bound=2), \ + "all_sim_started(H_some_gvn) should\'ve returned True with cancelled and adjusted lower bound." def test_als_all_sim_end(): @@ -412,7 +412,7 @@ def test_als_points_by_priority(): test_als_evaluate_gens() test_als_sim_work() test_als_gen_work() - test_als_all_sim_start() + test_als_all_sim_started() test_als_all_sim_end() test_als_all_gen_informed() test_als_points_by_priority() diff --git a/libensemble/tools/alloc_support.py b/libensemble/tools/alloc_support.py index fb3c522e2..4db59b6ba 100644 --- a/libensemble/tools/alloc_support.py +++ b/libensemble/tools/alloc_support.py @@ -237,7 +237,7 @@ def _filter_points(self, H_in, pt_filter, low_bound): pfilter = pt_filter return H, pfilter - def all_sim_start(self, H, pt_filter=None, low_bound=None): + def all_sim_started(self, H, pt_filter=None, low_bound=None): """Returns ``True`` if all expected points have started their sim Excludes cancelled points. From e360407ff2999b4f59e1fde1cf182b7e909cf1cc Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 12:55:19 -0500 Subject: [PATCH 28/33] More sim_started --- docs/data_structures/history_array.rst | 8 ++++---- docs/function_guides/allocator.rst | 2 +- docs/tutorials/calib_cancel_tutorial.rst | 4 ++-- docs/tutorials/local_sine_tutorial.rst | 2 +- postproc_scripts/compare_npy.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 3ebb05587..b30f43a14 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -58,7 +58,7 @@ function to request that libEnsemble cancels evaluation of the point. The remaining reserved fields are protected (populated by libEnsemble), and store information about each entry. These include boolean fields for the -current scheduling status of the point (``sim_start`` when the sim evaluation +current scheduling status of the point (``sim_started`` when the sim evaluation has started out, ``sim_end`` when sim evaluation has completed, and ``gen_informed`` when the sim output has been passed back to the generator). Timing fields give the time (since the epoch) corresponding to each state, and @@ -91,9 +91,9 @@ The full list of these reserved fields is given below. ``sim_worker`` [int]: Worker that did (or is doing) the sim evaluation for this entry -``sim_start`` [bool]: True if entry was given to sim_worker for sim evaluation +``sim_started`` [bool]: True if entry was given to sim_worker for sim evaluation -``sim_start_time`` [float]: Time entry was given to sim_worker for a sim evaluation +``sim_started_time`` [float]: Time entry was given to sim_worker for a sim evaluation ``sim_end`` [bool]: True if entry's sim evaluation completed @@ -119,7 +119,7 @@ Example workflow updating history array The history array is initialized using the libEnsemble reserved field and the user-provided ``gen_specs['out']`` and ``sim_specs['out']`` entries. In the figure below, only the -reserved fields: ``sim_id``, ``sim_start``, and ``sim_end`` are shown for brevity. +reserved fields: ``sim_id``, ``sim_started``, and ``sim_end`` are shown for brevity. .. figure:: ../images/history_init.png :scale: 40 diff --git a/docs/function_guides/allocator.rst b/docs/function_guides/allocator.rst index 13cb69f0b..db423f324 100644 --- a/docs/function_guides/allocator.rst +++ b/docs/function_guides/allocator.rst @@ -96,7 +96,7 @@ routine can be found in ``libE_info``, passed into the allocation function:: libE_info = {'exit_criteria': dict, # Criteria for ending routine 'elapsed_time': float, # Time elapsed since start of routine 'manager_kill_canceled_sims': bool, # True if manager is to send kills to cancelled simulations - 'sim_start_count': int, # Total number of points given for simulation function evaluation + 'sim_started_count': int, # Total number of points given for simulation function evaluation 'sim_end_count': int, # Total number of points returned from simulation function evaluations 'gen_informed_count': int, # Total number of evaluated points given back to a generator function 'sim_max_given': bool, # True if `sim_max` simulations have been given out to workers diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst index 01faa64c1..20747fac3 100644 --- a/docs/tutorials/calib_cancel_tutorial.rst +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -186,7 +186,7 @@ prepared for irregular sending /receiving of data. .. Between routines to call the allocation function and distribute allocated work .. to each Worker, the Manager selects points from the History array that are: .. -.. 1) Marked as ``'sim_start'`` by the allocation function +.. 1) Marked as ``'sim_started'`` by the allocation function .. 2) Marked with ``'cancel_requested'`` by the generator .. 3) *Not* been marked as ``'sim_end'`` by the Manager .. 4) *Not* been marked with ``'kill_sent'`` by the Manager @@ -198,7 +198,7 @@ prepared for irregular sending /receiving of data. .. function calls, any points in the Manager's History array that have ``'cancel_requested'`` .. as ``True`` are not allocated:: .. -.. task_avail = ~H['sim_start'] & ~H['cancel_requested'] +.. task_avail = ~H['sim_started'] & ~H['cancel_requested'] .. .. This ``alloc_f`` also can prioritize allocating points that have .. higher ``'priority'`` values from the ``gen_f`` values in the local History array:: diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 30be14c57..138f8cda4 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'sim_start_time', 'gen_worker', 'sim_worker', 'sim_start', 'sim_end', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'sim_started_time', 'gen_worker', 'sim_worker', 'sim_started', 'sim_end', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/postproc_scripts/compare_npy.py b/postproc_scripts/compare_npy.py index 3c7f64547..2b8f9a346 100755 --- a/postproc_scripts/compare_npy.py +++ b/postproc_scripts/compare_npy.py @@ -28,7 +28,7 @@ ./compare_npy.py out1.npy out2.npy --rtol 1e-03 --atol 1e-06 ''' -exclude_fields = ['gen_worker', 'sim_worker', 'gen_end_time', 'sim_start_time'] # list of fields to ignore +exclude_fields = ['gen_worker', 'sim_worker', 'gen_ended_time', 'sim_started_time'] # list of fields to ignore locate_mismatch = True parser = argparse.ArgumentParser(description=desc, epilog=example, From 5582a9d056ea9ef49ebfb69c9ac121c1e4ce359f Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 12:58:33 -0500 Subject: [PATCH 29/33] sim_end to sim_ended --- docs/data_structures/history_array.rst | 8 +- docs/function_guides/allocator.rst | 4 +- docs/tutorials/calib_cancel_tutorial.rst | 2 +- docs/tutorials/local_sine_tutorial.rst | 2 +- .../alloc_funcs/fast_alloc_and_pausing.py | 6 +- .../alloc_funcs/fast_alloc_to_aposmm.py | 4 +- .../alloc_funcs/give_sim_work_first.py | 4 +- .../alloc_funcs/inverse_bayes_allocf.py | 2 +- libensemble/alloc_funcs/only_one_gen_alloc.py | 2 +- .../alloc_funcs/persistent_aposmm_alloc.py | 6 +- .../alloc_funcs/start_fd_persistent.py | 2 +- .../alloc_funcs/start_only_persistent.py | 12 +-- .../alloc_funcs/start_persistent_consensus.py | 2 +- .../start_persistent_local_opt_gens.py | 4 +- .../gen_funcs/aposmm_localopt_support.py | 4 +- libensemble/gen_funcs/old_aposmm.py | 14 ++-- libensemble/gen_funcs/persistent_aposmm.py | 12 +-- libensemble/history.py | 32 ++++---- libensemble/libE.py | 6 +- libensemble/manager.py | 12 +-- libensemble/sim_funcs/executor_hworld.py | 22 +++--- .../test_nan_func_old_aposmm.py | 2 +- .../deprecated_tests/test_old_aposmm_logic.py | 12 +-- .../test_old_aposmm_one_residual_at_a_time.py | 2 +- .../test_old_aposmm_pounders.py | 2 +- .../test_old_aposmm_pounders_splitcomm.py | 2 +- .../test_old_aposmm_pounders_subcomm.py | 2 +- .../test_old_aposmm_sim_dirs.py | 2 +- .../test_old_aposmm_with_gradients.py | 2 +- .../test_evaluate_existing_sample.py | 2 +- .../test_evaluate_mixed_sample.py | 6 +- .../test_executor_hworld_pass_fail.py | 2 +- .../test_persistent_aposmm_dfols.py | 2 +- .../test_persistent_aposmm_scipy.py | 2 +- .../test_persistent_aposmm_with_grad.py | 6 +- .../test_persistent_surmise_calib.py | 4 +- .../test_persistent_surmise_killsims.py | 4 +- .../tests/scaling_tests/warpx/plot_results.py | 2 +- .../test_allocation_funcs_and_support.py | 54 +++++++------- libensemble/tests/unit_tests/test_history.py | 74 +++++++++---------- .../tests/unit_tests/test_libE_main.py | 12 +-- .../tests/unit_tests/test_manager_main.py | 2 +- .../unit_tests/test_persistent_aposmm.py | 12 +-- libensemble/tools/alloc_support.py | 4 +- libensemble/tools/check_inputs.py | 4 +- libensemble/tools/fields_keys.py | 4 +- libensemble/tools/tools.py | 2 +- postproc_scripts/print_npy.py | 4 +- postproc_scripts/readme.rst | 2 +- 49 files changed, 196 insertions(+), 196 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index b30f43a14..6d436a090 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -59,7 +59,7 @@ function to request that libEnsemble cancels evaluation of the point. The remaining reserved fields are protected (populated by libEnsemble), and store information about each entry. These include boolean fields for the current scheduling status of the point (``sim_started`` when the sim evaluation -has started out, ``sim_end`` when sim evaluation has completed, and +has started out, ``sim_ended`` when sim evaluation has completed, and ``gen_informed`` when the sim output has been passed back to the generator). Timing fields give the time (since the epoch) corresponding to each state, and when the point was generated. Other protected fields include the worker IDs on @@ -95,9 +95,9 @@ The full list of these reserved fields is given below. ``sim_started_time`` [float]: Time entry was given to sim_worker for a sim evaluation -``sim_end`` [bool]: True if entry's sim evaluation completed +``sim_ended`` [bool]: True if entry's sim evaluation completed -``sim_end_time`` [float]: Time entry's sim evaluation completed +``sim_ended_time`` [float]: Time entry's sim evaluation completed ``gen_informed`` [bool]: True if gen_worker was informed about the sim evaluation of this entry @@ -119,7 +119,7 @@ Example workflow updating history array The history array is initialized using the libEnsemble reserved field and the user-provided ``gen_specs['out']`` and ``sim_specs['out']`` entries. In the figure below, only the -reserved fields: ``sim_id``, ``sim_started``, and ``sim_end`` are shown for brevity. +reserved fields: ``sim_id``, ``sim_started``, and ``sim_ended`` are shown for brevity. .. figure:: ../images/history_init.png :scale: 40 diff --git a/docs/function_guides/allocator.rst b/docs/function_guides/allocator.rst index db423f324..9e790cf69 100644 --- a/docs/function_guides/allocator.rst +++ b/docs/function_guides/allocator.rst @@ -97,7 +97,7 @@ routine can be found in ``libE_info``, passed into the allocation function:: 'elapsed_time': float, # Time elapsed since start of routine 'manager_kill_canceled_sims': bool, # True if manager is to send kills to cancelled simulations 'sim_started_count': int, # Total number of points given for simulation function evaluation - 'sim_end_count': int, # Total number of points returned from simulation function evaluations + 'sim_ended_count': int, # Total number of points returned from simulation function evaluations 'gen_informed_count': int, # Total number of evaluated points given back to a generator function 'sim_max_given': bool, # True if `sim_max` simulations have been given out to workers 'use_resource_sets': bool} # True if num_resource_sets has been explicitly set. @@ -113,7 +113,7 @@ allocation function and detect impending timeouts, then pack up cleanup work req or mark points for cancellation. The remaining values above are useful for efficient filtering of H values -(e.g. ``sim_end_count``), saves a filtering an entire column of H. +(e.g. ``sim_ended_count``), saves a filtering an entire column of H. .. note:: An error occurs when the ``alloc_f`` returns nothing while all workers are idle diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst index 20747fac3..9a00bb907 100644 --- a/docs/tutorials/calib_cancel_tutorial.rst +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -188,7 +188,7 @@ prepared for irregular sending /receiving of data. .. .. 1) Marked as ``'sim_started'`` by the allocation function .. 2) Marked with ``'cancel_requested'`` by the generator -.. 3) *Not* been marked as ``'sim_end'`` by the Manager +.. 3) *Not* been marked as ``'sim_ended'`` by the Manager .. 4) *Not* been marked with ``'kill_sent'`` by the Manager .. .. If any points match these characteristics, the Workers that are processing these diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 138f8cda4..789434f0e 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'sim_started_time', 'gen_worker', 'sim_worker', 'sim_started', 'sim_end', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'sim_started_time', 'gen_worker', 'sim_worker', 'sim_started', 'sim_ended', 'x', 'allocated', 'sim_id', 'gen_end_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index 7b7927608..81e2fd89e 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -78,7 +78,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'local_pt' in H.dtype.names and H['local_pt'][a1][0]: persis_info['local_pt_ids'].add(pt_id) - if np.all(H['sim_end'][a1]): + if np.all(H['sim_ended'][a1]): persis_info['complete'].add(pt_id) values = gen_specs['user']['combine_component_func'](H['f_i'][a1]) persis_info['best_complete_val'] = min(persis_info['best_complete_val'], values) @@ -124,13 +124,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if len(H): # Don't give gen instances in batch mode if points are unfinished if (alloc_specs['user'].get('batch_mode') - and not all(np.logical_or(H['sim_end'][last_size:], H['paused'][last_size:]))): + and not all(np.logical_or(H['sim_ended'][last_size:], H['paused'][last_size:]))): break # Don't call APOSMM if there are runs going but none need advancing if len(persis_info[lw]['run_order']): runs_needing_to_advance = np.zeros(len(persis_info[lw]['run_order']), dtype=bool) for run, inds in enumerate(persis_info[lw]['run_order'].values()): - runs_needing_to_advance[run] = np.all(H['sim_end'][inds]) + runs_needing_to_advance[run] = np.all(H['sim_ended'][inds]) if not np.any(runs_needing_to_advance): break diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index a55acf4b4..a2153b400 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -49,13 +49,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if len(H): # Don't give gen instances in batch mode if points are unfinished - if user.get('batch_mode') and not support.all_sim_end(pt_filter=~H['paused'], low_bound=last_size): + if user.get('batch_mode') and not support.all_sim_ended(pt_filter=~H['paused'], low_bound=last_size): break # Don't call APOSMM if there are runs going but none need advancing if len(persis_info[lw]['run_order']): runs_needing_to_advance = np.zeros(len(persis_info[lw]['run_order']), dtype=bool) for run, inds in enumerate(persis_info[lw]['run_order'].values()): - runs_needing_to_advance[run] = H['sim_end'][inds[-1]] + runs_needing_to_advance[run] = H['sim_ended'][inds[-1]] if not np.any(runs_needing_to_advance): break diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index daf800ddb..b18929830 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -33,7 +33,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if 'cancel_sims_time' in user: # Cancel simulations that are taking too long - rows = np.where(np.logical_and.reduce((H['sim_started'], ~H['sim_end'], ~H['cancel_requested'])))[0] + rows = np.where(np.logical_and.reduce((H['sim_started'], ~H['sim_ended'], ~H['cancel_requested'])))[0] inds = time.time() - H['sim_started_time'][rows] > user['cancel_sims_time'] to_request_cancel = rows[inds] for row in to_request_cancel: @@ -69,7 +69,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li break # Do not start gen instances in batch mode if workers still working - if user.get('batch_mode') and not support.all_sim_end(H): + if user.get('batch_mode') and not support.all_sim_ended(H): break # Give gen work diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index bb8da0e7a..b12e3bcca 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -31,7 +31,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe # if > 1 persistent generator, assign the correct work to it inds_generated_by_wid = (H['gen_worker'] == wid) - if support.all_sim_end(H, inds_generated_by_wid): + if support.all_sim_ended(H, inds_generated_by_wid): # Has sim_f completed everything from this persistent worker? # Then give back everything in the last batch diff --git a/libensemble/alloc_funcs/only_one_gen_alloc.py b/libensemble/alloc_funcs/only_one_gen_alloc.py index 079bdf16b..338b790b6 100644 --- a/libensemble/alloc_funcs/only_one_gen_alloc.py +++ b/libensemble/alloc_funcs/only_one_gen_alloc.py @@ -38,7 +38,7 @@ def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info, elif not support.test_any_gen() and gen_flag: - if not support.all_sim_end(H): + if not support.all_sim_ended(H): break # Give gen work diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index a9b342305..5232de321 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -32,7 +32,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if persis_info.get('first_call', True): assert support.all_sim_started(H), "Initial points in H have never been given." - assert support.all_sim_end(H), "Initial points in H have never been returned." + assert support.all_sim_ended(H), "Initial points in H have never been returned." assert support.all_gen_informed(H), "Initial points in H have never been given back to gen." persis_info['samples_in_H0'] = sum(H['local_pt'] == 0) @@ -44,11 +44,11 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info # If any persistent worker's calculated values have returned, give them back. for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): - if (persis_info.get('sample_done') or sum(H['sim_end']) >= init_sample_size + persis_info['samples_in_H0']): + if (persis_info.get('sample_done') or sum(H['sim_ended']) >= init_sample_size + persis_info['samples_in_H0']): # Don't return if the initial sample is not complete persis_info['sample_done'] = True - returned_but_not_given = np.logical_and(H['sim_end'], ~H['gen_informed']) + returned_but_not_given = np.logical_and(H['sim_ended'], ~H['gen_informed']) if np.any(returned_but_not_given): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index ea8ccb18d..defe15447 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -42,7 +42,7 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE inds_to_send = np.array([], dtype=int) for x_ind in range(gen_specs['user']['n']): for f_ind in range(gen_specs['user']['p']): - inds = np.logical_and.reduce((H_tmp['x_ind'] == x_ind, H_tmp['f_ind'] == f_ind, H_tmp['sim_end'])) + inds = np.logical_and.reduce((H_tmp['x_ind'] == x_ind, H_tmp['f_ind'] == f_ind, H_tmp['sim_ended'])) if sum(inds) == gen_specs['user']['nf']: inds_to_send = np.append(inds_to_send, H_tmp['sim_id'][inds]) diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 40eaee0a2..707e0ad82 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -63,7 +63,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l Work = {} # Asynchronous return to generator - async_return = user.get('async_return', False) and sum(H['sim_end']) >= init_sample_size + async_return = user.get('async_return', False) and sum(H['sim_ended']) >= init_sample_size if gen_count < persis_info.get('num_gens_started', 0): # When a persistent worker is done, trigger a shutdown (returning exit condition of 1) @@ -72,9 +72,9 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_ended'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): - if async_return or support.all_sim_end(H, gen_inds): + if async_return or support.all_sim_ended(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), persistent=True, active_recv=active_recv_gen) @@ -174,7 +174,7 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info Work = {} # Asynchronous return to generator - async_return = user.get('async_return', False) and sum(H['sim_end']) >= init_sample_size + async_return = user.get('async_return', False) and sum(H['sim_ended']) >= init_sample_size if gen_count < persis_info.get('num_gens_started', 0): # When a persistent gen worker is done, trigger a shutdown (returning exit condition of 1) @@ -183,9 +183,9 @@ def only_persistent_workers(W, H, sim_specs, gen_specs, alloc_specs, persis_info # Give evaluated results back to a running persistent gen for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG, active_recv=active_recv_gen): gen_inds = (H['gen_worker'] == wid) - returned_but_not_given = np.logical_and.reduce((H['sim_end'], ~H['gen_informed'], gen_inds)) + returned_but_not_given = np.logical_and.reduce((H['sim_ended'], ~H['gen_informed'], gen_inds)) if np.any(returned_but_not_given): - if async_return or support.all_sim_end(H, gen_inds): + if async_return or support.all_sim_ended(H, gen_inds): point_ids = np.where(returned_but_not_given)[0] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], point_ids, persis_info.get(wid), persistent=True, active_recv=active_recv_gen) diff --git a/libensemble/alloc_funcs/start_persistent_consensus.py b/libensemble/alloc_funcs/start_persistent_consensus.py index 65ba7b0da..20f16c811 100644 --- a/libensemble/alloc_funcs/start_persistent_consensus.py +++ b/libensemble/alloc_funcs/start_persistent_consensus.py @@ -105,7 +105,7 @@ def start_consensus_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, per [l_H_id, r_H_id] = persis_info[wid].get('curr_H_ids') num_sims_req = r_H_id - l_H_id - num_fin_sims = np.sum(H['sim_end'][l_H_id:r_H_id]) + num_fin_sims = np.sum(H['sim_ended'][l_H_id:r_H_id]) completed_all_sims_for_gen_i = num_fin_sims == num_sims_req diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index ab42aca80..172a2cc9e 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -50,7 +50,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per # returned, give them back to i. Otherwise, give nothing to wid for wid in support.avail_worker_ids(persistent=EVAL_GEN_TAG): gen_inds = (H['gen_worker'] == wid) - if support.all_sim_end(H, gen_inds): + if support.all_sim_ended(H, gen_inds): last_time_pos = np.argmax(H['sim_started_time'][gen_inds]) last_ind = np.nonzero(gen_inds)[0][last_time_pos] Work[wid] = support.gen_work(wid, gen_specs['persis_in'], last_ind, persis_info[wid], persistent=True) @@ -58,7 +58,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per for wid in support.avail_worker_ids(persistent=False): # Find candidates to start local opt runs if a sample has been evaluated - if np.any(np.logical_and(~H['local_pt'], H['sim_end'], ~H['cancel_requested'])): + if np.any(np.logical_and(~H['local_pt'], H['sim_ended'], ~H['cancel_requested'])): n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs) update_history_dist(H, n, gen_specs['user'], c_flag=False) starting_inds = decide_where_to_start_localopt(H, r_k, mu, nu) diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index cd750d771..16357558f 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -563,10 +563,10 @@ def simulate_recv_from_manager(local_H, gen_specs): # output as if the calculations were performed externally by libEnsemble. user = gen_specs['user']['standalone'] - if np.sum(local_H['sim_end']) >= user['eval_max']: + if np.sum(local_H['sim_ended']) >= user['eval_max']: return STOP_TAG, {}, {} - H_rows = np.where(~local_H['sim_end'])[0] + H_rows = np.where(~local_H['sim_ended'])[0] H_fields = [i[0] for i in gen_specs['out']] Work = {'libE_info': {'H_rows': H_rows}, 'H_fields': H_fields} diff --git a/libensemble/gen_funcs/old_aposmm.py b/libensemble/gen_funcs/old_aposmm.py index 18a9eca4d..7d06bac40 100644 --- a/libensemble/gen_funcs/old_aposmm.py +++ b/libensemble/gen_funcs/old_aposmm.py @@ -243,7 +243,7 @@ def aposmm_logic(H, persis_info, gen_specs, _): # Find next point in any uncompleted run using persis_info['run_order'] for run in active_runs: - if not np.all(H['sim_end'][persis_info['run_order'][run]]): + if not np.all(H['sim_ended'][persis_info['run_order'][run]]): continue # Can't advance a run if all points aren't returned. x_opt, exit_code, persis_info, sorted_run_inds, x_new = advance_local_run(H, gen_specs['user'], c_flag, run, persis_info) @@ -399,9 +399,9 @@ def update_history_dist(H, n, gen_specs, c_flag): H['f'][inds] = np.inf H['f'][np.where(inds)[0][0]] = gen_specs['user']['combine_component_func'](H['f_i'][inds]) - p = np.logical_and.reduce((H['sim_end'], H['obj_component'] == 0, ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_ended'], H['obj_component'] == 0, ~np.isnan(H['f']))) else: - p = np.logical_and.reduce((H['sim_end'], ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_ended'], ~np.isnan(H['f']))) for new_ind in new_inds: # Loop over new returned points and update their distances @@ -783,7 +783,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): if nu > 0: test_2_through_5 = np.logical_and.reduce(( - H['sim_end'] == 1, # have a returned function value + H['sim_ended'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -795,7 +795,7 @@ def decide_where_to_start_localopt(H, r_k, mu=0, nu=0, gamma_quantile=1): )) else: test_2_through_5 = np.logical_and.reduce(( - H['sim_end'] == 1, # have a returned function value + H['sim_ended'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -915,12 +915,12 @@ def initialize_APOSMM(H, gen_specs): if c_flag: # Get the pt_id for non-nan, returned points - pt_ids = H['pt_id'][np.logical_and(H['sim_end'], ~np.isnan(H['f_i']))] + pt_ids = H['pt_id'][np.logical_and(H['sim_ended'], ~np.isnan(H['f_i']))] _, counts = np.unique(pt_ids, return_counts=True) n_s = np.sum(counts == user_specs['components']) else: # Number of returned sampled points (excluding nans) - n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['sim_end']))) + n_s = np.sum(np.logical_and.reduce((~np.isnan(H['f']), ~H['local_pt'], H['sim_ended']))) # Rather than build up a large output, we will just make changes in the # given H, and then send back the rows corresponding to updated H entries. diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index 54128afe0..d35993ce0 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -261,7 +261,7 @@ def update_local_H_after_receiving(local_H, n, n_s, user_specs, Work, calc_in, f for name in calc_in.dtype.names: local_H[name][Work['libE_info']['H_rows']] = calc_in[name] - local_H['sim_end'][Work['libE_info']['H_rows']] = True + local_H['sim_ended'][Work['libE_info']['H_rows']] = True n_s += np.sum(~local_H[Work['libE_info']['H_rows']]['local_pt']) n_r = len(Work['libE_info']['H_rows']) @@ -323,7 +323,7 @@ def update_history_dist(H, n): new_inds = np.where(~H['known_to_aposmm'])[0] - p = np.logical_and.reduce((H['sim_end'], ~np.isnan(H['f']))) + p = np.logical_and.reduce((H['sim_ended'], ~np.isnan(H['f']))) for new_ind in new_inds: # Loop over new returned points and update their distances @@ -491,7 +491,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): if nu > 0: test_2_through_5 = np.logical_and.reduce(( - H['sim_end'] == 1, # have a returned function value + H['sim_ended'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -503,7 +503,7 @@ def decide_where_to_start_localopt(H, n, n_s, rk_const, ld=0, mu=0, nu=0): )) else: test_2_through_5 = np.logical_and.reduce(( - H['sim_end'] == 1, # have a returned function value + H['sim_ended'] == 1, # have a returned function value H['dist_to_better_s'] > r_k, # no better sample point within r_k (L2) ~H['started_run'], # have not started a run (L3) @@ -605,7 +605,7 @@ def initialize_APOSMM(H, user_specs, libE_info): ('local_min', bool), ('sim_id', int), ('paused', bool), - ('sim_end', bool), + ('sim_ended', bool), ] if 'components' in user_specs: @@ -623,7 +623,7 @@ def initialize_APOSMM(H, user_specs, libE_info): assert 'f' in H.dtype.names, "Must give 'f' values to persistent_aposmm in gen_specs['in']" assert 'sim_id' in H.dtype.names, "Must give 'sim_id' to persistent_aposmm in gen_specs['in']" - assert 'sim_end' in H.dtype.names, "Must give 'sim_end' status to persistent_aposmm in gen_specs['in']" + assert 'sim_ended' in H.dtype.names, "Must give 'sim_ended' status to persistent_aposmm in gen_specs['in']" over_written_fields = ['dist_to_unit_bounds', 'dist_to_better_l', 'dist_to_better_s', 'ind_of_better_l', 'ind_of_better_s'] if any([i in H.dtype.names for i in over_written_fields]): diff --git a/libensemble/history.py b/libensemble/history.py index 15f0b0052..b8feb4687 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -31,10 +31,10 @@ class History: :ivar int sim_started_count: Number of points given to sim functions (according to H) - :ivar int sim_end_count: + :ivar int sim_ended_count: Number of points evaluated (according to H) - Note that index, sim_started_count and sim_end_count reflect the total number of points + Note that index, sim_started_count and sim_ended_count reflect the total number of points in H and therefore include those prepended to H in addition to the current run. """ @@ -64,12 +64,12 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): # H[field][ind] = val if 'sim_started' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'sim_started' and 'sim_end'") + logger.manager_warning("Marking entries in H0 as having been 'sim_started' and 'sim_ended'") H['sim_started'][:len(H0)] = 1 - H['sim_end'][:len(H0)] = 1 - elif 'sim_end' not in fields: - logger.manager_warning("Marking entries in H0 as having been 'sim_end' if 'sim_started'") - H['sim_end'][:len(H0)] = H0['sim_started'] + H['sim_ended'][:len(H0)] = 1 + elif 'sim_ended' not in fields: + logger.manager_warning("Marking entries in H0 as having been 'sim_ended' if 'sim_started'") + H['sim_ended'][:len(H0)] = H0['sim_started'] if 'sim_id' not in fields: logger.manager_warning("Assigning sim_ids to entries in H0") @@ -85,12 +85,12 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.grow_count = 0 self.sim_started_count = np.sum(H['sim_started']) - self.sim_end_count = np.sum(H['sim_end']) + self.sim_ended_count = np.sum(H['sim_ended']) self.gen_informed_count = np.sum(H['gen_informed']) self.given_back_warned = False self.sim_started_offset = self.sim_started_count - self.sim_end_offset = self.sim_end_count + self.sim_ended_offset = self.sim_ended_count self.gen_informed_offset = self.gen_informed_count def update_history_f(self, D, safe_mode): @@ -118,9 +118,9 @@ def update_history_f(self, D, safe_mode): else: self.H[field][ind][:H0_size] = returned_H[field][j] # Slice View - self.H['sim_end'][ind] = True - self.H['sim_end_time'][ind] = time.time() - self.sim_end_count += 1 + self.H['sim_ended'][ind] = True + self.H['sim_ended_time'][ind] = time.time() + self.sim_ended_count += 1 def update_history_x_out(self, q_inds, sim_worker): """ @@ -149,16 +149,16 @@ def update_history_to_gen(self, q_inds): t = time.time() if q_inds.size > 0: - if np.all(self.H['sim_end'][q_inds]): + if np.all(self.H['sim_ended'][q_inds]): self.H['gen_informed'][q_inds] = True - elif np.any(self.H['sim_end'][q_inds]): # sporadic returned points need updating - for ind in q_inds[self.H['sim_end'][q_inds]]: + elif np.any(self.H['sim_ended'][q_inds]): # sporadic returned points need updating + for ind in q_inds[self.H['sim_ended'][q_inds]]: self.H['gen_informed'][ind] = True if self.using_H0 and not self.given_back_warned: logger.manager_warning( - "Giving entries in H0 back to gen. Marking entries in H0 as 'gen_informed' if 'sim_end'.") + "Giving entries in H0 back to gen. Marking entries in H0 as 'gen_informed' if 'sim_ended'.") self.given_back_warned = True self.H['gen_informed_time'][q_inds] = t diff --git a/libensemble/libE.py b/libensemble/libE.py index f98fe156a..f3a200d9f 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -612,9 +612,9 @@ def _dump_on_abort(hist, persis_info, save_H=True): """Dump history and persis_info on abort""" logger.error("Manager exception raised .. aborting ensemble:") logger.error("Dumping ensemble history with {} sims evaluated:". - format(hist.sim_end_count)) + format(hist.sim_ended_count)) if save_H: - np.save('libE_history_at_abort_' + str(hist.sim_end_count) + '.npy', hist.trim_H()) - with open('libE_persis_info_at_abort_' + str(hist.sim_end_count) + '.pickle', "wb") as f: + np.save('libE_history_at_abort_' + str(hist.sim_ended_count) + '.npy', hist.trim_H()) + with open('libE_persis_info_at_abort_' + str(hist.sim_ended_count) + '.pickle', "wb") as f: pickle.dump(persis_info, f) diff --git a/libensemble/manager.py b/libensemble/manager.py index f47d76ede..6f8143cf7 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -204,7 +204,7 @@ def term_test_wallclock(self, max_elapsed): def term_test_sim_max(self, sim_max): """Checks against max simulations""" - return self.hist.sim_end_count >= sim_max + self.hist.sim_end_offset + return self.hist.sim_ended_count >= sim_max + self.hist.sim_ended_offset def term_test_gen_max(self, gen_max): """Checks against max generator calls""" @@ -214,7 +214,7 @@ def term_test_stop_val(self, stop_val): """Checks against stop value criterion""" key, val = stop_val H = self.hist.H - return np.any(filter_nans(H[key][H['sim_end']]) <= val) + return np.any(filter_nans(H[key][H['sim_ended']]) <= val) def term_test(self, logged=True): """Checks termination criteria""" @@ -247,7 +247,7 @@ def _save_every_k(self, fname, count, k): def _save_every_k_sims(self): """Saves history every kth sim step""" self._save_every_k('libE_history_for_run_starting_{}_after_sim_{}.npy', - self.hist.sim_end_count, + self.hist.sim_ended_count, self.libE_specs['save_every_k_sims']) def _save_every_k_gens(self): @@ -469,7 +469,7 @@ def _kill_cancelled_sims(self): """Send kill signals to any sims marked as cancel_requested""" if self.kill_canceled_sims: kill_sim = self.hist.H['sim_started'] & self.hist.H['cancel_requested'] \ - & ~self.hist.H['sim_end'] & ~self.hist.H['kill_sent'] + & ~self.hist.H['sim_ended'] & ~self.hist.H['kill_sent'] # Note that a return is still expected when running sims are killed if np.any(kill_sim): @@ -496,7 +496,7 @@ def _final_receive_and_kill(self, persis_info): for w in self.W['worker_id'][self.W['persis_state'] > 0]: logger.debug("Manager sending PERSIS_STOP to worker {}".format(w)) if 'final_fields' in self.libE_specs: - rows_to_send = self.hist.trim_H()['sim_end'] + rows_to_send = self.hist.trim_H()['sim_ended'] fields_to_send = self.libE_specs['final_fields'] H_to_send = self.hist.trim_H()[rows_to_send][fields_to_send] self.wcomms[w-1].send(PERSIS_STOP, H_to_send) @@ -540,7 +540,7 @@ def _get_alloc_libE_info(self): 'elapsed_time': self.elapsed(), 'manager_kill_canceled_sims': self.kill_canceled_sims, 'sim_started_count': self.hist.sim_started_count, - 'sim_end_count': self.hist.sim_end_count, + 'sim_ended_count': self.hist.sim_ended_count, 'gen_informed_count': self.hist.gen_informed_count, 'sim_max_given': self._sim_max_given(), 'use_resource_sets': 'num_resource_sets' in self.libE_specs} diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index 5bc29816e..837aaec3a 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -13,7 +13,7 @@ __all__ = ['executor_hworld'] # Alt send values through X -sim_end_count = 0 +sim_ended_count = 0 def custom_polling_loop(exctr, task, timeout_sec=5.0, delay=0.3): @@ -83,24 +83,24 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): timeout = 65.0 else: - global sim_end_count - sim_end_count += 1 + global sim_ended_count + sim_ended_count += 1 timeout = 6.0 launch_shc = False - print(sim_end_count) + print(sim_ended_count) - if sim_end_count == 1: + if sim_ended_count == 1: args_for_sim = 'sleep 1' # Should finish - elif sim_end_count == 2: + elif sim_ended_count == 2: args_for_sim = 'sleep 1 Error' # Worker kill on error - elif sim_end_count == 3: + elif sim_ended_count == 3: wait = True args_for_sim = 'sleep 1' # Should finish launch_shc = True - elif sim_end_count == 4: + elif sim_ended_count == 4: args_for_sim = 'sleep 8' # Worker kill on timeout timeout = 1.0 - elif sim_end_count == 5: + elif sim_ended_count == 5: args_for_sim = 'sleep 2 Fail' # Manager kill - if signal received else completes if USE_BALSAM: @@ -127,9 +127,9 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): else: if not ELAPSED_TIMEOUT: - if sim_end_count >= 2 and not USE_BALSAM: + if sim_ended_count >= 2 and not USE_BALSAM: calc_status = exctr.polling_loop(task, timeout=timeout, delay=0.3, poll_manager=True) - if sim_end_count == 2 and task.stdout_exists() and 'Error' in task.read_stdout(): + if sim_ended_count == 2 and task.stdout_exists() and 'Error' in task.read_stdout(): calc_status = WORKER_KILL_ON_ERR else: diff --git a/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py b/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py index 5c0e3aabc..953af2070 100644 --- a/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py +++ b/libensemble/tests/deprecated_tests/test_nan_func_old_aposmm.py @@ -32,7 +32,7 @@ gen_out += [('x', float, n), ('x_on_cube', float, n), ('obj_component', int)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'f_i', 'sim_end'], + 'in': [o[0] for o in gen_out] + ['f', 'f_i', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': -2*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py b/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py index ba23f43da..ab8dd37da 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py @@ -23,7 +23,7 @@ def test_failing_localopt_method(): hist, sim_specs_0, gen_specs_0, exit_criteria_0, alloc = setup.hist_setup1() - hist.H['sim_end'] = 1 + hist.H['sim_ended'] = 1 gen_specs_0['user']['localopt_method'] = 'BADNAME' @@ -37,7 +37,7 @@ def test_failing_localopt_method(): def test_exception_raising(): hist, sim_specs_0, gen_specs_0, exit_criteria_0, alloc = setup.hist_setup1(n=2) - hist.H['sim_end'] = 1 + hist.H['sim_ended'] = 1 for method in ['LN_SBPLX', 'pounders', 'scipy_COBYLA']: gen_specs_0['user']['localopt_method'] = method @@ -48,10 +48,10 @@ def test_exception_raising(): def test_decide_where_to_start_localopt(): - H = np.zeros(10, dtype=gen_out + [('f', float), ('sim_end', bool)]) + H = np.zeros(10, dtype=gen_out + [('f', float), ('sim_ended', bool)]) H['x'] = np.random.uniform(0, 1, (10, 2)) H['f'] = np.random.uniform(0, 1, 10) - H['sim_end'] = 1 + H['sim_ended'] = 1 b = al.decide_where_to_start_localopt(H, 9, 1) assert len(b) == 0 @@ -94,10 +94,10 @@ def test_declare_opt(): def test_localopt_error_saving(): _, sim_specs_0, gen_specs_0, _, _ = setup.hist_setup1() - H = np.zeros(4, dtype=gen_out + [('f', float), ('fvec', float, 2), ('sim_end', bool)]) + H = np.zeros(4, dtype=gen_out + [('f', float), ('fvec', float, 2), ('sim_ended', bool)]) H['x'] = np.random.uniform(0, 1, (4, 2)) H['f'] = np.random.uniform(0, 1, 4) - H['sim_end'] = True + H['sim_ended'] = True H['local_pt'][1:] = True gen_specs_0['user']['initial_sample_size'] = 1 gen_specs_0['user']['localopt_method'] = 'scipy_COBYLA' diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py b/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py index 048404572..5beca24a8 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py @@ -44,7 +44,7 @@ UB = 2*np.ones(n) LB = (-2-np.pi/10)*np.ones(n) gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f_i', 'sim_end'], + 'in': [o[0] for o in gen_out] + ['f_i', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': LB, diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py index 009ba2b08..7185dbb03 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py @@ -43,7 +43,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py index 5381382a8..e21258b3f 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py @@ -48,7 +48,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py index dbf6489b9..6e2aff583 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py @@ -52,7 +52,7 @@ # lb tries to avoid x[1]=-x[2], which results in division by zero in chwirut. gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_end'], + 'in': [o[0] for o in gen_out]+['f', 'fvec', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 5, 'lb': (-2-np.pi/10)*np.ones(n), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py b/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py index 32fdc50d7..a08e32185 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py @@ -47,7 +47,7 @@ n = 2 gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'sim_end'], + 'in': [o[0] for o in gen_out] + ['f', 'sim_ended'], 'out': gen_out, 'user': {'lb': np.array([-5, 0]), 'ub': np.array([10, 15]), diff --git a/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py b/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py index ef90953ce..4767c0f1b 100644 --- a/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py +++ b/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py @@ -40,7 +40,7 @@ gen_out += [('x', float, n), ('x_on_cube', float, n)] gen_specs = {'gen_f': gen_f, - 'in': [o[0] for o in gen_out] + ['f', 'grad', 'sim_end'], + 'in': [o[0] for o in gen_out] + ['f', 'grad', 'sim_ended'], 'out': gen_out, 'user': {'initial_sample_size': 100, 'sample_points': np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py index 3276611e1..c29bd22b8 100644 --- a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py @@ -52,6 +52,6 @@ if is_manager: assert len(H) == len(H0) assert np.array_equal(H0['x'], H['x']) - assert np.all(H['sim_end']) + assert np.all(H['sim_ended']) print("\nlibEnsemble correctly didn't add anything to initial sample") save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 3effbc9ce..5eecf2964 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -35,7 +35,7 @@ n_samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_started', bool), ('sim_end', bool)]) +H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_started', bool), ('sim_ended', bool)]) np.random.seed(0) H0['x'] = gen_borehole_input(n_samp) @@ -44,7 +44,7 @@ H0['f'][i] = borehole_func(H0['x'][i]) H0['sim_started'][:500] = True -H0['sim_end'][:500] = True +H0['sim_ended'][:500] = True alloc_specs = {'alloc_f': alloc_f, 'out': [('x', float, n)]} @@ -56,6 +56,6 @@ if is_manager: assert len(H) == len(H0) assert np.array_equal(H0['x'], H['x']) - assert np.all(H['sim_end']) + assert np.all(H['sim_ended']) print("\nlibEnsemble correctly didn't add anything to initial sample") save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py b/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py index 863950466..8c94dd9d7 100644 --- a/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py +++ b/libensemble/tests/regression_tests/test_executor_hworld_pass_fail.py @@ -81,7 +81,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) -# num sim_end_count conditions in executor_hworld +# num sim_ended_count conditions in executor_hworld exit_criteria = {'sim_max': nworkers * 5} # Perform the run diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py index e96f7bb0d..f7e97a8db 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py @@ -94,7 +94,7 @@ if is_manager: assert persis_info[1].get('run_order'), "Run_order should have been given back" assert flag == 0 - assert np.min(H['f'][H['sim_end']]) <= 3000, "Didn't find a value below 3000" + assert np.min(H['f'][H['sim_ended']]) <= 3000, "Didn't find a value below 3000" save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py index fad84cb5e..a26c11f0c 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py @@ -124,4 +124,4 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert np.sum(H['sim_end']) >= exit_criteria['sim_max'], "Run didn't finish" + assert np.sum(H['sim_ended']) >= exit_criteria['sim_max'], "Run didn't finish" diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index af54f4b9c..9b2d17441 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -58,7 +58,7 @@ ('local_pt', bool), ] -gen_in = ['x', 'f', 'grad', 'local_pt', 'sim_id', 'sim_end', 'x_on_cube', 'local_min'] +gen_in = ['x', 'f', 'grad', 'local_pt', 'sim_id', 'sim_ended', 'x_on_cube', 'local_min'] gen_specs = { 'gen_f': gen_f, @@ -92,7 +92,7 @@ ('grad', float, n), ('sim_id', int), ('x_on_cube', float, n), - ('sim_end', bool), + ('sim_ended', bool), ('f', float), ('gen_informed', bool), ('sim_started', bool), @@ -104,7 +104,7 @@ H0['x'] = np.round(minima, 1) H0['x_on_cube'] = (H0['x'] - gen_specs['user']['lb']) / (gen_specs['user']['ub'] - gen_specs['user']['lb']) H0['sim_id'] = range(sample_size) -H0[['sim_started', 'gen_informed', 'sim_end']] = True +H0[['sim_started', 'gen_informed', 'sim_ended']] = True for i in range(sample_size): H0['f'][i] = six_hump_camel_func(H0['x'][i]) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py index 330cb1a94..66f529254 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -77,7 +77,7 @@ gen_specs = { 'gen_f': gen_f, - 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_end', 'sim_id'], + 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_ended', 'sim_id'], 'out': gen_out, 'user': { 'n_init_thetas': n_init_thetas, # Num thetas in initial batch @@ -114,6 +114,6 @@ if is_manager: print('Cancelled sims', H['sim_id'][H['cancel_requested']]) - sims_done = np.count_nonzero(H['sim_end']) + sims_done = np.count_nonzero(H['sim_ended']) save_libE_output(H, persis_info, __file__, nworkers) assert sims_done == max_evals, 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py index 48ae5b25a..b4c752206 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -97,7 +97,7 @@ gen_specs = { 'gen_f': gen_f, - 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_end', 'sim_id'], + 'persis_in': [o[0] for o in gen_out] + ['f', 'sim_ended', 'sim_id'], 'out': gen_out, 'user': { 'n_init_thetas': n_init_thetas, # Num thetas in initial batch @@ -131,6 +131,6 @@ if is_manager: print('Cancelled sims', H['sim_id'][H['cancel_requested']]) print('Killed sims', H['sim_id'][H['kill_sent']]) - sims_done = np.count_nonzero(H['sim_end']) + sims_done = np.count_nonzero(H['sim_ended']) save_libE_output(H, persis_info, __file__, nworkers) assert sims_done == max_evals, 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/scaling_tests/warpx/plot_results.py b/libensemble/tests/scaling_tests/warpx/plot_results.py index 753f7abc7..7b5a2b69b 100755 --- a/libensemble/tests/scaling_tests/warpx/plot_results.py +++ b/libensemble/tests/scaling_tests/warpx/plot_results.py @@ -34,7 +34,7 @@ nbatches = len(pickle_data[1]['run_order']) # Remove un-returned (i.e., not submitted) simulations -results = results[results['sim_end'] == 1] +results = results[results['sim_ended'] == 1] # Re-organize results into a dictionary. Each key is a field in libE_output, # each value is an array with the value of this field for all WarpX simulations diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 826ba64ff..27a43bf95 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -23,8 +23,8 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-2.11, -1.63], 0., 2, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], - dtype=[('sim_started', '?'), ('gen_informed', '?'), ('gen_worker', '= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" + assert np.sum(H['sim_ended']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" assert persis_info.get('run_order'), "Standalone persistent_aposmm didn't do any localopt runs" tol = 1e-3 @@ -120,7 +120,7 @@ def combined_func(x): H, persis_info, exit_code = al.aposmm(H, persis_info, gen_specs, libE_info) assert exit_code == FINISHED_PERSISTENT_GEN_TAG, "Standalone persistent_aposmm didn't exit correctly" - assert np.sum(H['sim_end']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" + assert np.sum(H['sim_ended']) >= eval_max, "Standalone persistent_aposmm, didn't evaluate enough points" assert persis_info.get('run_order'), "Standalone persistent_aposmm didn't do any localopt runs" diff --git a/libensemble/tools/alloc_support.py b/libensemble/tools/alloc_support.py index 4db59b6ba..328e48cd7 100644 --- a/libensemble/tools/alloc_support.py +++ b/libensemble/tools/alloc_support.py @@ -250,7 +250,7 @@ def all_sim_started(self, H, pt_filter=None, low_bound=None): excluded_points = H['cancel_requested'] return np.all(H['sim_started'][pfilter & ~excluded_points]) - def all_sim_end(self, H, pt_filter=None, low_bound=None): + def all_sim_ended(self, H, pt_filter=None, low_bound=None): """Returns ``True`` if all expected points have had their sim_end Excludes cancelled points that were not already sim_started. @@ -261,7 +261,7 @@ def all_sim_end(self, H, pt_filter=None, low_bound=None): """ H, pfilter = self._filter_points(H, pt_filter, low_bound) excluded_points = H['cancel_requested'] & ~H['sim_started'] - return np.all(H['sim_end'][pfilter & ~excluded_points]) + return np.all(H['sim_ended'][pfilter & ~excluded_points]) def all_gen_informed(self, H, pt_filter=None, low_bound=None): """Returns ``True`` if gen has been informed of all expected points diff --git a/libensemble/tools/check_inputs.py b/libensemble/tools/check_inputs.py index dabd8bd7f..5c6c8861d 100644 --- a/libensemble/tools/check_inputs.py +++ b/libensemble/tools/check_inputs.py @@ -140,11 +140,11 @@ def check_H(H0, sim_specs, alloc_specs, gen_specs): format(set(fields).difference(set(Dummy_H.dtype.names))) # Prior history cannot contain unreturned points - # assert 'sim_end' not in fields or np.all(H0['sim_end']), \ + # assert 'sim_ended' not in fields or np.all(H0['sim_ended']), \ # "H0 contains unreturned points." # Fail if prior history contains unreturned points (or returned but not given). - assert('sim_end' not in fields or np.all(H0['sim_started'] == H0['sim_end'])), \ + assert('sim_ended' not in fields or np.all(H0['sim_started'] == H0['sim_ended'])), \ 'H0 contains unreturned or invalid points' # # Fail if points in prior history don't have a sim_id. diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index a43bfc68c..c1fa9918b 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -11,7 +11,7 @@ ('sim_started', bool), # True if entry was given to sim_worker for sim eval ('sim_started_time', float), # Time entry was given to sim_worker for a sim eval ('sim_ended', bool), # True if entry's sim eval completed - ('sim_end_time', float), # Time entry's sim eval completed + ('sim_ended_time', float), # Time entry's sim eval completed ('gen_informed', bool), # True if gen_worker was informed about the sim eval of this entry ('gen_informed_time', float), # Time gen_worker was informed about the sim eval of this entry ('cancel_requested', bool), # True if cancellation was requested for this entry @@ -27,7 +27,7 @@ 'sim_started', 'sim_started_time', 'sim_ended', - 'sim_end_time', + 'sim_ended_time', 'gen_informed', 'gen_informed_time', 'kill_sent'] diff --git a/libensemble/tools/tools.py b/libensemble/tools/tools.py index 39177aa17..216d68abb 100644 --- a/libensemble/tools/tools.py +++ b/libensemble/tools/tools.py @@ -103,7 +103,7 @@ def save_libE_output(H, persis_info, calling_file, nworkers, mess='Run completed script_name = os.path.splitext(os.path.basename(calling_file))[0] short_name = script_name.split("test_", 1).pop() prob_str = 'length=' + str(len(H)) \ - + '_evals=' + str(sum(H['sim_end'])) \ + + '_evals=' + str(sum(H['sim_ended'])) \ + '_workers=' + str(nworkers) h_filename = short_name + '_history_' + prob_str diff --git a/postproc_scripts/print_npy.py b/postproc_scripts/print_npy.py index 0caaca5b2..b826d3c70 100755 --- a/postproc_scripts/print_npy.py +++ b/postproc_scripts/print_npy.py @@ -23,13 +23,13 @@ if done_only: count = 0 for line in results: - if line['sim_end']: + if line['sim_ended']: count += 1 results_filtered = np.zeros(count, dtype=results.dtype) count = 0 for i, line in enumerate(results): - if line['sim_end']: + if line['sim_ended']: results_filtered[count] = results[i] count += 1 diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index c2f88c3d4..765950958 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -25,7 +25,7 @@ Results analysis scripts * ``print_npy.py``: Prints to screen from a given ``*.npy`` file containing a NumPy structured array. Use ``done`` to print only the lines containing - ``'sim_end'`` points. Example:: + ``'sim_ended'`` points. Example:: ./print_npy.py run_libe_forces_results_History_length=1000_evals=8.npy done From 35a7cdbedf0019e51d7a2ba7354dbc76a1367ada Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 13:04:42 -0500 Subject: [PATCH 30/33] gen_start/end_time to gen_started/ended_time --- docs/data_structures/history_array.rst | 4 ++-- docs/tutorials/local_sine_tutorial.rst | 2 +- libensemble/history.py | 8 ++++---- libensemble/manager.py | 6 +++--- .../test_persistent_sim_uniform_sampling.py | 2 +- .../test_persistent_uniform_gen_decides_stop.py | 8 ++++---- .../regression_tests/test_persistent_uniform_sampling.py | 2 +- .../test_persistent_uniform_sampling_adv.py | 2 +- .../test_persistent_uniform_sampling_async.py | 6 +++--- .../tests/unit_tests/test_allocation_funcs_and_support.py | 2 +- libensemble/tests/unit_tests/test_history.py | 8 ++++---- libensemble/tools/fields_keys.py | 8 ++++---- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 6d436a090..9cbba3d21 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -85,9 +85,9 @@ The full list of these reserved fields is given below. ``gen_worker`` [int]: Worker that generated this entry -``gen_start_time`` [ float]: Time gen_worker was initiated that produced this entry +``gen_started_time`` [ float]: Time gen_worker was initiated that produced this entry -``gen_end_time`` [ float]: Time gen_worker requested this entry +``gen_ended_time`` [ float]: Time gen_worker requested this entry ``sim_worker`` [int]: Worker that did (or is doing) the sim evaluation for this entry diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 789434f0e..c8820feda 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -292,7 +292,7 @@ columns might be rearranged). .. code-block:: - ['y', 'sim_started_time', 'gen_worker', 'sim_worker', 'sim_started', 'sim_ended', 'x', 'allocated', 'sim_id', 'gen_end_time'] + ['y', 'sim_started_time', 'gen_worker', 'sim_worker', 'sim_started', 'sim_ended', 'x', 'allocated', 'sim_id', 'gen_ended_time'] [(-0.37466051, 1.559+09, 2, 2, True, True, [-0.38403059], True, 0, 1.559+09) (-0.29279634, 1.559+09, 2, 3, True, True, [-2.84444261], True, 1, 1.559+09) ( 0.29358492, 1.559+09, 2, 4, True, True, [ 0.29797487], True, 2, 1.559+09) diff --git a/libensemble/history.py b/libensemble/history.py index b8feb4687..3bca979d1 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -164,7 +164,7 @@ def update_history_to_gen(self, q_inds): self.H['gen_informed_time'][q_inds] = t self.gen_informed_count += len(q_inds) - def update_history_x_in(self, gen_worker, D, safe_mode, gen_start_time): + def update_history_x_in(self, gen_worker, D, safe_mode, gen_started_time): """ Updates the history (in place) when new points have been returned from a gen @@ -212,9 +212,9 @@ def update_history_x_in(self, gen_worker, D, safe_mode, gen_start_time): assert field not in protected_libE_fields, "The field '" + field + "' is protected" self.H[field][update_inds] = D[field] - first_gen_inds = update_inds[self.H['gen_end_time'][update_inds] == 0] - self.H['gen_start_time'][first_gen_inds] = gen_start_time - self.H['gen_end_time'][first_gen_inds] = t + first_gen_inds = update_inds[self.H['gen_ended_time'][update_inds] == 0] + self.H['gen_started_time'][first_gen_inds] = gen_started_time + self.H['gen_ended_time'][first_gen_inds] = t self.H['gen_worker'][first_gen_inds] = gen_worker self.index += num_new diff --git a/libensemble/manager.py b/libensemble/manager.py index 6f8143cf7..44245f4b1 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -151,7 +151,7 @@ class Manager: ('active', int), ('persis_state', int), ('active_recv', int), - ('gen_start_time', float), + ('gen_started_time', float), ('zero_resource_worker', bool)] def __init__(self, hist, libE_specs, alloc_specs, @@ -317,7 +317,7 @@ def _send_work_order(self, Work, w): self.wcomms[w-1].send(Work['tag'], Work) if Work['tag'] == EVAL_GEN_TAG: - self.W[w]['gen_start_time'] = time.time() + self.W[w]['gen_started_time'] = time.time() work_rows = Work['libE_info']['H_rows'] work_name = calc_type_strings[Work['tag']] @@ -429,7 +429,7 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv, self.safe_mode) if calc_type == EVAL_GEN_TAG: - self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode, self.W[w]['gen_start_time']) + self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode, self.W[w]['gen_started_time']) assert len(D_recv['calc_out']) or np.any(self.W['active']) or self.W[w-1]['persis_state'], \ "Gen must return work when is is the only thing active and not persistent." if 'libE_info' in D_recv and 'persistent' in D_recv['libE_info']: diff --git a/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py index a8d6743b7..6eab0ba27 100644 --- a/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_persistent_sim_uniform_sampling.py @@ -69,7 +69,7 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_end_time'])) == 8 + assert len(np.unique(H['gen_ended_time'])) == 8 assert not any((H['f'] == 0)) # Should overwrite the last value (in fact last (nworker-1) values) with f(1,1) = 3.23333333 assert not np.isclose(H['f'][0], 3.23333333) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py index ad9e63290..167874b22 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_gen_decides_stop.py @@ -71,11 +71,11 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - [_, counts] = np.unique(H['gen_end_time'], return_counts=True) + [_, counts] = np.unique(H['gen_ended_time'], return_counts=True) print('Num. points in each gen iteration:', counts) - assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" - assert counts[1] == nworkers, "The second gen_end_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "All gen_end_times are the same; they should be different for the async case" + assert counts[0] == nworkers, "The first gen_ended_time should be common among initial_batch_size number of points" + assert counts[1] == nworkers, "The second gen_ended_time should be common among initial_batch_size number of points" + assert len(np.unique(counts)) > 1, "All gen_ended_times are the same; they should be different for the async case" gen_workers = np.unique(H['gen_worker']) print('Generators that issued points', gen_workers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py index 0b252baa6..e3fcccaed 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling.py @@ -62,6 +62,6 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_end_time'])) == 2 + assert len(np.unique(H['gen_ended_time'])) == 2 save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py index 8794d2501..1277240e5 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py @@ -70,7 +70,7 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - assert len(np.unique(H['gen_end_time'])) == 1, "Everything should have been generated in one batch" + assert len(np.unique(H['gen_ended_time'])) == 1, "Everything should have been generated in one batch" if prob_id == 1: assert np.all(H['x'][0:sim_max] == -1.23), "The persistent gen should have set these at shutdown" diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py index 1c2ea2811..526c19426 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py @@ -64,9 +64,9 @@ H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) if is_manager: - [_, counts] = np.unique(H['gen_end_time'], return_counts=True) + [_, counts] = np.unique(H['gen_ended_time'], return_counts=True) print('Num. points in each gen iteration:', counts) - assert counts[0] == nworkers, "The first gen_end_time should be common among initial_batch_size number of points" - assert len(np.unique(counts)) > 1, "All gen_end_times are the same; they should be different for the async case" + assert counts[0] == nworkers, "The first gen_ended_time should be common among initial_batch_size number of points" + assert len(np.unique(counts)) > 1, "All gen_ended_times are the same; they should be different for the async case" save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py index 27a43bf95..7ee215021 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs_and_support.py @@ -24,7 +24,7 @@ (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-1.88, -0.61], 0., 3, False, False, [0., 0.], np.inf), (False, False, 1, 0., 1., False, 1.6e09, 0, 1, False, [-0.61, 0.15], 0., 4, False, False, [0., 0.], np.inf)], dtype=[('sim_started', '?'), ('gen_informed', '?'), ('gen_worker', ' Date: Mon, 4 Apr 2022 13:21:30 -0500 Subject: [PATCH 31/33] black and dimension of 'f' in test --- .../tests/regression_tests/test_evaluate_mixed_sample.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 5eecf2964..31a7b1e41 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -27,18 +27,18 @@ sim_specs = { 'sim_f': sim_f, 'in': ['x'], - 'out': [('f', float, 8)], + 'out': [('f', float)], } gen_specs = {} -n_samp = 1000 +samp = 1000 n = 8 -H0 = np.zeros(n_samp, dtype=[('x', float, 8), ('f', float, 8), ('sim_id', int), ('sim_started', bool), ('sim_ended', bool)]) +H0 = np.zeros(samp, dtype=[('x', float, n), ('f', float), ('sim_id', int), ('sim_started', bool), ('sim_ended', bool)]) np.random.seed(0) -H0['x'] = gen_borehole_input(n_samp) +H0['x'] = gen_borehole_input(samp) for i in range(500): H0['f'][i] = borehole_func(H0['x'][i]) From 8fdf8428a138cb07b25772fd4d912703edbfd586 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 14:49:12 -0500 Subject: [PATCH 32/33] Correct indexing of W with minus 1 --- libensemble/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libensemble/manager.py b/libensemble/manager.py index 44245f4b1..684d72e08 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -317,7 +317,7 @@ def _send_work_order(self, Work, w): self.wcomms[w-1].send(Work['tag'], Work) if Work['tag'] == EVAL_GEN_TAG: - self.W[w]['gen_started_time'] = time.time() + self.W[w-1]['gen_started_time'] = time.time() work_rows = Work['libE_info']['H_rows'] work_name = calc_type_strings[Work['tag']] @@ -429,7 +429,7 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): if calc_type == EVAL_SIM_TAG: self.hist.update_history_f(D_recv, self.safe_mode) if calc_type == EVAL_GEN_TAG: - self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode, self.W[w]['gen_started_time']) + self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode, self.W[w-1]['gen_started_time']) assert len(D_recv['calc_out']) or np.any(self.W['active']) or self.W[w-1]['persis_state'], \ "Gen must return work when is is the only thing active and not persistent." if 'libE_info' in D_recv and 'persistent' in D_recv['libE_info']: From 21efe93554521ee15a06ad43225c566f5812c571 Mon Sep 17 00:00:00 2001 From: Jeffrey Larson Date: Mon, 4 Apr 2022 15:21:37 -0500 Subject: [PATCH 33/33] Passing gen_started_time even on final kill --- .github/workflows/ci.yml | 10 +++++----- libensemble/manager.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 723e8afb0..8393c0ce0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,11 +32,11 @@ jobs: python-version: 3.8 mpi-version: 'mpich' comms-type: t - - os: ubuntu-latest - do-balsam: false - mpi-version: 'openmpi' - python-version: 3.8 - comms-type: l + #- os: ubuntu-latest + # do-balsam: false + # mpi-version: 'openmpi' + # python-version: 3.8 + # comms-type: l - os: ubuntu-latest python-version: 3.8 comms-type: l diff --git a/libensemble/manager.py b/libensemble/manager.py index 684d72e08..274d69288 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -412,7 +412,7 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): final_data = D_recv.get('calc_out', None) if isinstance(final_data, np.ndarray): if calc_status is FINISHED_PERSISTENT_GEN_TAG and self.libE_specs.get('use_persis_return_gen', False): - self.hist.update_history_x_in(w, final_data, self.safe_mode) + self.hist.update_history_x_in(w, final_data, self.safe_mode, self.W[w-1]['gen_started_time']) elif calc_status is FINISHED_PERSISTENT_SIM_TAG and self.libE_specs.get('use_persis_return_sim', False): self.hist.update_history_f(D_recv, self.safe_mode) else: