Updating regression and unit tests after API changes from persistent

worker example

code/examples/alloc_funcs/give_sim_work_first.py

@@ -7,30 +7,29 @@
 from message_numbers import EVAL_SIM_TAG 
 from message_numbers import EVAL_GEN_TAG 
 
-def give_sim_work_first(active_w, idle_w, H, H_ind, sim_specs, gen_specs, term_test, gen_info):
+def give_sim_work_first(active_w, idle_w, persis_w, H, H_ind, sim_specs, gen_specs, gen_info):
     """ Decide what should be given to workers. Note that everything put into
     the Work dictionary will be given, so we are careful not to put more gen or
     sim items into Work than necessary.
     """
 
     Work = {}
     gen_count = 0
+    already_in_Work = np.zeros(H_ind,dtype=bool) # To mark points as they are included in Work, but not yet marked as 'given' in H.
 
     if len(gen_info) == 0: 
         gen_info[0] = {}
         gen_info[0]['rand_stream'] = {i:np.random.RandomState(i) for i in idle_w}
 
     for i in idle_w:
-        if term_test(H, H_ind):
-            break
 
         # Only consider giving to worker i if it's resources are not blocked by some other calculation
         blocked_set = active_w['blocked'].union(*[j['libE_info']['blocking'] for j in Work.values() if 'blocking' in j['libE_info']])
         if i in blocked_set:
             continue
 
         # Find indices of H where that are not given nor paused
-        q_inds_logical = np.logical_and(~H['given'][:H_ind],~H['paused'][:H_ind])
+        q_inds_logical = np.logical_and.reduce((~H['given'][:H_ind],~H['paused'][:H_ind],~already_in_Work))
 
         if np.any(q_inds_logical):
             # Give sim work if possible
@@ -64,6 +63,7 @@ def give_sim_work_first(active_w, idle_w, H, H_ind, sim_specs, gen_specs, term_t
                        'libE_info': {'H_rows': sim_ids_to_send,
                                 },
                       }
+            already_in_Work[sim_ids_to_send] = True
             print('This is being given to be evaluated!', sim_ids_to_send)
 
             if block_others:
@@ -93,5 +93,5 @@ def give_sim_work_first(active_w, idle_w, H, H_ind, sim_specs, gen_specs, term_t
                                 }
                        }
 
-    return Work, gen_info
+    return Work, persis_w, gen_info
 

code/examples/gen_funcs/aposmm_logic.py

@@ -208,14 +208,14 @@ def update_history_dist(H, gen_specs, c_flag):
     else:
         p = np.logical_and.reduce((H['returned'],~np.isnan(H['f'])))
 
-    H['known_to_aposmm'][new_inds] = True # These points are now known to APOSMM
 
     for new_ind in new_inds:
-        H['dist_to_unit_bounds'][new_ind] = min(min(np.ones(n) - H['x_on_cube'][new_ind]),min(H['x_on_cube'][new_ind] - np.zeros(n)))
-
         # Loop over new returned points and update their distances
         if p[new_ind]:
+            H['known_to_aposmm'][new_ind] = True # These points are now known to APOSMM
+
             # Compute distance to boundary
+            H['dist_to_unit_bounds'][new_ind] = min(min(np.ones(n) - H['x_on_cube'][new_ind]),min(H['x_on_cube'][new_ind] - np.zeros(n)))
 
             dist_to_all = sp.spatial.distance.cdist(np.atleast_2d(H['x_on_cube'][new_ind]), H['x_on_cube'][p], 'euclidean').flatten()
             new_better_than = H['f'][new_ind] < H['f'][p]

code/src/libE_manager.py

@@ -117,7 +117,17 @@ def receive_from_sim_and_gen(comm, active_w, idle_w, persis_w, H, H_ind, sim_spe
 
                 D_recv = comm.recv(source=w, tag=MPI.ANY_TAG, status=status)
                 recv_tag = status.Get_tag()
-                assert recv_tag in [EVAL_SIM_TAG, EVAL_GEN_TAG, FINISHED_PERSISTENT_SIM_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_SIM_TAG, PERSIS_GEN_TAG], 'Unknown calculation tag received. Exiting'
+                assert recv_tag in [EVAL_SIM_TAG, EVAL_GEN_TAG, FINISHED_PERSISTENT_SIM_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_SIM_TAG, PERSIS_GEN_TAG, PERSIS_STOP], 'Unknown calculation tag received. Exiting'
+
+                if recv_tag == PERSIS_STOP:
+                    idle_w.add(w)
+                    if w in persis_w[PERSIS_SIM_TAG]:
+                        active_w[EVAL_SIM_TAG].remove(w)
+                        persis_w[PERSIS_SIM_TAG].remove(w)
+                    else:
+                        persis_w[PERSIS_GEN_TAG].remove(w)
+                        active_w[EVAL_GEN_TAG].remove(w)
+
 
                 if recv_tag == EVAL_SIM_TAG:
                     idle_w.add(w)

code/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py

@@ -78,11 +78,14 @@
       ]
 
 # Tell libEnsemble when to stop
-exit_criteria = {'sim_max': 100}
+exit_criteria = {'sim_max': 1000}
 
 np.random.seed(1)
 
 alloc_specs = {'out':gen_out, 'alloc_f':start_persistent_local_opt_gens, 'manager_ranks': set([0]), 'worker_ranks': set(range(1,MPI.COMM_WORLD.Get_size()))}
+# Don't do a "persistent worker run" if only one wokrer
+if len(alloc_specs['worker_ranks'])==1:
+    quit()
 # Perform the run
 H, gen_info, flag = libE(sim_specs, gen_specs, exit_criteria, alloc_specs=alloc_specs)
 

code/tests/unit_tests/test_allocation_funcs.py

@@ -9,18 +9,19 @@
 from test_manager_main import make_criteria_and_specs_1
 from give_sim_work_first import give_sim_work_first
 
-al = {'alloc_f': give_sim_work_first, 'worker_ranks':set([1,2]),'persist_gen_ranks':set([])}
+al = {'alloc_f': give_sim_work_first, 'worker_ranks':set([1,2]),'persist_gen_ranks':set([]),'out':[]}
 
 def test_decide_work_and_resources():
 
     sim_specs, gen_specs, exit_criteria = make_criteria_and_specs_1()
-    H, Hs_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
+
+    H, H_ind, _, _, _, persis_w = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
 
 
     # Don't give out work when idle is empty
     active_w = set([1,2,3,4])
     idle_w = set()
-    Work, gen_info = al['alloc_f'](active_w, idle_w, H, Hs_ind, sim_specs, gen_specs, term_test, {})
+    Work, _, gen_info = al['alloc_f'](active_w, idle_w, persis_w, H, H_ind, sim_specs, gen_specs, {})
     assert len(Work) == 0 
     # 
 

code/tests/unit_tests/test_aposmm_logic.py

@@ -9,11 +9,11 @@
 
 from test_manager_main import make_criteria_and_specs_0
 
-alloc = {'worker_ranks':set([1,2]),'persist_gen_ranks':set([])}
+alloc = {'worker_ranks':set([1,2]),'persist_gen_ranks':set([]),'out':[]}
 
 def test_failing_localopt_method():
     sim_specs_0, gen_specs_0, exit_criteria_0 = make_criteria_and_specs_0()
-    H, _, _, _, _ = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[]) 
+    H = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[])[0]
     H['returned'] = 1
 
     gen_specs_0['localopt_method'] = 'BADNAME'
@@ -28,7 +28,7 @@ def test_failing_localopt_method():
 
 def test_exception_raising():
     sim_specs_0, gen_specs_0, exit_criteria_0 = make_criteria_and_specs_0()
-    H, _, _, _, _ = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[]) 
+    H = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[])[0]
     H['returned'] = 1
 
     for method in ['LN_SBPLX','pounders']:
@@ -66,7 +66,7 @@ def test_calc_rk():
 
 def test_initialize_APOSMM():
     sim_specs_0, gen_specs_0, exit_criteria_0 = make_criteria_and_specs_0()
-    H, _, _, _, _ = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[]) 
+    H = man.initialize(sim_specs_0, gen_specs_0, alloc, exit_criteria_0,[])[0]
 
     al.initialize_APOSMM(H,gen_specs_0)
 

code/tests/unit_tests/test_manager_main.py

@@ -7,7 +7,7 @@
 
 import libE_manager as man
 
-al = {'worker_ranks':set([1,2]),'persist_gen_ranks':set([])}
+al = {'worker_ranks':set([1,2]),'persist_gen_ranks':set([]),'out':[]}
 
 def test_update_history_x_out():
     assert True
@@ -31,36 +31,36 @@ def test_termination_test():
     # termination_test should be True when we want to stop
 
     sim_specs_0, gen_specs_0, exit_criteria_0 = make_criteria_and_specs_0()
-    H, H_ind, term_test,_,_ = man.initialize(sim_specs_0, gen_specs_0, al, exit_criteria_0,[]) 
+    H, H_ind, term_test,_,_,_ = man.initialize(sim_specs_0, gen_specs_0, al, exit_criteria_0,[]) 
     assert not term_test(H, H_ind)
 
 
 
     # Shouldn't terminate
     sim_specs, gen_specs, exit_criteria = make_criteria_and_specs_1()
-    H, H_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
+    H, H_ind,term_test,_,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
     assert not term_test(H, H_ind)
     # 
 
 
     # Terminate because we've found a good 'g' value
-    H, H_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
+    H, H_ind,term_test,_,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
     H['g'][0] = -1
     H_ind = 1
     assert term_test(H, H_ind)
     # 
 
 
     # Terminate because everything has been given.
-    H, H_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
+    H, H_ind,term_test,_,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
     H['given'] = np.ones
     assert term_test(H, H_ind)
     # 
 
 
     # Terminate because enough time has passed
     H0 = np.zeros(3,dtype=sim_specs['out'] + gen_specs['out'])
-    H, H_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,H0) 
+    H, H_ind,term_test,_,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,H0) 
     H_ind = 4
     H['given_time'][0] = time.time()
     time.sleep(0.5)
@@ -72,7 +72,7 @@ def test_update_history_x_in():
 
     # Don't take more points than there is space in history.
     sim_specs, gen_specs, exit_criteria = make_criteria_and_specs_1()
-    H, H_ind,term_test,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
+    H, H_ind,term_test,_,_,_ = man.initialize(sim_specs, gen_specs, al, exit_criteria,[]) 
 
     O = np.zeros(2*len(H), dtype=gen_specs['out'])
     print(len(O))