Merge pull request #640 from Libensemble/feature/balsam_dry_run_test

Balsam dry_run addition to regression test

.github/workflows/libE-ci.yml

@@ -85,7 +85,7 @@ jobs:
             pip install deap
             python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git@development/PCGPwM
             pip install flake8
-            pip install coverage
+            pip install coverage==4.5.4
             pip install pytest
             pip install pytest-cov
             pip install pytest-timeout
@@ -178,7 +178,7 @@ jobs:
           run: |
             python -m pip install --upgrade pip
             pip install flake8
-            pip install coverage
+            pip install coverage==4.5.4
             pip install pytest
             pip install pytest-cov
             pip install pytest-timeout
@@ -212,6 +212,13 @@ jobs:
            source balsamactivate test-balsam
            ./libensemble/tests/run-tests.sh -r -y 'test_balsam*' -z -l
 
+        - name: Merge coverage, run Coveralls
+          env:
+              COVERALLS_PARALLEL: true
+          run: |
+            mv libensemble/tests/.cov* .
+            coveralls --service=github
+
     coveralls:
         name: Notify coveralls of all jobs completing
         needs: [test-libE, test-libE-with-Balsam]

install/configure_balsam_install.py

@@ -32,8 +32,10 @@ def configure_coverage():
 
     newlines = [i for i in lines if i != '    */balsam_executor.py\n']
 
+    print('New libensemble/tests/.coveragerc: \n')
     with open(coveragerc, 'w') as f:
         for line in newlines:
+            print(line)
             f.write(line)
 
 

libensemble/sim_funcs/executor_hworld.py

@@ -3,6 +3,7 @@
                                          MAN_SIGNAL_FINISH, WORKER_DONE,
                                          TASK_FAILED, WORKER_KILL_ON_TIMEOUT)
 import numpy as np
+import os
 
 __all__ = ['executor_hworld']
 
@@ -127,6 +128,13 @@ def executor_hworld(H, persis_info, sim_specs, libE_info):
         except ValueError:
             pass
 
+        task = exctr.submit(app_name='sim_hump_camel_dry_run', num_procs=cores, app_args=args_for_sim,
+                            hyperthreads=True, machinefile='notused', stdout='notused',
+                            wait_on_run=True, dry_run=True, stage_inout=os.getcwd())
+
+        task.poll()
+        task.wait()
+
     # This is temp - return something - so doing six_hump_camel_func again...
     batch = len(H['x'])
     H_o = np.zeros(batch, dtype=sim_specs['out'])

libensemble/tests/regression_tests/script_test_balsam_hworld.py

@@ -32,6 +32,7 @@
 exctr = BalsamMPIExecutor(auto_resources=False, central_mode=False, custom_info={'not': 'used'})
 exctr.register_calc(full_path=sim_app, calc_type='sim')  # Default 'sim' app - backward compatible
 exctr.register_calc(full_path=sim_app2, app_name='six_hump_camel')  # Named app
+exctr.register_calc(full_path=sim_app2, app_name='sim_hump_camel_dry_run')
 
 sim_specs = {'sim_f': executor_hworld,
              'in': ['x'],
@@ -68,15 +69,12 @@
 
     assert np.array_equal(H['cstat'], calc_status_list), "Error - unexpected calc status. Received: " + str(H['cstat'])
 
-    # Check summary file:
-    print('Checking expected task status against task summary file ...\n')
+    # Check dry_run submissions inside ensemble.log
+    with open('ensemble.log', 'r') as f:
+        lines = f.readlines()
 
-    calc_desc_list_in = ['Completed', 'Worker killed task on Error', 'Completed',
-                         'Worker killed task on Timeout', 'Task Failed',
-                         'Manager killed on finish']
-
-    # Repeat N times for N workers and insert Completed at start for generator
-    calc_desc_list = ['Completed'] + calc_desc_list_in*nworkers
+    assert len([i for i in lines if 'Test (No submit) Runline:' in i]) == (len(calc_status_list_in) - 1) * nworkers, \
+        "Dry run runlines not listed in ensemble.log for each dry_run submission instance."
 
     # Cleanup (maybe cover del_apps() and del_tasks())
     exctr.del_apps()

libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py

@@ -109,9 +109,13 @@ def print_job_output(outscript):
 
 def move_job_coverage(jobdir):
     # Move coverage files from Balsam DB to ./regression_tests (for concatenation)
+    print('Moving job coverage results.')
     here = os.getcwd()
     covname = '.cov_reg_out.'
 
+    assert any([file.startswith(covname) for file in os.listdir(jobdir)]), \
+        "Coverage results not detected in Balsam Job directory."
+
     for file in os.listdir(jobdir):
         if file.startswith(covname):
             balsam_cov = os.path.join(jobdir, file)

libensemble/tests/run-tests.sh

@@ -587,8 +587,16 @@ if [ "$root_found" = true ]; then
           coverage html #Should create cov_merge/ dir
           echo -e "..Combined Unit Test/Regression Test Coverage HTML written to dir $COV_MERGE_DIR/cov_merge/"
 
-        fi;
+        else
 
+          # Still need to move reg cov results to COV_MERGE_DIR
+          cd $ROOT_DIR/$COV_MERGE_DIR
+          cp $ROOT_DIR/$REG_TEST_SUBDIR/.cov_reg_out .
+
+          coverage combine .cov_reg_out
+          coverage html
+          echo -e "..Combined Regression Test Coverage HTML written to dir $COV_MERGE_DIR/cov_merge/"
+        fi;
       fi;
     fi;