From d4b7b3813c4dd1a55c4ea4cc0edeeb45341cdd06 Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 28 Jun 2021 14:33:49 -0500 Subject: [PATCH 1/7] try a dry_run submission during balsam executor regression test --- libensemble/sim_funcs/executor_hworld.py | 4 ++++ .../regression_tests/script_test_balsam_hworld.py | 14 ++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index f17df5e84..ec2bca0cc 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -127,6 +127,10 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): except ValueError: pass + task = exctr.submit(app_name='sim_hump_camel_dry_run', num_procs=cores, app_args=args_for_sim, + hyperthreads=True, machinefile='notused', stdout='notused', + wait_on_run=True, dry_run=True) + # This is temp - return something - so doing six_hump_camel_func again... batch = len(H['x']) H_o = np.zeros(batch, dtype=sim_specs['out']) diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index 1addcb240..e46a68177 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -32,6 +32,7 @@ exctr = BalsamMPIExecutor(auto_resources=False, central_mode=False, custom_info={'not': 'used'}) exctr.register_calc(full_path=sim_app, calc_type='sim') # Default 'sim' app - backward compatible exctr.register_calc(full_path=sim_app2, app_name='six_hump_camel') # Named app +exctr.register_calc(full_path=sim_app2, app_name='sim_hump_camel_dry_run') sim_specs = {'sim_f': executor_hworld, 'in': ['x'], @@ -68,15 +69,12 @@ assert np.array_equal(H['cstat'], calc_status_list), "Error - unexpected calc status. Received: " + str(H['cstat']) - # Check summary file: - print('Checking expected task status against task summary file ...\n') + # Check dry_run submissions inside ensemble.log + with open('ensemble.log', 'r') as f: + lines = f.readlines() - calc_desc_list_in = ['Completed', 'Worker killed task on Error', 'Completed', - 'Worker killed task on Timeout', 'Task Failed', - 'Manager killed on finish'] - - # Repeat N times for N workers and insert Completed at start for generator - calc_desc_list = ['Completed'] + calc_desc_list_in*nworkers + assert len([i for i in lines if 'Test (No submit) Runline:' in i]) == (len(calc_status_list_in) - 1) * nworkers, \ + "Dry run runlines not listed in ensemble.log for each dry_run submission instance." # Cleanup (maybe cover del_apps() and del_tasks()) exctr.del_apps() From 874bc9f7b1ae1457f7d9e5fc37b873995a6e77ed Mon Sep 17 00:00:00 2001 From: jlnav Date: Mon, 28 Jun 2021 16:19:21 -0500 Subject: [PATCH 2/7] had forgot to add coveralls step to Balsam test --- .github/workflows/libE-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/libE-ci.yml b/.github/workflows/libE-ci.yml index 5c1442c8c..1eb058f04 100644 --- a/.github/workflows/libE-ci.yml +++ b/.github/workflows/libE-ci.yml @@ -212,6 +212,13 @@ jobs: source balsamactivate test-balsam ./libensemble/tests/run-tests.sh -r -y 'test_balsam*' -z -l + - name: Merge coverage, run Coveralls + env: + COVERALLS_PARALLEL: true + run: | + mv libensemble/tests/.cov* . + coveralls --service=github + coveralls: name: Notify coveralls of all jobs completing needs: [test-libE, test-libE-with-Balsam] From 539ed1de52ecb0f5a510e36c8ff830f690b301ee Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 29 Jun 2021 11:38:49 -0500 Subject: [PATCH 3/7] still debugging coverage, adjusting coverage version downloaded --- .github/workflows/libE-ci.yml | 4 ++-- install/configure_balsam_install.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/libE-ci.yml b/.github/workflows/libE-ci.yml index 1eb058f04..dcab47844 100644 --- a/.github/workflows/libE-ci.yml +++ b/.github/workflows/libE-ci.yml @@ -85,7 +85,7 @@ jobs: pip install deap python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git@development/PCGPwM pip install flake8 - pip install coverage + pip install coverage==4.5.4 pip install pytest pip install pytest-cov pip install pytest-timeout @@ -178,7 +178,7 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 - pip install coverage + pip install coverage==4.5.4 pip install pytest pip install pytest-cov pip install pytest-timeout diff --git a/install/configure_balsam_install.py b/install/configure_balsam_install.py index 09e30049a..5879ac2f5 100644 --- a/install/configure_balsam_install.py +++ b/install/configure_balsam_install.py @@ -32,8 +32,10 @@ def configure_coverage(): newlines = [i for i in lines if i != ' */balsam_executor.py\n'] + print('New libensemble/tests/.coveragerc: \n') with open(coveragerc, 'w') as f: for line in newlines: + print(line) f.write(line) From c7f14bc38b1a0fbefecedc3d6702864043516a3c Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 30 Jun 2021 11:23:19 -0500 Subject: [PATCH 4/7] assert coverage results are produced in balsam job dir, modify run-tests to merge regression cov results even if not running unit tests --- .../scripts_used_by_reg_tests/test_balsam_hworld.py | 4 ++++ libensemble/tests/run-tests.sh | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py b/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py index dcb58c9dd..cd7227d22 100644 --- a/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py @@ -109,9 +109,13 @@ def print_job_output(outscript): def move_job_coverage(jobdir): # Move coverage files from Balsam DB to ./regression_tests (for concatenation) + print('Moving job coverage results.') here = os.getcwd() covname = '.cov_reg_out.' + assert any[file.startswith(covname) for file in os.listdir(jobdir)], \ + "Coverage results not detected in Balsam Job directory." + for file in os.listdir(jobdir): if file.startswith(covname): balsam_cov = os.path.join(jobdir, file) diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index d73d57950..fa7a48b2d 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -587,8 +587,16 @@ if [ "$root_found" = true ]; then coverage html #Should create cov_merge/ dir echo -e "..Combined Unit Test/Regression Test Coverage HTML written to dir $COV_MERGE_DIR/cov_merge/" - fi; + else + # Still need to move reg cov results to COV_MERGE_DIR + cd $ROOT_DIR/$COV_MERGE_DIR + cp $ROOT_DIR/$REG_TEST_SUBDIR/.cov_reg_out . + + coverage combine .cov_reg_out + coverage html + echo -e "..Combined Regression Test Coverage HTML written to dir $COV_MERGE_DIR/cov_merge/" + fi; fi; fi; From a214d4f6c80c6b91e1c7b64dc36ae5a2a68d6f5d Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 30 Jun 2021 11:27:33 -0500 Subject: [PATCH 5/7] flake8 --- .../scripts_used_by_reg_tests/test_balsam_hworld.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py b/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py index cd7227d22..0558e6a27 100644 --- a/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/scripts_used_by_reg_tests/test_balsam_hworld.py @@ -113,7 +113,7 @@ def move_job_coverage(jobdir): here = os.getcwd() covname = '.cov_reg_out.' - assert any[file.startswith(covname) for file in os.listdir(jobdir)], \ + assert any([file.startswith(covname) for file in os.listdir(jobdir)]), \ "Coverage results not detected in Balsam Job directory." for file in os.listdir(jobdir): From 051144328b2433d56885bad2f0d036a955eae28a Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 30 Jun 2021 13:00:06 -0500 Subject: [PATCH 6/7] call additional methods, use additional parameters for coverage --- libensemble/sim_funcs/executor_hworld.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index ec2bca0cc..f5f757451 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -129,7 +129,10 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): task = exctr.submit(app_name='sim_hump_camel_dry_run', num_procs=cores, app_args=args_for_sim, hyperthreads=True, machinefile='notused', stdout='notused', - wait_on_run=True, dry_run=True) + wait_on_run=True, dry_run=True, stage_inout=os.getcwd()) + + task.poll() + task.wait() # This is temp - return something - so doing six_hump_camel_func again... batch = len(H['x']) From 812ae0827482ca136eac584a66708ab634108879 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 30 Jun 2021 13:01:51 -0500 Subject: [PATCH 7/7] flake8 --- libensemble/sim_funcs/executor_hworld.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index f5f757451..91d05ef90 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -3,6 +3,7 @@ MAN_SIGNAL_FINISH, WORKER_DONE, TASK_FAILED, WORKER_KILL_ON_TIMEOUT) import numpy as np +import os __all__ = ['executor_hworld']