Merge 748bbe1 into b8ae625

.travis.yml

@@ -24,6 +24,9 @@ matrix:
       language: generic
       python: 3
 
+services:
+    - postgresql
+
 # matrix:
 #  allow_failures:
 #    - env: MPI=openmpi
@@ -80,10 +83,12 @@ install:
   - python conda/find_mpi.py # locate compilers
   - mpiexec --version # Show MPI library details
   - pip install -e .
-
+  - python conda/install-balsam.py
 
 before_script:
   - flake8 libensemble
+  - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh
+  - source setbalsampath.sh
   # Set conda compilers to use new SDK instead of Travis default.
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
         echo "export CONDA_BUILD_SYSROOT=/Users/travis/build/Libensemble/sdk/MacOSX10.13.sdk" > setenv.sh;
@@ -92,7 +97,7 @@ before_script:
 
 # Run test (-z show output)
 script:
-  - libensemble/tests/run-tests.sh -z
+  - ./libensemble/tests/run-tests.sh -z
 
 # Coverage
 after_success:

conda/configure-balsam-test.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Run this script between each repeated local run of the Balsam
+#   job_control_hworld_balsam test in the base libensemble directory.
+#   Besides ensuring that postgres and the generated Balsam db have the proper
+#   permissions, this script flushes previous apps and jobs in the db before
+#   submitting the test_balsam app/job script.
+#
+#   Most of this comes from scaling_tests/forces/balsam_local.sh
+
+# Can't run this line in calling Python file. Balsam installation hasn't been
+#   noticed by the Python runtime yet.
+python -c 'from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage; modify_Balsam_pyCoverage()'
+export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam.py
+export NUM_WORKERS=2
+export WORKFLOW_NAME=libe_test-balsam
+export LIBE_WALLCLOCK=3
+export THIS_DIR=$PWD
+export SCRIPT_BASENAME=script_test_balsam
+
+# Set proper permissions, initialize Balsam DB, activate DB
+export BALSAM_DB_PATH='~/test-balsam'
+sudo chown -R postgres:postgres /var/run/postgresql
+sudo chmod a+w /var/run/postgresql
+balsam init ~/test-balsam
+sudo chmod -R 700 ~/test-balsam/balsamdb
+source balsamactivate test-balsam
+
+# Refresh DB
+balsam rm apps --all --force
+balsam rm jobs --all --force
+
+# Submit script_test_balsam as app
+balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME"
+
+# Submit job based on script_test_balsam app
+balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME --application $SCRIPT_BASENAME.app --wall-time-minutes $LIBE_WALLCLOCK --num-nodes 1 --ranks-per-node $((NUM_WORKERS+1)) --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" --url-in="local:/$THIS_DIR/*" --yes

conda/install-balsam.py

@@ -0,0 +1,23 @@
+#! /usr/bin/env python
+
+# Installing Balsam consists of both cloning and pip installing the latest version
+#   of Balsam from source but also moving the Balsam-relevant test to the regression_tests
+#   directory. This way, run-tests won't run a non-relevant test if Balsam isn't
+#   installed or the necessary Python version isn't installed.
+
+import sys
+import os
+import subprocess
+
+
+balsamclone = 'git clone https://github.com/balsam-alcf/balsam.git ../balsam'
+
+if int(sys.version[2]) >= 6:  # Balsam only supports Python 3.6+
+    subprocess.check_call(balsamclone.split())
+    os.chdir('../balsam')
+    subprocess.check_call('pip install -e .'.split())
+    os.chdir('../libensemble')
+    if not os.path.isfile('./libensemble/tests/regression_tests/test_balsam.py'):
+        os.rename('./conda/test_balsam.py',
+                  './libensemble/tests/regression_tests/test_balsam.py')
+    subprocess.run('./conda/configure-balsam-test.sh'.split())

conda/run_travis_locally/build_mpich_libE.sh

@@ -3,7 +3,7 @@
 # Note for other MPIs may need to install some packages from source (eg. petsc)
 
 # -x echo commands
-set -x # problem with this - loads of conda internal commands shown - overwhelming.
+# set -x # problem with this - loads of conda internal commands shown - overwhelming.
 
 export PYTHON_VERSION=3.7       # override with -p <version>
 export LIBE_BRANCH="develop"    # override with -b <branchname>
@@ -41,7 +41,10 @@ done
 
 echo -e "\nBuilding libE on ${SYSTEM} with python $PYTHON_VERSION and branch ${LIBE_BRANCH}\n"
 
-# sudo apt-get update
+sudo pip install --upgrade pip
+sudo /etc/init.d/postgresql stop 9.2
+sudo /etc/init.d/postgresql start 9.6
+export PATH=$PATH:/usr/lib/postgresql/9.6/bin
 
 # This works if not sourced but if sourced its no good.
 # set -e
@@ -69,11 +72,6 @@ else
   conda install gcc_linux-64 || return
 fi
 conda install nlopt petsc4py petsc mumps-mpi=5.1.2=h5bebb2f_1007 mpi4py scipy $MPI
-#conda install numpy || return #scipy includes numpy
-# conda install scipy || return
-# conda install mpi4py || return
-# conda install petsc4py petsc || return
-# conda install nlopt || return
 
 # pip install these as the conda installs downgrade pytest on python3.4
 pip install pytest || return
@@ -86,8 +84,10 @@ pip install coveralls || return
 git clone -b $LIBE_BRANCH https://github.com/Libensemble/libensemble.git || return
 cd libensemble/ || return
 pip install -e . || return
+python conda/install-balsam.py
+export BALSAM_DB_PATH=~/test-balsam
 
-libensemble/tests/run-tests.sh
+./libensemble/tests/run-tests.sh -z
 
 echo -e "\n\nScript completed...\n\n"
 set +ex

conda/test_balsam.py

@@ -0,0 +1,95 @@
+import subprocess
+import os
+import time
+import sys
+import libensemble
+from libensemble.tests.regression_tests.common import parse_args, modify_Balsam_worker
+
+# TESTSUITE_COMMS: local
+# TESTSUITE_NPROCS: 3
+
+# This test is NOT submitted as a job to Balsam. That would be script_test_balsam.py
+#   This test executes that job through the 'runstr' line defined further down.
+
+nworkers, is_master, libE_specs, _ = parse_args()  # None used. Bug-prevention
+
+# Set libensemble base directory to environment variable for Balsam coverage
+#   orientation purposes. Otherwise coverage in the Balsam data directory is
+#   (unsuccessfully) collected.
+libepath = os.path.dirname(libensemble.__file__)
+os.environ['LIBE_PATH'] = libepath
+
+# Balsam is meant for HPC systems that commonly distribute jobs across many
+#   nodes. For our purposes, we append (hack) ten workers to Balsam's WorkerGroup
+print("Currently in {}. Beginning Balsam worker modification".format(os.getcwd()))
+modify_Balsam_worker()
+
+# By this point, script_test_balsam.py has been submitted as an app and job to Balsam
+# This line launches the queued job in the Balsam database
+runstr = 'balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1'
+print('Executing Balsam job with command: {}'.format(runstr))
+subprocess.Popen(runstr.split())
+
+# Location of Balsam DB location defined in configure-balsam-test.sh
+basedb = os.path.expanduser('~/test-balsam/data/libe_test-balsam')
+
+# Periodically wait for Workflow and Job directory within Balsam DB
+sleeptime = 0
+print('{}: Waiting for Workflow directory'.format(sleeptime))
+while not os.path.isdir(basedb) and sleeptime < 58:
+    sleeptime += 1
+    print('{}'.format(sleeptime), end=" ")
+    sys.stdout.flush()
+    time.sleep(1)
+
+print('{}: Waiting for Job Directory'.format(sleeptime))
+while len(os.listdir(basedb)) == 0 and sleeptime < 58:
+    sleeptime += 1
+    print('{}'.format(sleeptime), end=" ")
+    sys.stdout.flush()
+    time.sleep(1)
+
+# Job directory now exists
+jobdirname = os.listdir(basedb)[0]
+jobdir = os.path.join(basedb, jobdirname)
+outscript = os.path.join(jobdir, 'job_script_test_balsam.out')
+
+# Periodically wait for Balsam Job output
+print('{}: Checking for Balsam output file: {}'.format(sleeptime, outscript))
+while not os.path.isfile(outscript) and sleeptime < 58:
+    sleeptime += 2
+    print('{}'.format(sleeptime), end=" ")
+    sys.stdout.flush()
+    time.sleep(2)
+
+# Print sections of Balsam output to screen every second until complete
+print('{}: Output file exists. Waiting for Balsam Job Output.'.format(sleeptime))
+lastposition = 0
+lastlines = ['Job 4 done on worker 1\n', 'Job 4 done on worker 2\n']
+while sleeptime < 58:
+    with open(outscript, 'r') as f:
+        f.seek(lastposition)    # (should) prevent outputting already printed sections
+        new = f.read()
+        lastposition = f.tell()
+    if len(new) > 0:
+        print(new)
+        sys.stdout.flush()
+    if any(new.endswith(line) for line in lastlines):
+        break
+    print('{}'.format(sleeptime), end=" ")
+    time.sleep(1)
+    sleeptime += 1
+
+print('{}: Importing any coverage from Balsam.'.format(sleeptime))
+here = os.getcwd()
+
+# Move coverage files from Balsam DB to ./regression_tests (for concatenation)
+covname = '.cov_reg_out.'
+for file in os.listdir(jobdir):
+    if file.startswith(covname):
+        balsam_cov = os.path.join(jobdir, file)
+        here_cov = os.path.join(here, file)
+        print('Moved {} from {} to {}'.format(file, jobdir, here))
+        os.rename(balsam_cov, here_cov)
+
+print('Test complete.')

libensemble/sim_funcs/job_control_hworld.py

@@ -72,7 +72,7 @@ def job_control_hworld(H, persis_info, sim_specs, libE_specs):
     # pref send this in X as a sim_in from calling script
     global sim_count
     sim_count += 1
-    timeout = 3.0
+    timeout = 6.0
     if sim_count == 1:
         args_for_sim = 'sleep 1'  # Should finish
     elif sim_count == 2:

libensemble/tests/regression_tests/.bal_coveragerc

@@ -0,0 +1,30 @@
+[run]
+source = $LIBE_PATH
+
+omit =
+    */tests/*
+    */unit_tests/*
+    */unit_tests_nompi/*
+    */regression_tests/*
+    */branin_*/*
+
+branch = true
+
+data_file = .cov_reg_out
+
+parallel = true
+
+[report]
+omit =
+    */__init__.py/*
+    */setup.py/*
+    */tests/*
+    */unit_tests/*
+    */unit_tests_nompi/*
+    */regression_tests/*
+
+exclude_lines =
+    if __name__ == .__main__.:
+
+[html]
+directory = cov_reg

libensemble/tests/regression_tests/common.py

@@ -170,3 +170,63 @@ def build_simfunc():
     buildstring = 'mpicc -o my_simjob.x ../unit_tests/simdir/my_simjob.c'
     # subprocess.run(buildstring.split(),check=True) #Python3.5+
     subprocess.check_call(buildstring.split())
+
+
+def modify_Balsam_worker():
+    # Balsam is meant for HPC systems that commonly distribute jobs across many
+    #   nodes. Due to the nature of testing Balsam on local or CI systems which
+    #   usually only contain a single node, we need to change Balsam's default
+    #   worker setup so multiple workers can be run on a single node (until this
+    #   feature is [hopefully] added!).For our purposes, we append ten workers
+    #   to Balsam's WorkerGroup
+    import balsam
+
+    new_lines = ["        for idx in range(10):\n",
+                 "            w = Worker(1, host_type='DEFAULT', num_nodes=1)\n",
+                 "            self.workers.append(w)\n"]
+
+    workerfile = 'worker.py'
+    balsam_path = os.path.dirname(balsam.__file__) + '/launcher'
+    balsam_worker_path = os.path.join(balsam_path, workerfile)
+
+    with open(balsam_worker_path, 'r') as f:
+        lines = f.readlines()
+
+    if lines[-3] != new_lines[0]:
+        lines = lines[:-2]  # effectively inserting new_lines[0] above
+        lines.extend(new_lines)
+
+    with open(balsam_worker_path, 'w') as f:
+        for line in lines:
+            f.write(line)
+
+    print("Modified worker file in {}".format(balsam_path))
+
+
+def modify_Balsam_pyCoverage():
+    # Tracking line coverage of our code through our tests requires running a test
+    #   with the format 'python -m coverage run test.py args'. Balsam explicitely
+    #   configures Python runs with 'python test.py args' with no current
+    #   capability for specifying runtime Python options. This hack attempts to
+    #   resolve this for our purposes only.
+    import balsam
+
+    old_line = "            path = ' '.join((exe, script_path, args))\n"
+    new_line = "            path = ' '.join((exe, '-m coverage run --parallel-mode --rcfile=./libensemble/tests/regression_tests/.bal_coveragerc', script_path, args))\n"
+
+    commandfile = 'cli_commands.py'
+    balsam_path = os.path.dirname(balsam.__file__) + '/scripts'
+    balsam_commands_path = os.path.join(balsam_path, commandfile)
+
+    with open(balsam_commands_path, 'r') as f:
+        lines = f.readlines()
+
+    for i in range(len(lines)):
+        if lines[i] == old_line:
+            lines[i] = new_line
+
+    with open(balsam_commands_path, 'w') as f:
+        for line in lines:
+            f.write(line)
+
+    print("Modified cli_commands file in {}".format(balsam_path))