Release/v 0.9.3

.flake8

@@ -42,8 +42,8 @@ per-file-ignores =
         examples/calling_scripts/run_libensemble_on_warpx.py:E402
         libensemble/gen_funcs/persistent_aposmm.py:E402, E501
         libensemble/tests/regression_tests/test_persistent_aposmm*:E402
-        libensemble/tests/regression_tests/dont_run_test_persistent_aposmm*:E402
-        libensemble/tests/regression_tests/test_uniform_sampling_then_persistent_localopt_runs.py:E402
+        libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py:E402
+        libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py:E402
         libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py:E402
         libensemble/tests/deprecated_tests/test_old_aposmm*:E402
         libensemble/tests/unit_tests/test_persistent_aposmm.py:E402

.github/workflows/ci.yml

@@ -42,6 +42,17 @@ jobs:
                       comms-type: l
                       mpi-version: 'mpich'
                       do-balsam: true
+                    - os: windows-latest
+                      python-version: '3.10'
+                      comms-type: l
+                      mpi-version: 'msmpi'
+                      do-balsam: false
+                    - os: windows-latest
+                      python-version: '3.10'
+                      comms-type: m
+                      mpi-version: 'msmpi'
+                      do-balsam: false
+
         env:
             HYDRA_LAUNCHER: 'fork'
             TERM: xterm-256color
@@ -68,6 +79,13 @@ jobs:
             python --version
             pip install -I --upgrade certifi
 
+        - name: Windows - Add clang path to $PATH env
+          shell: bash
+          if: matrix.os == 'windows-latest'
+          run: |
+            echo "PATH=$PATH:C:\msys64\mingw64\bin" >> $GITHUB_ENV
+            echo "PATH=$PATH:C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin\amd64" >> $GITHUB_ENV
+
         - name: Install Ubuntu compilers
           if: matrix.os == 'ubuntu-latest'
           run: conda install gcc_linux-64
@@ -90,34 +108,53 @@ jobs:
             mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk
             conda install clang_osx-64=9.0.1
 
-        - name: Install MPI, mpi4py from conda
-          if: matrix.python-version != '3.10'
+        - name: Setup MPI (${{ matrix.mpi-version }})
+          uses: mpi4py/setup-mpi@v1
+          if: matrix.os == 'windows-latest'
+          with:
+            mpi: ${{ matrix.mpi-version }}
+
+        - name: Install mpi4py on Windows
+          if: matrix.os == 'windows-latest'
+          run: pip install mpi4py
+
+        - name: Install mpi4py and MPI from conda
+          if: matrix.python-version != '3.10' && matrix.os != 'windows-latest'
           run: |
             conda install ${{ matrix.mpi-version }}
             conda install mpi4py
 
-        - name: Install MPI, mpi4py from pip
-          if: matrix.python-version == '3.10'
+        - name: Install mpi4py from pip, MPI from conda
+          if: matrix.python-version == '3.10' && matrix.os != 'windows-latest'
           run: |
             conda install ${{ matrix.mpi-version }}
             pip install mpi4py
 
         - name: Install generator dependencies
-          if: contains('3.7_3.8_3.9_3.10', matrix.python-version) && matrix.do-balsam == false
+          if: contains('3.7_3.8_3.9_3.10', matrix.python-version) && matrix.do-balsam == false && matrix.os != 'windows-latest'
           run: |
             python -m pip install --upgrade pip
             conda install nlopt
             conda install scipy
             conda install superlu_dist
             conda install hypre
             conda install mumps-mpi
-            # pip install petsc
-            # pip install petsc4py
+            conda install petsc
+            conda install petsc4py
             pip install DFO-LS
 
             pip install mpmath
+            pip install ax-platform
             python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git@development/PCGPwM
 
+        - name: Install some generator dependencies on Windows
+          if: matrix.os == 'windows-latest'
+          run: |
+            python -m pip install --upgrade pip
+            conda install nlopt
+            conda install scipy
+            pip install mpmath
+
         - name: Install generator dependencies for Ubuntu tests
           if: matrix.os == 'ubuntu-latest' && matrix.do-balsam == false
           run: |
@@ -132,21 +169,24 @@ jobs:
             cd heffte/build
             pwd
             cmake -D CMAKE_BUILD_TYPE=Release -D BUILD_SHARED_LIBS=ON -D CMAKE_INSTALL_PREFIX=./ -D Heffte_ENABLE_AVX=ON -D Heffte_ENABLE_FFTW=ON ../
-            make
+            make -j 4
             make install
             cp ./benchmarks/speed3d_c2c ../../libensemble/tests/regression_tests/
             # end heffte build and dependencies
 
-            pip install dragonfly-opt
-            pip install torch
-            pip install gpytorch
+            # pip install dragonfly-opt
+            pip install git+https://github.com/dragonfly/dragonfly.git
+
             pip install ax-platform
 
         - name: Install other testing dependencies
+          if: matrix.do-balsam == false
           run: |
             pip install -r install/testing_requirements.txt
             pip install psutil
             pip install pyyaml
+            pip install funcx
+            pip install balsam
 
         - name: Install Tasmanian on Ubuntu
           if: matrix.os == 'ubuntu-latest' && matrix.do-balsam == false
@@ -158,6 +198,7 @@ jobs:
           env:
               BALSAM_DB_PATH: $HOME/test-balsam
           run: |
+            pip install -r install/testing_requirements.txt
             wget https://github.com/argonne-lcf/balsam/archive/refs/tags/0.5.0.tar.gz
             mkdir ../balsam; tar xf 0.5.0.tar.gz -C ../balsam;
             python install/configure_balsam_install.py
@@ -176,11 +217,8 @@ jobs:
 
         - name: Find MPI, Install libEnsemble, flake8, ulimit adjust
           run: |
-            python install/find_mpi.py
-            mpiexec --version
             pip install -e .
             flake8 libensemble
-            ulimit -Sn 10000
 
         - name: Activate API unit test if using mpich
           if: matrix.mpi-version == 'mpich'
@@ -192,10 +230,10 @@ jobs:
           run: |
            ./libensemble/tests/run-tests.sh -e -A "-W error" -z -${{ matrix.comms-type }}
 
-        #- name: Run simple tests, Ubuntu, Python 3.11
-        #  if: matrix.python-version == '3.11'
-        #  run: |
-        #   ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }}
+        - name: Run simple tests, Windows
+          if: matrix.os == 'windows-latest'
+          run: |
+           ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }}
 
         - name: Run extensive tests, macOS
           if: matrix.os == 'macos-latest'

.gitignore

@@ -15,6 +15,7 @@ x.log
 *.out
 *.err
 *.stat
+*.csv
 
 .vscode
 build/

.wci.yml

@@ -17,8 +17,8 @@ description: |
 language: Python
 
 release:
-  version: 0.9.2
-  date: 2022-07-06
+  version: 0.9.3
+  date: 2022-10-13
 
 documentation:
   general: https://libensemble.readthedocs.io
@@ -32,5 +32,6 @@ execution_environment:
     - Slurm
     - LSF
     - Cobalt
+    - PBS
   transfer_protocols:
     - Globus

CHANGELOG.rst

@@ -8,6 +8,34 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r
 
 .. _`github releases page`: https://github.com/Libensemble/libensemble/releases
 
+Release 0.9.3
+-------------
+
+:Date: October 13, 2022
+
+New capabilities:
+
+* New pair of utilities, `liberegister` and `libesubmit` (based on *PSI/J*), for easily preparing and launching libEnsemble workflows with local comms onto most machines and schedulers. #807
+* New persistent support function to cancel sim_ids (`request_cancel_sim_ids`). #880
+* `keep_state` option for persistent workers: this lets the manager know that the information being sent is intermediate. #880 
+
+Other enhancements:
+
+* The Executor `manager_poll()` interface now sets consistent flags instead of literal strings. #877
+* Some internal modules and the test suite now work on Windows. #869 #888
+* Specifying the `num_resource_sets` *libE_specs* option instead of `zero_resource_workers` is now recommended except when using a fixed worker/resource mapping. Use ``persis_info["gen_resources"]`` to assign persistent generator resources (default is zero). #905
+* An extraneous warning removed. #903
+
+:Note:
+
+* Tested platforms include Linux, MacOS, Windows, Theta (Cray XC40/Cobalt), Summit (IBM Power9/LSF), Bebop (Cray CS400/Slurm), Swing (A100 GPU system), Perlmutter (HPE Cray EX with A100 NVIDIA GPUs). For Perlmutter, see "Known issues" below.
+* Tested Python versions: (Cpython) 3.7, 3.8, 3.9, 3.10.
+
+:Known issues:
+
+* At time of testing on Perlmutter there was an issue running concurrent applications on a node, following a recent system update. This also affects previous versions of libEnsemble, and is being investigated.
+* See known issues section in the documentation.
+
 Release 0.9.2
 -------------
 
@@ -19,9 +47,9 @@ New capabilities:
 * Added configuration options for `libE_stats.txt` file. #743
 * Support for `spawn` and `forkserver` multiprocessing start methods. #797
 
- * Note that macOS no longer switches to using `fork`. macOS (since Python 3.8) and Windows
- default to using `spawn`. When using `spawn`, we recommend placing calling script code in
- an ``if __name__ == "__main__":`` block. The multiprocessing interface can be used to switch methods (https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method).
+ * Note that macOS no longer switches to using `fork`. macOS (since Python 3.8) and Windows default to
+   using `spawn`. When using `spawn`, we recommend placing calling script code in an ``if __name__ == "__main__":`` block.
+   The multiprocessing interface can be used to switch methods (https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method).
 
 Updates to example functions:
 

README.rst

@@ -31,8 +31,8 @@ Introduction to libEnsemble
 
 libEnsemble is a Python_ toolkit for coordinating workflows of asynchronous and dynamic ensembles of calculations.
 
-libEnsemble can help users take advantage of massively parallel resources to solve design,
-decision, and inference problems and expand the class of problems that can benefit from
+libEnsemble helps users take advantage of massively parallel resources to solve design,
+decision, and inference problems and expands the class of problems that can benefit from
 increased parallelism.
 
 libEnsemble aims for:
@@ -58,20 +58,23 @@ With a basic familiarity of Python and NumPy_, users can easily incorporate
 any other mathematics, machine-learning, or resource-management libraries into libEnsemble
 workflows.
 
-libEnsemble employs a manager/worker scheme that runs on MPI, multiprocessing,
+libEnsemble employs a manager/worker scheme that communicates via MPI, multiprocessing,
 or TCP. Workers control and monitor any level of work using the aforementioned
 generator and simulator functions, from small subnode tasks to huge many-node computations.
 
 libEnsemble includes an Executor interface so application-launching functions are
 portable, resilient, and flexible; it also automatically detects available nodes
 and cores, and can dynamically assign resources to workers.
 
+libEnsemble performs best on Unix-like systems like Linux and macOS. See the
+:ref:`FAQ<faqwindows>` for more information.
+
 .. before_dependencies_rst_tag
 
 Dependencies
 ~~~~~~~~~~~~
 
-Required dependencies:
+**Required dependencies**:
 
 * Python_ 3.7 or above
 * NumPy_
@@ -83,7 +86,7 @@ When using  ``mpi4py`` for libEnsemble communications:
 * A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries
 * mpi4py_ v2.0.0 or above
 
-Optional dependencies:
+**Optional dependencies**:
 
 * Balsam_
 
@@ -93,8 +96,7 @@ Balsam site, including to remote machines.
 
 * pyyaml_
 
-libEnsemble is typically configured and parameterized via Python dictionaries.
-As of v0.8.0, libEnsemble can also be parameterized via yaml.
+libEnsemble is typically configured and parameterized via Python dictionaries. libEnsemble can also be parameterized via yaml.
 
 * funcX_
 
@@ -103,6 +105,12 @@ a function-as-a-service platform to which workers can submit remote generator or
 simulator function instances. This feature can help distribute an ensemble
 across systems and heterogeneous resources.
 
+* `psi-j-python`_
+
+As of v0.9.3, libEnsemble features a set of command-line utilities for submitting
+libEnsemble jobs to almost any system or scheduler via a `PSI/J`_ Python interface. tqdm_
+is also required.
+
 The example simulation and generation functions and tests require the following:
 
 * SciPy_
@@ -229,7 +237,8 @@ Resources
 **Further Information:**
 
 - Documentation is provided by ReadtheDocs_.
-- An overview of libEnsemble's structure and capabilities is given in this manuscript_ and poster_
+- Contributions_ to libEnsemble are welcome.
+- An overview of libEnsemble's structure and capabilities is given in this manuscript_ and poster_.
 - Examples of production user functions and complete workflows can be viewed, downloaded, and contributed to in the libEnsemble `Community Examples repository`_.
 
 **Citation:**
@@ -243,7 +252,7 @@ Resources
     author  = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and
                David Bindel and John-Luke Navarro},
     institution = {Argonne National Laboratory},
-    number  = {Revision 0.9.2},
+    number  = {Revision 0.9.3},
     year    = {2022},
     url     = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf}
   }
@@ -305,6 +314,7 @@ See a complete list of `example user scripts`_.
 .. _Community Examples repository: https://github.com/Libensemble/libe-community-examples
 .. _Conda: https://docs.conda.io/en/latest/
 .. _conda-forge: https://conda-forge.org/
+.. _Contributions: https://github.com/Libensemble/libensemble/blob/main/CONTRIBUTING.rst
 .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=main
 .. _DEAP: https://deap.readthedocs.io/en/master/overview.html
 .. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols
@@ -330,6 +340,8 @@ See a complete list of `example user scripts`_.
 .. _petsc4py: https://bitbucket.org/petsc/petsc4py
 .. _PETSc/TAO: http://www.mcs.anl.gov/petsc
 .. _poster: https://figshare.com/articles/libEnsemble_A_Python_Library_for_Dynamic_Ensemble-Based_Computations/12559520
+.. _PSI/J: https://exaworks.org/psij
+.. _psi-j-python: https://github.com/ExaWorks/psi-j-python
 .. _psutil: https://pypi.org/project/psutil/
 .. _PyPI: https://pypi.org
 .. _pytest-cov: https://pypi.org/project/pytest-cov/
@@ -348,6 +360,7 @@ See a complete list of `example user scripts`_.
 .. _tarball: https://github.com/Libensemble/libensemble/releases/latest
 .. _Tasmanian: https://tasmanian.ornl.gov/
 .. _Theta: https://www.alcf.anl.gov/alcf-resources/theta
+.. _tqdm: https://tqdm.github.io/
 .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html
 .. _VTMOP: https://github.com/Libensemble/libe-community-examples#vtmop
 .. _WarpX: https://warpx.readthedocs.io/en/latest/