Merge 2a20597 into f231328

.flake8

@@ -43,6 +43,8 @@ per-file-ignores =
         libensemble/gen_funcs/persistent_aposmm.py:E402, E501
         libensemble/tests/regression_tests/test_persistent_aposmm*:E402
         libensemble/tests/regression_tests/dont_run_test_persistent_aposmm*:E402
+        libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py:E402
+        libensemble/tests/regression_tests/dontrun_test_persistent_gp_multitask_ax.py:E402
         libensemble/tests/regression_tests/test_uniform_sampling_then_persistent_localopt_runs.py:E402
         libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py:E402
         libensemble/tests/deprecated_tests/test_old_aposmm*:E402

.github/workflows/ci.yml

@@ -42,6 +42,17 @@ jobs:
                       comms-type: l
                       mpi-version: 'mpich'
                       do-balsam: true
+                    - os: windows-latest
+                      python-version: '3.10'
+                      comms-type: l
+                      mpi-version: 'msmpi'
+                      do-balsam: false
+                    - os: windows-latest
+                      python-version: '3.10'
+                      comms-type: m
+                      mpi-version: 'msmpi'
+                      do-balsam: false
+
         env:
             HYDRA_LAUNCHER: 'fork'
             TERM: xterm-256color
@@ -68,6 +79,13 @@ jobs:
             python --version
             pip install -I --upgrade certifi
 
+        - name: Windows - Add clang path to $PATH env
+          shell: bash
+          if: matrix.os == 'windows-latest'
+          run: |
+            echo "PATH=$PATH:C:\msys64\mingw64\bin" >> $GITHUB_ENV
+            echo "PATH=$PATH:C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin\amd64" >> $GITHUB_ENV
+
         - name: Install Ubuntu compilers
           if: matrix.os == 'ubuntu-latest'
           run: conda install gcc_linux-64
@@ -90,34 +108,53 @@ jobs:
             mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk
             conda install clang_osx-64=9.0.1
 
-        - name: Install MPI, mpi4py from conda
-          if: matrix.python-version != '3.10'
+        - name: Setup MPI (${{ matrix.mpi-version }})
+          uses: mpi4py/setup-mpi@v1
+          if: matrix.os == 'windows-latest'
+          with:
+            mpi: ${{ matrix.mpi-version }}
+
+        - name: Install mpi4py on Windows
+          if: matrix.os == 'windows-latest'
+          run: pip install mpi4py
+
+        - name: Install mpi4py and MPI from conda
+          if: matrix.python-version != '3.10' && matrix.os != 'windows-latest'
           run: |
             conda install ${{ matrix.mpi-version }}
             conda install mpi4py
 
-        - name: Install MPI, mpi4py from pip
-          if: matrix.python-version == '3.10'
+        - name: Install mpi4py from pip, MPI from conda
+          if: matrix.python-version == '3.10' && matrix.os != 'windows-latest'
           run: |
             conda install ${{ matrix.mpi-version }}
             pip install mpi4py
 
         - name: Install generator dependencies
-          if: contains('3.7_3.8_3.9_3.10', matrix.python-version) && matrix.do-balsam == false
+          if: contains('3.7_3.8_3.9_3.10', matrix.python-version) && matrix.do-balsam == false && matrix.os != 'windows-latest'
           run: |
             python -m pip install --upgrade pip
             conda install nlopt
             conda install scipy
             conda install superlu_dist
             conda install hypre
             conda install mumps-mpi
-            # pip install petsc
-            # pip install petsc4py
+            conda install petsc
+            conda install petsc4py
             pip install DFO-LS
 
             pip install mpmath
+            pip install ax-platform
             python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git@development/PCGPwM
 
+        - name: Install some generator dependencies on Windows
+          if: matrix.os == 'windows-latest'
+          run: |
+            python -m pip install --upgrade pip
+            conda install nlopt
+            conda install scipy
+            pip install mpmath
+
         - name: Install generator dependencies for Ubuntu tests
           if: matrix.os == 'ubuntu-latest' && matrix.do-balsam == false
           run: |
@@ -132,21 +169,24 @@ jobs:
             cd heffte/build
             pwd
             cmake -D CMAKE_BUILD_TYPE=Release -D BUILD_SHARED_LIBS=ON -D CMAKE_INSTALL_PREFIX=./ -D Heffte_ENABLE_AVX=ON -D Heffte_ENABLE_FFTW=ON ../
-            make
+            make -j 4
             make install
             cp ./benchmarks/speed3d_c2c ../../libensemble/tests/regression_tests/
             # end heffte build and dependencies
 
-            pip install dragonfly-opt
-            pip install torch
-            pip install gpytorch
+            # pip install dragonfly-opt
+            pip install git+https://github.com/dragonfly/dragonfly.git
+
             pip install ax-platform
 
         - name: Install other testing dependencies
+          if: matrix.do-balsam == false
           run: |
             pip install -r install/testing_requirements.txt
             pip install psutil
             pip install pyyaml
+            pip install funcx
+            pip install balsam
 
         - name: Install Tasmanian on Ubuntu
           if: matrix.os == 'ubuntu-latest' && matrix.do-balsam == false
@@ -158,6 +198,7 @@ jobs:
           env:
               BALSAM_DB_PATH: $HOME/test-balsam
           run: |
+            pip install -r install/testing_requirements.txt
             wget https://github.com/argonne-lcf/balsam/archive/refs/tags/0.5.0.tar.gz
             mkdir ../balsam; tar xf 0.5.0.tar.gz -C ../balsam;
             python install/configure_balsam_install.py
@@ -176,11 +217,8 @@ jobs:
 
         - name: Find MPI, Install libEnsemble, flake8, ulimit adjust
           run: |
-            python install/find_mpi.py
-            mpiexec --version
             pip install -e .
             flake8 libensemble
-            ulimit -Sn 10000
 
         - name: Activate API unit test if using mpich
           if: matrix.mpi-version == 'mpich'
@@ -192,10 +230,10 @@ jobs:
           run: |
            ./libensemble/tests/run-tests.sh -e -A "-W error" -z -${{ matrix.comms-type }}
 
-        #- name: Run simple tests, Ubuntu, Python 3.11
-        #  if: matrix.python-version == '3.11'
-        #  run: |
-        #   ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }}
+        - name: Run simple tests, Windows
+          if: matrix.os == 'windows-latest'
+          run: |
+           ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }}
 
         - name: Run extensive tests, macOS
           if: matrix.os == 'macos-latest'

.gitignore

@@ -15,6 +15,7 @@ x.log
 *.out
 *.err
 *.stat
+*.csv
 
 .vscode
 build/

CHANGELOG.rst

@@ -19,9 +19,9 @@ New capabilities:
 * Added configuration options for `libE_stats.txt` file. #743
 * Support for `spawn` and `forkserver` multiprocessing start methods. #797
 
- * Note that macOS no longer switches to using `fork`. macOS (since Python 3.8) and Windows
- default to using `spawn`. When using `spawn`, we recommend placing calling script code in
- an ``if __name__ == "__main__":`` block. The multiprocessing interface can be used to switch methods (https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method).
+ * Note that macOS no longer switches to using `fork`. macOS (since Python 3.8) and Windows default to
+   using `spawn`. When using `spawn`, we recommend placing calling script code in an ``if __name__ == "__main__":`` block.
+   The multiprocessing interface can be used to switch methods (https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method).
 
 Updates to example functions:
 

README.rst

@@ -31,8 +31,8 @@ Introduction to libEnsemble
 
 libEnsemble is a Python_ toolkit for coordinating workflows of asynchronous and dynamic ensembles of calculations.
 
-libEnsemble can help users take advantage of massively parallel resources to solve design,
-decision, and inference problems and expand the class of problems that can benefit from
+libEnsemble helps users take advantage of massively parallel resources to solve design,
+decision, and inference problems and expands the class of problems that can benefit from
 increased parallelism.
 
 libEnsemble aims for:
@@ -58,20 +58,23 @@ With a basic familiarity of Python and NumPy_, users can easily incorporate
 any other mathematics, machine-learning, or resource-management libraries into libEnsemble
 workflows.
 
-libEnsemble employs a manager/worker scheme that runs on MPI, multiprocessing,
+libEnsemble employs a manager/worker scheme that communicates via MPI, multiprocessing,
 or TCP. Workers control and monitor any level of work using the aforementioned
 generator and simulator functions, from small subnode tasks to huge many-node computations.
 
 libEnsemble includes an Executor interface so application-launching functions are
 portable, resilient, and flexible; it also automatically detects available nodes
 and cores, and can dynamically assign resources to workers.
 
+libEnsemble performs best on Unix-like systems like Linux and macOS. See the
+:ref:`FAQ<faqwindows>` for more information.
+
 .. before_dependencies_rst_tag
 
 Dependencies
 ~~~~~~~~~~~~
 
-Required dependencies:
+**Required dependencies**:
 
 * Python_ 3.7 or above
 * NumPy_
@@ -83,7 +86,7 @@ When using  ``mpi4py`` for libEnsemble communications:
 * A functional MPI 1.x/2.x/3.x implementation, such as MPICH_, built with shared/dynamic libraries
 * mpi4py_ v2.0.0 or above
 
-Optional dependencies:
+**Optional dependencies**:
 
 * Balsam_
 
@@ -103,6 +106,12 @@ a function-as-a-service platform to which workers can submit remote generator or
 simulator function instances. This feature can help distribute an ensemble
 across systems and heterogeneous resources.
 
+* `psi-j-python`_
+
+As of v0.9.2+dev, libEnsemble features a set of command-line utilities for submitting
+libEnsemble jobs to almost any system or scheduler via a `PSI/J`_ Python interface. tqdm_
+is also required.
+
 The example simulation and generation functions and tests require the following:
 
 * SciPy_
@@ -229,7 +238,8 @@ Resources
 **Further Information:**
 
 - Documentation is provided by ReadtheDocs_.
-- An overview of libEnsemble's structure and capabilities is given in this manuscript_ and poster_
+- Contributions_ to libEnsemble are welcome.
+- An overview of libEnsemble's structure and capabilities is given in this manuscript_ and poster_.
 - Examples of production user functions and complete workflows can be viewed, downloaded, and contributed to in the libEnsemble `Community Examples repository`_.
 
 **Citation:**
@@ -243,7 +253,7 @@ Resources
     author  = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and
                David Bindel and John-Luke Navarro},
     institution = {Argonne National Laboratory},
-    number  = {Revision 0.9.2},
+    number  = {Revision 0.9.2+dev},
     year    = {2022},
     url     = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf}
   }
@@ -305,6 +315,7 @@ See a complete list of `example user scripts`_.
 .. _Community Examples repository: https://github.com/Libensemble/libe-community-examples
 .. _Conda: https://docs.conda.io/en/latest/
 .. _conda-forge: https://conda-forge.org/
+.. _Contributions: https://github.com/Libensemble/libensemble/blob/main/CONTRIBUTING.rst
 .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=main
 .. _DEAP: https://deap.readthedocs.io/en/master/overview.html
 .. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols
@@ -330,6 +341,8 @@ See a complete list of `example user scripts`_.
 .. _petsc4py: https://bitbucket.org/petsc/petsc4py
 .. _PETSc/TAO: http://www.mcs.anl.gov/petsc
 .. _poster: https://figshare.com/articles/libEnsemble_A_Python_Library_for_Dynamic_Ensemble-Based_Computations/12559520
+.. _PSI/J: https://exaworks.org/psij
+.. _psi-j-python: https://github.com/ExaWorks/psi-j-python
 .. _psutil: https://pypi.org/project/psutil/
 .. _PyPI: https://pypi.org
 .. _pytest-cov: https://pypi.org/project/pytest-cov/
@@ -348,6 +361,7 @@ See a complete list of `example user scripts`_.
 .. _tarball: https://github.com/Libensemble/libensemble/releases/latest
 .. _Tasmanian: https://tasmanian.ornl.gov/
 .. _Theta: https://www.alcf.anl.gov/alcf-resources/theta
+.. _tqdm: https://tqdm.github.io/
 .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html
 .. _VTMOP: https://github.com/Libensemble/libe-community-examples#vtmop
 .. _WarpX: https://warpx.readthedocs.io/en/latest/

docs/FAQ.rst

@@ -221,16 +221,67 @@ This effectively puts libEnsemble in silent mode.
 
 See the :ref:`Logger Configuration<logger_config>` docs for more information.
 
-macOS-Specific Errors
----------------------
+macOS and Windows Errors
+------------------------
+
+.. _faqwindows:
+
+**Can I run libEnsemble on Windows**
+
+Although we run many libEnsemble workflows successfully on Windows using both MPI and local comms, we do not
+rigorously support Windows, and recommend unix-like systems as a preference. Windows tends to produce more
+platform-specific issues that are difficult to reproduce and troubleshoot.
+
+Feel free to check our `Github Actions`_ page to see what tests we run regularly on Windows.
+
+.. _`Github Actions`: https://github.com/Libensemble/libensemble/actions
+
+**Windows - How can I run libEnsemble with MPI comms?**
+
+We have run workflows with MPI comms. However, as most MPI distributions have either dropped Windows support
+ (MPICH and Open MPI) or are no longer being maintained (``msmpi``), we cannot guarantee success.
+
+If users wish to try, we recommend experimenting with the many Unix-like emulators, containers, virtual machines,
+and other such systems. The `Installing PETSc On Microsoft Windows`_ documentation contains valuable information.
+
+Otherwise, install ``msmpi`` and ``mpi4py`` from conda and experiment, or use ``local`` comms.
+
+.. _`Installing PETSc On Microsoft Windows`: https://petsc.org/release/install/windows/#recommended-installation-methods
+
+**Windows - 'A required privilege is not held by the client'**
+
+Assuming you were trying to use the ``sim_dir_symlink_files`` or ``gen_dir_symlink_files`` options, this indicates that to
+allow libEnsemble to create symlinks, you need to run your current ``cmd`` shell as administrator.
+
+**"RuntimeError: An attempt has been made to start a new process... this probably means that you are not using fork...
+" if __name__ == '__main__': freeze_support() ...**
+
+You need to place your main calling script code underneath an ``if __name__ == "__main__":`` block.
+
+Explanation: Python chooses one of three methods to start new processes when using multiprocessing
+(``--comms local`` with libEnsemble). These are ``'fork'``, ``'spawn'``, and ``'forkserver'``. ``'fork'``
+is the default on Unix, and in our experience is quicker and more reliable, but ``'spawn'`` is the default
+on Windows and macOS (See the `Python multiprocessing docs`_).
+
+Prior to libEnsemble v0.9.2, if libEnsemble detected macOS, it would automatically switch the multiprocessing
+method to ``'fork'``. We decided to stop doing this to avoid overriding defaults and compatibility issues with
+some libraries.
+
+If you'd prefer to use ``'fork'`` or not reformat your code, you can set the multiprocessing start method via
+the following, placed near the top of your calling script::
+
+  import multiprocessing
+  multiprocessing.set_start_method('fork', force=True)
+
+.. _`Python multiprocessing docs`: https://docs.python.org/3/library/multiprocessing.html
 
-**"Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"**
+**"macOS - Fatal error in MPI_Init_thread: Other MPI error, error stack: ... gethostbyname failed"**
 
 Resolve this by appending ``127.0.0.1   [your hostname]`` to /etc/hosts.
 Unfortunately, ``127.0.0.1   localhost`` isn't satisfactory for preventing this
 error.
 
-**How do I stop the Firewall Security popups when running with the Executor?**
+**macOS - How do I stop the Firewall Security popups when running with the Executor?**
 
 There are several ways to address this nuisance, but all involve trial and error.
 An easy (but insecure) solution is temporarily disabling the firewall through

docs/conf.py

@@ -79,6 +79,7 @@ def __getattr__(cls, name):
 sys.path.append(os.path.abspath("../libensemble/utils"))
 sys.path.append(os.path.abspath("../libensemble/tools"))
 sys.path.append(os.path.abspath("../libensemble/executors"))
+sys.path.append(os.path.abspath("../libensemble/executors/balsam_executors"))
 sys.path.append(os.path.abspath("../libensemble/resources"))
 # print(sys.path)
 

docs/data_structures/libE_specs.rst

@@ -33,7 +33,7 @@ libEnsemble is primarily customized by setting options within a ``libE_specs`` d
             processes are then terminated. multiprocessing default: 1
         'kill_canceled_sims' [bool]:
             Will libE try to kill sims that user functions mark 'cancel_requested' as True.
-            If False, the manager avoid this moderate overhead.
+            If False, the manager avoids this moderate overhead.
             Default: True
 
         Directory management options:
@@ -147,7 +147,9 @@ libEnsemble is primarily customized by setting options within a ``libE_specs`` d
             Distributed mode means workers share nodes with applications.
             Default: False
         'zero_resource_workers' [list of ints]:
-            List of workers that require no resources.
+            List of workers that require no resources. For when a fixed mapping of workers
+            to resources is required. Otherwise, use "num_resource_sets".
+            For use with supported allocation functions.
         'resource_info' [dict]:
             Provide resource information that will override automatically detected resources.
             The allowable fields are given below in 'Overriding Auto-detection'