diff --git a/.flake8 b/.flake8 index 9a77d11b5a..f735159a1a 100644 --- a/.flake8 +++ b/.flake8 @@ -23,7 +23,7 @@ per-file-ignores = libensemble/libensemble/__init__.py:F401 # worker uses regex with chars that resemble escape sequences - libensemble/libE_worker.py:W605 + libensemble/worker.py:W605 # Need to turn of matching probes (before other imports) on some # systems/versions of MPI: @@ -43,4 +43,10 @@ per-file-ignores = examples/calling_scripts/run_libensemble_on_warpx.py:E402 libensemble/gen_funcs/persistent_aposmm.py:E402, E501 libensemble/tests/regression_tests/test_persistent_aposmm*:E402 - libensemble/tests/regression_tests/test_old_aposmm*:E402 + libensemble/tests/deprecated_tests/test_old_aposmm*:E402 + + # Ignoring linelength in test_history.py + libensemble/tests/unit_tests/test_history.py:E501 + + # Allow undefined name '__version__' + setup.py:F821 diff --git a/.github/workflows/libE-ci.yml b/.github/workflows/libE-ci.yml new file mode 100644 index 0000000000..858c99a4ed --- /dev/null +++ b/.github/workflows/libE-ci.yml @@ -0,0 +1,143 @@ +name: libEnsemble-CI +on: [push, pull_request] +jobs: + test-libE: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + comms-type: [m, l, t] + exclude: + - os: macos-latest + python-version: 3.6 + - os: macos-latest + python-version: 3.7 + + env: + HYDRA_LAUNCHER: 'fork' + TERM: xterm-256color + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v2 + - name: Setup conda - Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: condaenv + miniconda-version: "latest" + python-version: ${{ matrix.python-version }} + channels: conda-forge + channel-priority: flexible + auto-update-conda: true + + - name: Install Ubuntu compilers + if: matrix.os == 'ubuntu-16.04' + run: conda install gcc_linux-64 + + # Roundabout solution on macos for proper linking with mpicc + - name: Install macOS compilers and older SDK + if: matrix.os == 'macos-latest' + run: | + wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.14.sdk.tar.xz + mkdir ../sdk; tar xf MacOSX10.14.sdk.tar.xz -C ../sdk + conda install clang_osx-64=9.0.1 + + - name: Install Octave and bc on Ubuntu + if: matrix.os == 'ubuntu-16.04' + run: | + sudo apt-get update + sudo apt-get install octave + sudo apt-get install bc + + - name: Install nlopt, scipy, mpich, mpi4py, blas, psutil + run: | + conda install nlopt + conda install scipy + conda install mpich + conda install mpi4py + conda install libblas libopenblas psutil + + - name: Install with-batch PETSc and petsc4py + if: matrix.python-version == 3.8 + env: + PETSC_CONFIGURE_OPTIONS: '--with-batch' + run: conda install petsc4py + + - name: Install mumps-mpi, PETSc, petsc4py + if: matrix.python-version != 3.8 + run: | + conda install mumps-mpi + conda install petsc + conda install petsc4py + + - name: Install DFO-LS, mpmath, deap, other test dependencies + run: | + python -m pip install --upgrade pip + pip install DFO-LS + pip install mpmath + pip install deap + python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git@development/PCGPwM + pip install flake8 + pip install coverage + pip install pytest + pip install pytest-cov + pip install pytest-timeout + pip install mock + pip install coveralls + + - name: Install Tasmanian on Ubuntu + if: matrix.os == 'ubuntu-16.04' + run: | + pip install scikit-build packaging Tasmanian --user + + - name: Find MPI, Install libEnsemble, flake8, ulimit adjust + run: | + python install/find_mpi.py + mpiexec --version + pip install -e . + flake8 libensemble + ulimit -Sn 10000 + + - name: Run tests, Ubuntu + if: matrix.os == 'ubuntu-16.04' + run: | + ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }} + + # Uncomment the following 2 lines to enable tmate debug sessions + # - name: SSH to GitHub Actions + # uses: P3TERX/ssh2actions@main + + - name: Run tests, macOS + if: matrix.os == 'macos-latest' + env: + CONDA_BUILD_SYSROOT: /Users/runner/work/libensemble/sdk/MacOSX10.14.sdk + run: | + ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }} + + - name: Merge coverage, run Coveralls + env: + COVERALLS_PARALLEL: true + run: | + mv libensemble/tests/.cov* . + coveralls --service=github + + coveralls: + name: Notify coveralls of all jobs completing + needs: test-libE + if: always() + runs-on: ubuntu-16.04 + container: python:3-slim + steps: + - name: Finished + run: | + pip3 install --upgrade coveralls + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 28292b0584..7fae151faa 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,11 @@ code/tests/regression_tests/output/ libensemble.egg-info docs/_build x.log + +build/ + +dist/ + +.spyproject/ + +.hypothesis diff --git a/.travis.yml b/.travis.yml index 18f0bd5a9d..b15dcfc041 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python dist: xenial python: - - 3.5 - 3.6 - 3.7 - 3.8 @@ -12,54 +11,22 @@ env: global: - HYDRA_LAUNCHER=fork - OMPI_MCA_rmaps_base_oversubscribe=yes - jobs: - MPI=mpich - matrix: - - COMMS_TYPE=m # mpi - COMMS_TYPE=l # local - - COMMS_TYPE=t # tcp jobs: include: - - os: osx - osx_image: xcode11.3 - env: MPI=mpich PY=3.7 COMMS_TYPE=m # mpi - language: generic - python: 3.7 - os: osx osx_image: xcode11.3 env: MPI=mpich PY=3.7 COMMS_TYPE=l # local language: generic python: 3.7 - - os: osx - osx_image: xcode11.3 - env: MPI=mpich PY=3.7 COMMS_TYPE=t # tcp - language: generic - python: 3.7 - - os: osx - osx_image: xcode11.3 - env: MPI=mpich PY=3.8 COMMS_TYPE=m # mpi - language: generic - python: 3.8 - os: osx osx_image: xcode11.3 env: MPI=mpich PY=3.8 COMMS_TYPE=l # local language: generic python: 3.8 - - os: osx - osx_image: xcode11.3 - env: MPI=mpich PY=3.8 COMMS_TYPE=t # tcp - language: generic - python: 3.8 fast_finish: true - allow_failures: - - python: 3.8 - - os: osx - env: MPI=mpich PY=3.8 COMMS_TYPE=m # mpi - - os: osx - env: MPI=mpich PY=3.8 COMMS_TYPE=l # mpi - - os: osx - env: MPI=mpich PY=3.8 COMMS_TYPE=t # mpi services: - postgresql @@ -70,9 +37,9 @@ cache: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12.1-MacOSX-x86_64.sh -O miniconda.sh; + wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh; else - wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh -O miniconda.sh; + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" @@ -94,15 +61,12 @@ install: else COMPILERS=gcc_linux-64; MUMPS=mumps-mpi=5.1.2=h5bebb2f_1007; - sudo add-apt-repository -y ppa:octave/stable; - sudo apt-get update -qq; - sudo apt install -y octave; fi - conda install $COMPILERS - conda install libblas libopenblas # Prevent 'File exists' error - - if [[ "$TRAVIS_PYTHON_VERSION" == "3.8" ]]; then + - if [[ "$TRAVIS_PYTHON_VERSION" == "3.8" ]] || [[ "$PY" == "3.8" ]]; then conda install nlopt mpi4py scipy mpich; export PETSC_CONFIGURE_OPTIONS='--with-batch'; conda install petsc4py; @@ -110,16 +74,9 @@ install: conda install nlopt petsc4py petsc $MUMPS mpi4py scipy $MPI; fi - # Begin: Dependencies only for regression tests - - pip install DFO-LS - - pip install deap - conda install psutil - pip install mpmath - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - pip install scikit-build packaging Tasmanian --user; - fi - # End: dependencies only for regression tests - # + - pip install DFO-LS - pip install flake8 - pip install coverage==4.5.4 - pip install pytest @@ -130,19 +87,16 @@ install: - python install/find_mpi.py # Locate compilers. Confirm MPI library - mpiexec --version # Show MPI library details - pip install -e . # Installing libEnsemble - - wget https://github.com/balsam-alcf/balsam/archive/0.3.8.tar.gz - - mkdir ../balsam; tar xf 0.3.8.tar.gz -C ../balsam; + - wget https://github.com/argonne-lcf/balsam/archive/refs/tags/0.4.tar.gz + - mkdir ../balsam; tar xf 0.4.tar.gz -C ../balsam; - python install/configure_balsam_install.py before_script: - flake8 libensemble - - echo "export BALSAM_DB_PATH=~/test-balsam" > setbalsampath.sh - - source setbalsampath.sh # Imperfect method for env var persist after setup - - ulimit -Sn 10000 # More concurrent file descriptors (for persis aposmm) # Run test (-z show output) script: - - ./libensemble/tests/run-tests.sh -A "-W error" -z -$COMMS_TYPE + - ./libensemble/tests/run-tests.sh -y 'test_balsam*' -z -$COMMS_TYPE # Track code coverage after_success: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a02ca6342f..72ea324041 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,57 @@ Release Notes Below are the notes from all libEnsemble releases. +GitHub issues are referenced, and can be viewed with hyperlinks on the `github releases page`_. + +.. _`github releases page`: https://github.com/Libensemble/libensemble/releases + +Release 0.7.2 +------------- + +:Date: May 03, 2021 + +API additions: + +* Active receive option added that allows irregular manager/worker communication patterns. (#527 / #595) +* A mechanism is added for the cancellation/killing of previously issued evaluations. (#528 / #595 / #596) +* A submit function is added in the base ``Executor`` class that runs a serial application locally. (#531 / #595) +* Added libEnsemble history array protected fields: `returned_time`, `last_given_time`, and `last_gen_time`. (#590) +* Updated libE_specs options (``mpi_comm`` and ``profile``). (#547 / #548) +* Explicit seeding of random streams in ``add_unique_random_streams()`` is now possible. (#542 / #545) + +Updates to example functions: + +* Added Surmise calibration generator function and two examples (regression tests). (#595) + +Other changes: + +* Better support for uneven worker to node distribution (including at sub-node level). (#591 / #600) +* Fixed crash when running on Windows. (#534) +* Fixed crash when running with empty `persis_info`. (#571 / #578) +* Error handling has been made more robust. (#592) +* Improve ``H0`` processing (esp. for pre-generated, but not evaluated points). (#536 / #537) +* A global ``sim_id`` is now given, rather than a local count, in _libE_stats.txt_. Also a global gen count is given. (#587, #588) +* Added support for Python 3.9. (#532 / Removed support for Python 3.5. (#562) +* Improve SLURM nodelist detection (more robust). (#560) +* Add check that user does not change protected history fields (Disable via ``libE_specs['safe_mode'] = False``). (#541) +* Added ``print_fields.py`` script for better interrogating the output history files. (#558) +* In examples, ``is_master`` changed to ``is_manager`` to be consistent with manager/worker nomenclature. (#524) + +Documentation: + +* Added tutorial **Borehole Calibration with Selective Simulation Cancellation**. (#581 / #595) + +:Note: + +* Tested platforms include Linux, MacOS, Theta (Cray XC40/Cobalt), Summit (IBM Power9/LSF), Bebop (Cray CS400/Slurm). +* Tested Python versions: (Cpython) 3.6, 3.7, 3.8, 3.9. + +:Known issues: + +* OpenMPI does not work with direct MPI job launches in ``mpi4py`` comms mode, since it does not support nested MPI launches + (Either use local mode or Balsam Executor). +* See known issues section in the documentation for more issues. + Release 0.7.1 ------------- @@ -17,9 +68,9 @@ API additions: * Executor updates: - * Addition of a zero-resource worker option for persistent gens (does not allocate nodes to gen). (#500) - * Multiple applications can be registered to the Executor (and submitted) by name. (#498) - * Wait function added to Tasks. (#499) + * Addition of a zero-resource worker option for persistent gens (does not allocate nodes to gen). (#500) + * Multiple applications can be registered to the Executor (and submitted) by name. (#498) + * Wait function added to Tasks. (#499) * Gen directories can now be created with options analogous to those for sim dirs. (#349 / #489) @@ -47,9 +98,9 @@ Documentation: :Known issues: -* We currently recommended running in Central mode on Bridges as distributed runs are experiencing hangs. +* We currently recommend running in Central mode on Bridges, as distributed runs are experiencing hangs. * OpenMPI does not work with direct MPI job launches in mpi4py comms mode, since it does not support nested MPI launches - (Either use local mode or Balsam job controller). + (Either use local mode or Balsam Executor). * See known issues section in the documentation for more issues. Release 0.7.0 diff --git a/README.rst b/README.rst index 1e5f996b1b..b7128def46 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,9 @@ .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master :target: https://travis-ci.org/Libensemble/libensemble +.. image:: https://github.com/Libensemble/libensemble/workflows/libEnsemble-CI/badge.svg?branch=master + :target: https://github.com/Libensemble/libensemble/actions + .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge.svg?branch=master :target: https://coveralls.io/github/Libensemble/libensemble?branch=master @@ -39,10 +42,10 @@ libEnsemble aims for the following: • Portability and flexibility • Exploitation of persistent data/control flow -The user selects or supplies a function that generates simulation -input as well as a function that performs and monitors the -simulations. For example, the generation function may contain an -optimization routine to generate new simulation parameters on the fly based on +The user selects or supplies a *generator function* that produces +input parameters for a *simulator function* that performs and monitors +simulations. For example, the generator function may contain an +optimization routine to generate new simulation parameters on-the-fly based on the results of previous simulations. Examples and templates of such functions are included in the library. @@ -62,7 +65,7 @@ Dependencies Required dependencies: -* Python_ 3.5 or above +* Python_ 3.6 or above * NumPy_ * psutil_ @@ -83,11 +86,14 @@ libEnsemble can also be run on launch nodes using multiprocessing. The example simulation and generation functions and tests require the following: * SciPy_ +* mpmath_ * petsc4py_ +* DEAP_ * DFO-LS_ * Tasmanian_ * NLopt_ -* PETSc_ - Can optionally be installed by pip along with petsc4py +* `PETSc/TAO`_ - Can optionally be installed by pip along with petsc4py +* Surmise_ PETSc and NLopt must be built with shared libraries enabled and present in ``sys.path`` (e.g., via setting the ``PYTHONPATH`` environment variable). NLopt @@ -131,8 +137,9 @@ Testing ~~~~~~~ The provided test suite includes both unit and regression tests and is run -regularly on +regularly on: +* `GitHub Actions`_ * `Travis CI`_ The test suite requires the mock_, pytest_, pytest-cov_, and pytest-timeout_ @@ -156,8 +163,8 @@ Coverage reports are produced separately for unit tests and regression tests under the relevant directories. For parallel tests, the union of all processors is taken. Furthermore, a combined coverage report is created at the top level, which can be viewed at ``libensemble/tests/cov_merge/index.html`` -after ``run_tests.sh`` is completed. The Travis CI coverage results are -available online at Coveralls_. +after ``run_tests.sh`` is completed. The coverage results are available +online at Coveralls_. .. note:: The executor tests can be run by using the direct-launch or @@ -201,41 +208,86 @@ Resources **Further Information:** - Documentation is provided by ReadtheDocs_. -- A visual overview of libEnsemble is given in this poster_. +- An overview of libEnsemble's structure and capabilities is given in this manuscript_ and poster_ **Citation:** -- Please use the following to cite libEnsemble in a publication: +- Please use the following to cite libEnsemble: .. code-block:: bibtex @techreport{libEnsemble, - author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and - David Bindel and John-Luke Navarro}, - title = {{libEnsemble} Users Manual}, + title = {{libEnsemble} Users Manual}, + author = {Stephen Hudson and Jeffrey Larson and Stefan M. Wild and + David Bindel and John-Luke Navarro}, institution = {Argonne National Laboratory}, - number = {Revision 0.7.1+dev}, - year = {2020}, - url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + number = {Revision 0.7.2+dev}, + year = {2021}, + url = {https://buildmedia.readthedocs.org/media/pdf/libensemble/latest/libensemble.pdf} + } + + @article{Hudson2021, + title = {{libEnsemble}: A Library to Coordinate the Concurrent + Evaluation of Dynamic Ensembles of Calculations}, + author = {Stephen Hudson and Jeffrey Larson and John-Luke Navarro and Stefan Wild}, + journal = {{IEEE} Transactions on Parallel and Distributed Systems}, + year = {2021}, + doi = {10.1109/tpds.2021.3082815} } +**Capabilities:** + +libEnsemble generation capabilities include: + +- APOSMM_ Asynchronously parallel optimization solver for finding multiple minima. Supported local optimization routines include: + + - DFO-LS_ Derivative-free solver for (bound constrained) nonlinear least-squares minimization + - NLopt_ Library for nonlinear optimization, providing a common interface for various methods + - scipy.optimize_ Open-source solvers for nonlinear problems, linear programming, + constrained and nonlinear least-squares, root finding, and curve fitting. + - `PETSc/TAO`_ Routines for the scalable (parallel) solution of scientific applications + +- DEAP_ Distributed evolutionary algorithms +- ECNoise_ Estimating Computational Noise in Numerical Simulations +- Surmise_ Modular Bayesian calibration/inference framework +- Tasmanian_ Toolkit for Adaptive Stochastic Modeling and Non-Intrusive ApproximatioN +- VTMOP_ Fortran package for large-scale multiobjective multidisciplinary design optimization + +libEnsemble has also been used to coordinate many computational expensive +simulations. Select examples include: + +- OPAL_ Object Oriented Parallel Accelerator Library. (See this `IPAC manuscript`_.) +- WarpX_ Advanced electromagnetic particle-in-cell code. (See example `WarpX + libE scripts`_.) + +See a complete list of `example user scripts`_. + .. after_resources_rst_tag +.. _APOSMM: https://link.springer.com/article/10.1007/s12532-017-0131-4 +.. _AWA: https://link.springer.com/article/10.1007/s12532-017-0131-4 .. _Balsam: https://www.alcf.anl.gov/support-center/theta/balsam .. _Conda: https://docs.conda.io/en/latest/ .. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _DEAP: https://deap.readthedocs.io/en/master/overview.html .. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols +.. _ECNoise: https://www.mcs.anl.gov/~wild/cnoise/ +.. _example user scripts: https://libensemble.readthedocs.io/en/docs-capabilities_section/examples/examples_index.html .. _GitHub: https://github.com/Libensemble/libensemble +.. _GitHub Actions: https://github.com/Libensemble/libensemble/actions +.. _IPAC manuscript: https://doi.org/10.18429/JACoW-ICAP2018-SAPAF03 .. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble .. _libEnsemble Slack page: https://libensemble.slack.com +.. _manuscript: https://arxiv.org/abs/2104.08322 .. _mock: https://pypi.org/project/mock .. _mpi4py: https://bitbucket.org/mpi4py/mpi4py .. _MPICH: http://www.mpich.org/ +.. _mpmath: http://mpmath.org/ .. _NLopt documentation: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries .. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt .. _NumPy: http://www.numpy.org +.. _OPAL: http://amas.web.psi.ch/docs/opal/opal_user_guide-1.6.0.pdf .. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _PETSc: http://www.mcs.anl.gov/petsc +.. _PETSc/TAO: http://www.mcs.anl.gov/petsc .. _poster: https://figshare.com/articles/libEnsemble_A_Python_Library_for_Dynamic_Ensemble-Based_Computations/12559520 .. _psutil: https://pypi.org/project/psutil/ .. _PyPI: https://pypi.org @@ -245,10 +297,15 @@ Resources .. _Python: http://www.python.org .. _ReadtheDocs: http://libensemble.readthedocs.org/ .. _SciPy: http://www.scipy.org +.. _scipy.optimize: https://docs.scipy.org/doc/scipy/reference/optimize.html .. _Spack: https://spack.readthedocs.io/en/latest +.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Tasmanian: https://tasmanian.ornl.gov/ .. _Travis CI: https://travis-ci.org/Libensemble/libensemble .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html +.. _VTMOP: https://informs-sim.org/wsc20papers/311.pdf +.. _WarpX: https://warpx.readthedocs.io/en/latest/ +.. _WarpX + libE scripts: https://warpx.readthedocs.io/en/latest/usage/workflows/libensemble.html .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 88e5f0d621..bdc24d9843 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -104,6 +104,11 @@ does libEnsemble hang on certain systems when running with MPI?" For more information see https://bitbucket.org/mpi4py/mpi4py/issues/102/unpicklingerror-on-commrecv-after-iprobe. +**Error in `/bin/python': break adjusted to free malloc space: 0x0000010000000000** + +This error has been encountered on Cori when running with an incorrect installation of ``mpi4py``. +Make sure platform specific instructions are followed (e.g.~ :doc:`Cori`) + libEnsemble Help ---------------- diff --git a/docs/advanced_installation.rst b/docs/advanced_installation.rst index e1eaec86bb..5e85db225a 100644 --- a/docs/advanced_installation.rst +++ b/docs/advanced_installation.rst @@ -103,21 +103,28 @@ Install libEnsemble using the Spack_ distribution:: spack install py-libensemble -The above command will install the required dependencies only. There -are several other optional dependencies that can be specified -through variants. The following line installs libEnsemble -version 0.7.0 with all the variants:: +The above command will install the latest release of libEnsemble with +the required dependencies only. There are other optional +dependencies that can be specified through variants. The following +line installs libEnsemble version 0.7.2 with some common variants +(e.g.~ using :doc:`APOSMM<../examples/aposmm>`): - spack install py-libensemble @0.7.0 +mpi +scipy +petsc4py +nlopt +.. code-block:: bash + + spack install py-libensemble @0.7.2 +mpi +scipy +mpmath +petsc4py +nlopt + +The list of variants can be found by running:: + + spack info py-libensemble On some platforms you may wish to run libEnsemble without ``mpi4py``, using a serial PETSc build. This is often preferable if running on the launch nodes of a three-tier system (e.g. Theta/Summit):: - spack install py-libensemble @0.7.0 +scipy +petsc4py ^py-petsc4py~mpi ^petsc~mpi~hdf5~hypre~superlu-dist + spack install py-libensemble +scipy +mpmath +petsc4py ^py-petsc4py~mpi ^petsc~mpi~hdf5~hypre~superlu-dist The install will create modules for libEnsemble and the dependent -packages. These can be loaded by:: +packages. These can be loaded by running:: spack load -r py-libensemble @@ -139,17 +146,20 @@ For example, if you have an activated Conda environment with Python 3.7 and SciP packages: python: - paths: - python: $CONDA_PREFIX + externals: + - spec: "python" + prefix: $CONDA_PREFIX buildable: False py-numpy: - paths: - py-numpy: $CONDA_PREFIX/lib/python3.7/site-packages/numpy + externals: + - spec: "py-numpy" + prefix: $CONDA_PREFIX/lib/python3.7/site-packages/numpy buildable: False py-scipy: - paths: - py-scipy: $CONDA_PREFIX/lib/python3.7/site-packages/scipy - buildable: False + externals: + - spec: "py-scipy" + prefix: $CONDA_PREFIX/lib/python3.7/site-packages/scipy + buildable: True For more information on Spack builds and any particular considerations for specific systems, see the spack_libe_ repostory. In particular, this diff --git a/docs/bibliography.rst b/docs/bibliography.rst index 33e3263596..2f6461597e 100644 --- a/docs/bibliography.rst +++ b/docs/bibliography.rst @@ -3,3 +3,5 @@ Bibliography .. Note that you can't reference a bib file in a different directory .. bibliography:: references.bib + :list: enumerated + :all: diff --git a/docs/conf.py b/docs/conf.py index e80cf2d38f..4c90e394b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,6 +19,10 @@ # import os import sys +from datetime import datetime + +exec(open('../libensemble/version.py').read()) + if sys.version_info >= (3, 3): from unittest.mock import MagicMock @@ -30,7 +34,7 @@ class Mock(MagicMock): def __getattr__(cls, name): return MagicMock() -MOCK_MODULES = ['argparse', 'numpy', 'mpi4py' , 'dfols', 'scipy', 'numpy.lib', 'numpy.lib.recfunctions', 'math', 'petsc4py', 'PETSc', 'nlopt', 'scipy.spatial', 'scipy.spatial.distance', 'scipy.io', 'deap', 'Tasmanian', 'numpy.linalg', 'mpmath', 'psutil'] +MOCK_MODULES = ['argparse', 'numpy', 'mpi4py', 'dfols', 'scipy', 'numpy.lib', 'numpy.lib.recfunctions', 'math', 'petsc4py', 'PETSc', 'nlopt', 'scipy.spatial', 'scipy.spatial.distance', 'scipy.io', 'scipy.stats', 'deap', 'Tasmanian', 'numpy.linalg', 'mpmath', 'psutil', 'surmise.calibration', 'surmise.emulation'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) #from libensemble import * @@ -68,6 +72,8 @@ def __getattr__(cls, name): # 'sphinx.ext.intersphinx', 'sphinx.ext.imgconverter', 'sphinx.ext.mathjax'] +bibtex_bibfiles = ['references.bib'] +bibtex_default_style = 'unsrt' # autosectionlabel_prefix_document = True # extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.imgconverter'] #breathe_projects = { "libEnsemble": "../code/src/xml/" } @@ -87,6 +93,9 @@ def __getattr__(cls, name): # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] +# Configure bibtex_bibfiles setting for sphinxcontrib-bibtex 2.0.0 +bibtex_bibfiles = ['references.bib'] + # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # @@ -101,7 +110,7 @@ def __getattr__(cls, name): # General information about the project. project = 'libEnsemble' -copyright = '2020 Argonne National Laboratory' +copyright = str(datetime.now().year) + ' Argonne National Laboratory' author = 'Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro' today_fmt = '%B %-d, %Y' @@ -110,9 +119,9 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.7.1+dev' +version = __version__ # The full version, including alpha/beta/rc tags. -release = '0.7.1+dev' +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index b8b2e24027..85ed6863e8 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -6,15 +6,15 @@ alloc_specs Allocation function specifications to be set in the user calling script and passed to main ``libE()`` routine:: - alloc_specs: [dict, optional] : + alloc_specs: [dict, optional]: - 'alloc_f' [func] : + 'alloc_f' [func]: Default: give_sim_work_first - 'in' [list of strings] : + 'in' [list of strings]: Default: None - 'out' [list of tuples] : + 'out' [list of tuples]: Default: [('allocated',bool)] - 'user' [dict] : + 'user' [dict]: Default: {'batch_mode': True} .. note:: diff --git a/docs/data_structures/exit_criteria.rst b/docs/data_structures/exit_criteria.rst index 803f4e15f6..18df6cb705 100644 --- a/docs/data_structures/exit_criteria.rst +++ b/docs/data_structures/exit_criteria.rst @@ -7,22 +7,22 @@ Exit criteria for libEnsemble:: exit_criteria: [dict]: - Optional keys (At least one must be given) : + Optional keys (At least one must be given): - 'sim_max' [int] : + 'sim_max' [int]: Stop when this many new points have been evaluated by sim_f - 'gen_max' [int] : + 'gen_max' [int]: Stop when this many new points have been generated by gen_f - 'elapsed_wallclock_time' [float] : + 'elapsed_wallclock_time' [float]: Stop when this time (since the manager has been initialized) has elapsed - 'stop_val' [(str, float)] : + 'stop_val' [(str, float)]: Stop when H[str] < float for the given (str, float pair) .. seealso:: - From `test_sim_dirs.py`_. + From `test_persistent_aposmm_dfols.py`_. - .. literalinclude:: ../../libensemble/tests/regression_tests/test_old_aposmm_sim_dirs.py + .. literalinclude:: ../../libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py :start-at: exit_criteria :end-before: end_exit_criteria_rst_tag -.. _test_sim_dirs.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_sim_dirs.py +.. _test_persistent_aposmm_dfols.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py diff --git a/docs/data_structures/history_array.rst b/docs/data_structures/history_array.rst index 8d97920a71..a4efaf801b 100644 --- a/docs/data_structures/history_array.rst +++ b/docs/data_structures/history_array.rst @@ -11,7 +11,13 @@ Fields in ``H`` include those specified in ``sim_specs['out']``, ``gen_specs['out']``, and ``alloc_specs['out']``. All values are initiated to 0 for integers, 0.0 for floats, and False for Booleans. -Below are the protected fields used in ``H``: +Below are the protected fields used in ``H``. Other than ``'sim_id'`` and +``cancel_requested``, these fields cannot be overwritten by user functions (unless +``libE_spces['safe_mode']`` is set to ``False``). These fields are be updated +by libEnsemble so user functions can know the current status of a run. For +example, these fields let an allocation function know what points have been +``'given'`` to a worker to be evaluated and what points have ``'returned'`` +from the worker doing an evaluation. .. literalinclude:: ../../libensemble/tools/fields_keys.py :start-at: libE_fields diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index b8180cf6fc..61db6e6d67 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -5,69 +5,81 @@ libE_specs Specifications for libEnsemble:: - libE_specs: [dict, optional] : - 'comms' [string] : + libE_specs: [dict, optional]: + 'comms' [string]: Manager/Worker communications mode. Default: mpi Options are 'mpi', 'local', 'tcp' 'nworkers' [int]: Number of worker processes to spawn (in local/tcp modes) - 'comm' [MPI communicator] : + 'comm' [MPI communicator]: libEnsemble communicator. Default: MPI.COMM_WORLD - 'abort_on_exception' [boolean] : + 'abort_on_exception' [boolean]: In MPI mode, whether to call MPI_ABORT on an exception. Default: True IF False, an exception will be raised by the manager. - 'save_every_k_sims' [int] : + 'save_every_k_sims' [int]: Save history array to file after every k simulated points. - 'save_every_k_gens' [int] : + 'save_every_k_gens' [int]: Save history array to file after every k generated points. - 'sim_dirs_make' [boolean] : + 'sim_dirs_make' [boolean]: Whether to make simulation-specific calculation directories for each sim call. - This will create a directory for each simulation, even if no sim_input_dir is specified. - If False, all workers operate within the ensemble directory described below. + This will create a directory for each simulation, even if no sim_input_dir is + specified. If False, all workers operate within the ensemble directory + described below. Default: True - 'gen_dirs_make' [boolean] : - Whether to make generator-instance specific calculation directories for each gen call. - This will create a directory for each generator call, even if no gen_input_dir is specified. - If False, all workers operate within the ensemble directory. + 'gen_dirs_make' [boolean]: + Whether to make generator-instance specific calculation directories for each + gen call. This will create a directory for each generator call, even if no + gen_input_dir is specified. If False, all workers operate within the ensemble + directory. Default: True - 'ensemble_dir_path' [string] : - Path to main ensemble directory containing calculation directories. - Can serve as single working directory for all workers, or contain calculation directories. + 'ensemble_dir_path' [string]: + Path to main ensemble directory containing calculation directories. Can serve + as single working directory for workers, or contain calculation directories. Default: './ensemble' - 'use_worker_dirs' [boolean] : + 'use_worker_dirs' [boolean]: Whether to organize calculation directories under worker-specific directories. Default: False - 'sim_dir_copy_files' [list] : - List of paths to files or directories to copy into each sim dir, or ensemble dir. - 'sim_dir_symlink_files' [list] : - List of paths to files or directories to symlink into each sim dir. - 'gen_dir_copy_files' [list] : - List of paths to files or directories to copy into each gen dir, or ensemble dir. - 'gen_dir_symlink_files' [list] : - List of paths to files or directories to symlink into each gen dir. - 'ensemble_copy_back' [boolean] : - Whether to copy back directories within ensemble_dir_path back to launch location. - Useful if ensemble_dir placed on node-local storage. + 'sim_dir_copy_files' [list]: + Paths to files or directories to copy into each sim dir, or ensemble dir. + 'sim_dir_symlink_files' [list]: + Paths to files or directories to symlink into each sim dir. + 'gen_dir_copy_files' [list]: + Paths to files or directories to copy into each gen dir, or ensemble dir. + 'gen_dir_symlink_files' [list]: + Paths to files or directories to symlink into each gen dir. + 'ensemble_copy_back' [boolean]: + Whether to copy back directories within ensemble_dir_path back to launch + location. Useful if ensemble_dir placed on node-local storage. Default: False - 'sim_input_dir' [string] : - Copy this directory and it's contents for each simulation-specific directory. - If not using calculation directories, contents are copied to the ensemble directory. - 'gen_input_dir' [string] : - Copy this directory and it's contents for each generator-instance specific directory. - If not using calculation directories, contents are copied to the ensemble directory. - 'profile_worker' [boolean] : + 'sim_input_dir' [string]: + Copy this directory and its contents for each simulation-specific directory. + If not using calculation directories, contents are copied to the ensemble dir. + 'gen_input_dir' [string]: + Copy this directory and its contents for each generator-instance specific dir. + If not using calc directories, contents are copied to the ensemble directory. + 'profile_worker' [boolean]: Profile using cProfile. Default: False - 'disable_log_files' [boolean] : + 'disable_log_files' [boolean]: Disable the creation of 'ensemble.log' and 'libE_stats.txt' log files. Default: False - 'workers' list: + 'workers' [list]: TCP Only: A list of worker hostnames. - 'ip' [String]: + 'ip' [string]: TCP Only: IP address 'port' [int]: TCP Only: Port number - 'authkey' [String]: + 'authkey' [string]: TCP Only: Authkey + 'safe_mode' [boolean]: + Prevents user functions from overwritting protected libE fields. + Default: True + 'use_persis_return' [boolean]: + Adds persistent function H return to managers history array. + Default: False + 'final_fields' [list]: + List of fields in H that the manager will return to persistent + workers along with the PERSIS_STOP tag at the end of the libE run. + Default: None .. note:: The ``ensemble_dir_path`` option can create working directories on local node or diff --git a/docs/data_structures/persis_info.rst b/docs/data_structures/persis_info.rst index 0527c92efb..cd73641289 100644 --- a/docs/data_structures/persis_info.rst +++ b/docs/data_structures/persis_info.rst @@ -5,7 +5,7 @@ persis_info Supply persistent information to libEnsemble:: - persis_info: [dict] : + persis_info: [dict]: Dictionary containing persistent info Holds data that is passed to and from workers updating some state information. A typical example diff --git a/docs/data_structures/work_dict.rst b/docs/data_structures/work_dict.rst index 2ce324081d..0c576183ca 100644 --- a/docs/data_structures/work_dict.rst +++ b/docs/data_structures/work_dict.rst @@ -8,7 +8,7 @@ given to worker ``i``. ``Work[i]`` has the following form:: Work[i]: [dict]: - Required keys : + Required keys: 'persis_info' [dict]: Any persistent info to be sent to worker 'i' 'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' 'tag' [int]: 'EVAL_SIM_TAG'/'EVAL_GEN_TAG' if worker 'i' is to call sim/gen_func diff --git a/docs/data_structures/worker_array.rst b/docs/data_structures/worker_array.rst index 8cf1e7894f..201263fe71 100644 --- a/docs/data_structures/worker_array.rst +++ b/docs/data_structures/worker_array.rst @@ -11,6 +11,8 @@ worker array Is the worker active or not 'persis_state' [int]: Is the worker in a persis_state + 'active_recv' [int]: + Is the worker in an active receive state 'blocked' [int]: Is the worker's resources blocked by another calculation @@ -35,7 +37,7 @@ worker blocked by some other calculation 1 0 1 .. note:: * libE receives only from workers with a nonzero 'active' state - * libE calls the alloc_f only if some worker has an 'active' state of zero + * libE calls the alloc_f only if some worker has an 'active' state of zero, or is in an *active receive* state. .. seealso:: For an example allocation function that queries the worker array, see diff --git a/docs/dev_guide/dev_API/manager_module.rst b/docs/dev_guide/dev_API/manager_module.rst index ae4d97e9da..a527555e8d 100644 --- a/docs/dev_guide/dev_API/manager_module.rst +++ b/docs/dev_guide/dev_API/manager_module.rst @@ -1,6 +1,6 @@ Manager Module ============== -.. automodule:: libE_manager +.. automodule:: manager :members: manager_main :undoc-members: diff --git a/docs/dev_guide/dev_API/worker_module.rst b/docs/dev_guide/dev_API/worker_module.rst index 1b68af6687..80a85f4479 100644 --- a/docs/dev_guide/dev_API/worker_module.rst +++ b/docs/dev_guide/dev_API/worker_module.rst @@ -1,6 +1,6 @@ Worker Module ============= -.. automodule:: libE_worker +.. automodule:: worker :members: worker_main .. autoclass:: Worker diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index 40cb1af3c7..f63af295ac 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -17,9 +17,10 @@ Before release date, including a list of supported (tested) platforms. - Version number is updated wherever it appears (and ``+dev`` suffix is removed) - (in ``setup.py``, ``libensemble/__init__.py``, ``README.rst`` and twice in ``docs/conf.py``) + (in ``libensemble/version.py``and ``README.rst``). -- Year in ``README.rst`` under *Citing libEnsemble* and in ``docs/conf.py`` is checked for correctness. +- Year in ``README.rst`` under *Citing libEnsemble* is checked for correctness. + (Note: The year generated in docs by ``docs/conf.py`` should be automatic). - ``setup.py`` and ``libensemble/__init__.py`` are checked to ensure all information is up to date. @@ -28,7 +29,7 @@ Before release - Tests are run with source to be released (this may iterate): - - On-line CI (currently Travis) tests must pass. + - On-line CI (Travis and GitHub Actions) tests must pass. - Scaling tests must be run on HPC platforms listed as supported in release notes. Test variants by platform, launch mechanism, scale, and other factors can diff --git a/docs/examples/alloc_funcs.rst b/docs/examples/alloc_funcs.rst index 2ea4cca6d8..12eed4c536 100644 --- a/docs/examples/alloc_funcs.rst +++ b/docs/examples/alloc_funcs.rst @@ -28,6 +28,8 @@ fast_alloc_to_aposmm :members: :undoc-members: +.. _start_only_persistent_label: + start_only_persistent --------------------- .. automodule:: start_only_persistent diff --git a/docs/examples/aposmm.rst b/docs/examples/aposmm.rst index e676cfa23e..eeebb564ff 100644 --- a/docs/examples/aposmm.rst +++ b/docs/examples/aposmm.rst @@ -5,11 +5,15 @@ Asynchronously Parallel Optimization Solver for finding Multiple Minima (APOSMM) coordinates concurrent local optimization runs in order to identify many local minima. +Required: mpmath_, SciPy_ + +Optional (see below): petsc4py_, nlopt_, DFO-LS_ + Configuring APOSMM ^^^^^^^^^^^^^^^^^^ By default, APOSMM will import several optimizers which require -external packages and MPI. To import only the optimization packages you are using, +external packages. To import only the optimization packages you are using, add the following lines in that calling script, before importing APOSMM:: import libensemble.gen_funcs @@ -37,3 +41,9 @@ LocalOptInterfacer .. automodule:: aposmm_localopt_support :members: :undoc-members: + +.. _SciPy: https://pypi.org/project/scipy +.. _mpmath: https://pypi.org/project/mpmath +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt +.. _petsc4py: https://bitbucket.org/petsc/petsc4py +.. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols \ No newline at end of file diff --git a/docs/examples/deap_nsga2.rst b/docs/examples/deap_nsga2.rst index 5044866826..0d8efd99c8 100644 --- a/docs/examples/deap_nsga2.rst +++ b/docs/examples/deap_nsga2.rst @@ -1,5 +1,10 @@ persistent_deap_nsga2 ---------------------------- +--------------------- + +Required: DEAP_ + .. automodule:: persistent_deap_nsga2 :members: :undoc-members: + +.. _DEAP: https://deap.readthedocs.io/en/master/overview.html diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 90ca20bbad..ca2689ccb3 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -17,3 +17,4 @@ Below are example generation functions available in libEnsemble. tasmanian fd_param_finder vtmop + surmise diff --git a/docs/examples/surmise.rst b/docs/examples/surmise.rst new file mode 100644 index 0000000000..f036883ee3 --- /dev/null +++ b/docs/examples/surmise.rst @@ -0,0 +1,16 @@ +persistent_surmise +------------------ + +Required: Surmise_ + +Note that currently the github fork https://github.com/mosesyhc/surmise should be used:: + + python -m pip install --upgrade git+https://github.com/mosesyhc/surmise.git + +The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an example of the capability to cancel pending simulations. + +.. automodule:: persistent_surmise_calib + :members: + :no-undoc-members: + +.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html diff --git a/docs/examples/tasmanian.rst b/docs/examples/tasmanian.rst index 3a2d269d0e..e66c86aec3 100644 --- a/docs/examples/tasmanian.rst +++ b/docs/examples/tasmanian.rst @@ -1,5 +1,17 @@ persistent_tasmanian --------------------------- + +Required: Tasmanian_, pypackaging_, scikit-build_ + +Note that Tasmanian can be pip installed, but currently must +use either *venv* or *--user* install. + +``E.g: pip install scikit-build packaging Tasmanian --user`` + .. automodule:: persistent_tasmanian :members: :undoc-members: + +.. .. _Tasmanian: https://tasmanian.ornl.gov/ +.. _pypackaging: https://pypi.org/project/pypackaging/ +.. _scikit-build: https://scikit-build.readthedocs.io/en/latest/index.html diff --git a/docs/examples/uniform_or_localopt.rst b/docs/examples/uniform_or_localopt.rst index 1c90a5be30..e999d826de 100644 --- a/docs/examples/uniform_or_localopt.rst +++ b/docs/examples/uniform_or_localopt.rst @@ -1,5 +1,10 @@ uniform_or_localopt ------------------- + +Required: nlopt_ + .. automodule:: uniform_or_localopt :members: :undoc-members: + +.. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt diff --git a/docs/executor/balsam_executor.rst b/docs/executor/balsam_executor.rst index 66a2a5dc90..83b7a0482f 100644 --- a/docs/executor/balsam_executor.rst +++ b/docs/executor/balsam_executor.rst @@ -1,5 +1,5 @@ -Balsam Executor -=============== +Balsam MPI Executor +=================== .. automodule:: balsam_executor :no-undoc-members: diff --git a/docs/executor/executor.rst b/docs/executor/executor.rst index ae606ae7c0..738a7261f2 100644 --- a/docs/executor/executor.rst +++ b/docs/executor/executor.rst @@ -1,5 +1,5 @@ -Executor Module -=============== +Executor Modules +================ .. automodule:: executor :no-undoc-members: @@ -10,11 +10,21 @@ See the Executor APIs for optional arguments. .. toctree:: :maxdepth: 1 - :caption: Executors: + :caption: Alternative Executors: mpi_executor balsam_executor +Executor Class +--------------- + +Only create an object of this class for running local serial-launched applications. +To run MPI applications and use detected resources, use an alternative Executor +class, as shown above. + +.. autoclass:: Executor + :members: __init__, register_calc, submit + .. _task_tag: Task Class @@ -64,5 +74,3 @@ Run configuration attributes - some will be autogenerated: :task.app_args: (string) Application arguments as a string :task.stdout: (string) Name of file where the standard output of the task is written (in task.workdir) :task.stderr: (string) Name of file where the standard error of the task is written (in task.workdir) - -A list of Executor and task functions can be found under the ``executor`` module. diff --git a/docs/executor/mpi_executor.rst b/docs/executor/mpi_executor.rst index 019de4ec3e..d9d5fe97eb 100644 --- a/docs/executor/mpi_executor.rst +++ b/docs/executor/mpi_executor.rst @@ -45,22 +45,22 @@ means. When using the MPI Executor it is possible to override the detected infor The allowable fields are:: - 'mpi_runner' [string] : + 'mpi_runner' [string]: Select runner: 'mpich', 'openmpi', 'aprun', 'srun', 'jsrun', 'custom' All except 'custom' relate to runner classes in libEnsemble. Custom allows user to define their own run-lines but without parsing arguments or making use of auto-resources. - 'runner_name' [string] : + 'runner_name' [string]: Runner name: Replaces run command if present. All runners have a default except for 'custom'. - 'cores_on_node' [tuple (int,int)] : + 'cores_on_node' [tuple (int,int)]: Tuple (physical cores, logical cores) on nodes. - 'subgroup_launch' [Boolean] : + 'subgroup_launch' [Boolean]: Whether MPI runs should be initiatied in a new process group. This needs to be correct for kills to work correctly. Use the standalone test at libensemble/tests/standalone_tests/kill_test to determine correct value for a system. - 'node_file' [string] : + 'node_file' [string]: Name of file containing a node-list. Default is 'node_list'. For example:: diff --git a/docs/function_guides/allocator.rst b/docs/function_guides/allocator.rst index ca29c1b880..30249f0ae2 100644 --- a/docs/function_guides/allocator.rst +++ b/docs/function_guides/allocator.rst @@ -2,7 +2,7 @@ Allocation Functions ==================== Although the included allocation functions, or ``alloc_f``'s are sufficient for -most users, those who want to fine-tune how data is passed to their ``gen_f`` +most users, those who want to fine-tune how data or resources are allocated to their ``gen_f`` and ``sim_f`` can write their own. The ``alloc_f`` is unique since it is called by the libEnsemble's manager instead of a worker. @@ -94,9 +94,16 @@ In practice, the structure of many allocation functions resemble:: return Work, persis_info -The Work dictionary is returned to the manager with ``persis_info``. If ``1`` +The Work dictionary is returned to the manager alongside ``persis_info``. If ``1`` is returned as third value, this instructs the run to stop. +For allocation functions, as with the user functions, the level of complexity can +vary widely. Various scheduling and work distribution features are available in +the existing allocation functions, including prioritization of simulations, +returning evaluation outputs to the generator immediately or in batch, assigning +varying resource sets to evaluations, and other methods of fine-tuned control over +the data available to other user functions. + .. note:: An error occurs when the ``alloc_f`` returns nothing while all workers are idle diff --git a/docs/function_guides/function_guide_index.rst b/docs/function_guides/function_guide_index.rst index 31248910ab..cdd0a4830b 100644 --- a/docs/function_guides/function_guide_index.rst +++ b/docs/function_guides/function_guide_index.rst @@ -2,15 +2,25 @@ Writing User Functions ====================== -libEnsemble coordinates ensembles of calculations performed by three main -functions: a :ref:`Generator Function`, a :ref:`Simulator Function`, -and an :ref:`Allocation Function`, or ``gen_f``, ``sim_f``, and -``alloc_f`` respectively. These are all referred to as User Functions. Although -libEnsemble includes several ready-to-use User Functions like -:doc:`APOSMM<../examples/aposmm>`, it's expected many users will write their own or -adjust included functions for their own use-cases. -These guides describe common development patterns and optional components for -each kind of User Function. +An early part of libEnsemble's design was the decision to divide ensemble steps into +generator and simulator routines as an intuitive way to express problems and their inherent +dependencies. + +libEnsemble was consequently developed to coordinate ensemble computations defined by + +• a *generator function* that produces simulation inputs, +• a *simulator function* that performs and monitors simulations, and +• an *allocation function* that determines when (and with what resources) the other two functions should be invoked. + +Since each of these functions is supplied or selected by libEnsemble's users, they are +typically referred to as *user functions*. User functions need not be written only in +Python: they can (and often do) depend on routines from other +languages. The only restriction for user functions is that their inputs and outputs conform +to the :ref:`user function APIs`. Therefore, the level of computation and complexity of any user function +can vary dramatically based on the user's needs. + +These guides describe common development +patterns and optional components for each kind of User Function: .. toctree:: :maxdepth: 2 diff --git a/docs/function_guides/generator.rst b/docs/function_guides/generator.rst index 0ca4d69d3b..a98f732b0b 100644 --- a/docs/function_guides/generator.rst +++ b/docs/function_guides/generator.rst @@ -36,11 +36,18 @@ alongside ``persis_info``:: return local_H_out, persis_info +Between the output array definition and the function returning, any level and complexity +of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` +to submit applications to parallel resources if necessary, or plug in components from +any other libraries to serve their needs. + .. note:: State ``gen_f`` information like checkpointing should be appended to ``persis_info``. +.. _persistent-gens: + Persistent Generators --------------------- @@ -54,14 +61,16 @@ empty. Many users prefer persistent generators since they do not need to be re-initialized every time their past work is completed and evaluated by a -simulation, and an can evaluate returned simulation results over the course of -an entire libEnsemble routine as a single function instance. +simulation, and can evaluate returned simulation results over the course of +an entire libEnsemble routine as a single function instance. The :doc:`APOSMM<../examples/aposmm>` +optimization generator function included with libEnsemble is persistent so it can +maintain multiple local optimization subprocesses based on results from complete simulations. Functions for a persistent generator to communicate directly with the manager are available in the :ref:`libensemble.tools.gen_support` module. Additional necessary resources are the status tags ``STOP_TAG``, ``PERSIS_STOP``, and -``FINISHED_PERSISTENT_GEN_TAG`` from ``libensemble.message_numbers``, with return -values from the ``gen_support`` functions compared to these tags to determine when +``FINISHED_PERSISTENT_GEN_TAG`` from ``libensemble.message_numbers``. Return +values from the ``gen_support`` functions are compared to these tags to determine when the generator should break its loop and return. Implementing the above functions is relatively simple: @@ -108,5 +117,59 @@ the tag from the manager, it should return with an additional tag:: See :doc:`calc_status<../data_structures/calc_status>` for more information about the message tags. +Active receive mode +------------------- + +By default, a persistent worker (generator in this case) models the manager/worker +communications of a regular worker (i.e., the generator is expected to alternately +receive and send data in a *ping pong* fashion). To have an irregular communication +pattern, a worker can be initiated in *active receive* mode by the allocation +function (see :ref:`start_only_persistent`). + +The user is responsible for ensuring there are no communication deadlocks +in this mode. Note that in manager/worker message exchanges, only the worker-side +receive is blocking. + +Cancelling simulations +---------------------- + +Previously submitted simulations can be cancelled by sending a message to the manager. +To do this as a separate communication, a persistent generator should be +in *active receive* mode to prevent a deadlock. + +To send out cancellations of previously submitted simulations, the generator +can initiate a history array with just the ``sim_id`` and ``cancel_requested`` fields. +Then fill in the ``sim_id``'s to cancel and set the ``cancel_requested`` field to ``True``. +In the following example, ``sim_ids_to_cancel`` is a list of integers. + +.. code-block:: python + + # Send only these fields to existing H rows and libEnsemble will slot in the change. + H_o = np.zeros(len(sim_ids_to_cancel), dtype=[('sim_id', int), ('cancel_requested', bool)]) + H_o['sim_id'] = sim_ids_to_cancel + H_o['cancel_requested'] = True + send_mgr_worker_msg(comm, H_o) + +If a generated point is cancelled by the generator before it has been given to a +worker for evaluation, then it will never be given. If it has already returned from the +simulation, then results can be returned, but the ``cancel_requested`` field remains +as ``True``. However, if the simulation is running when the manager receives the cancellation +request, a kill signal will be sent to the worker. This can be caught and acted upon +by a user function, otherwise it will be ignored. + +The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` gives an example +of the capability to cancel pending simulations. + +Generator initiated shutdown +---------------------------- + +If using a supporting allocation function, the generator can prompt the ensemble to shutdown +by simply exiting the function (e.g., on a test for a converged value). For example, the +allocation function :ref:`start_only_persistent` closes down +the ensemble as soon a persistent generator returns. The usual return values should be given. + +Examples +-------- + Examples of normal and persistent generator functions can be found :doc:`here<../examples/gen_funcs>`. diff --git a/docs/function_guides/simulator.rst b/docs/function_guides/simulator.rst index 4b602b61d4..b8dc1feb0c 100644 --- a/docs/function_guides/simulator.rst +++ b/docs/function_guides/simulator.rst @@ -25,6 +25,11 @@ with ``persis_info`` should be familiar:: return local_H_out, persis_info +Between the output array definition and the function returning, any level and complexity +of computation can be performed. Users are encouraged to use the :doc:`executor<../executor/overview>` +to submit applications to parallel resources if necessary, or plug in components from +other libraries to serve their needs. + Simulator functions can also return a :doc:`calc_status<../data_structures/calc_status>` integer attribute from the ``libensemble.message_numbers`` module to be logged. diff --git a/docs/history_output.rst b/docs/history_output.rst index 6c570664bc..ddfbd5779f 100644 --- a/docs/history_output.rst +++ b/docs/history_output.rst @@ -17,14 +17,20 @@ the final history from libEnsemble will include the following: worker to be evaluated yet? * ``given_time`` [float]: At what time (since the epoch) was this ``gen_f`` - output given to a worker? + output first given to a worker? + +* ``last_given_time`` [float]: At what time (since the epoch) was this ``gen_f`` + output last given to a worker? * ``sim_worker`` [int]: libEnsemble worker that output was given to for evaluation * ``gen_worker`` [int]: libEnsemble worker that generated this ``sim_id`` * ``gen_time`` [float]: At what time (since the epoch) was this entry (or - collection of entries) put into ``H`` by the manager + collection of entries) first put into ``H`` by the manager + +* ``last_gen_time`` [float]: At what time (since the epoch) was this entry (or + collection of entries) first put into ``H`` by the manager * ``returned`` [bool]: Has this worker completed the evaluation of this unit of work? @@ -145,12 +151,12 @@ unspecified options. Each setting will be described in detail here: location. Default: ``False``. * ``'sim_input_dir'``: A path to a directory to copy for simulation - directories. This directory and it's contents are copied to form the base + directories. This directory and its contents are copied to form the base of new simulation directories. If ``'sim_dirs_make'`` is False, this directory's contents are copied into the ensemble directory. * ``'gen_input_dir'``: A path to a directory to copy for generator - directories. This directory and it's contents are copied to form the base + directories. This directory and its contents are copied to form the base of new generator directories. If ``'gen_dirs_make'`` is False, this directory's contents are copied into the ensemble directory. diff --git a/docs/images/centralized_new_detailed.png b/docs/images/centralized_new_detailed.png new file mode 100644 index 0000000000..4ff277e95a Binary files /dev/null and b/docs/images/centralized_new_detailed.png differ diff --git a/docs/images/centralized_new_detailed_balsam.png b/docs/images/centralized_new_detailed_balsam.png new file mode 100644 index 0000000000..427756152c Binary files /dev/null and b/docs/images/centralized_new_detailed_balsam.png differ diff --git a/docs/images/diagram_with_persis.png b/docs/images/diagram_with_persis.png new file mode 100644 index 0000000000..5373a4b392 Binary files /dev/null and b/docs/images/diagram_with_persis.png differ diff --git a/docs/images/distributed_new_detailed.png b/docs/images/distributed_new_detailed.png new file mode 100644 index 0000000000..e7d787f489 Binary files /dev/null and b/docs/images/distributed_new_detailed.png differ diff --git a/docs/images/gen_v_fail_or_cancel.png b/docs/images/gen_v_fail_or_cancel.png new file mode 100644 index 0000000000..b8f308d9c8 Binary files /dev/null and b/docs/images/gen_v_fail_or_cancel.png differ diff --git a/docs/index.rst b/docs/index.rst index 50e63bcaf1..1e5c0ee4f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ tutorials/local_sine_tutorial tutorials/executor_forces_tutorial tutorials/aposmm_tutorial + tutorials/calib_cancel_tutorial examples/examples_index .. toctree:: diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index 00f132ddd1..80273cb86a 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -22,21 +22,31 @@ We now present further information on running and testing libEnsemble. :start-after: before_dependencies_rst_tag :end-before: after_resources_rst_tag +.. _APOSMM: https://link.springer.com/article/10.1007/s12532-017-0131-4 +.. _AWA: https://link.springer.com/article/10.1007/s12532-017-0131-4 .. _Balsam: https://www.alcf.anl.gov/support-center/theta/balsam -.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master .. _Conda: https://docs.conda.io/en/latest/ +.. _Coveralls: https://coveralls.io/github/Libensemble/libensemble?branch=master +.. _DEAP: https://deap.readthedocs.io/en/master/overview.html .. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols +.. _ECNoise: https://www.mcs.anl.gov/~wild/cnoise/ +.. _example user scripts: https://libensemble.readthedocs.io/en/docs-capabilities_section/examples/examples_index.html .. _GitHub: https://github.com/Libensemble/libensemble +.. _GitHub Actions: https://github.com/Libensemble/libensemble/actions +.. _IPAC manuscript: https://doi.org/10.18429/JACoW-ICAP2018-SAPAF03 .. _libEnsemble mailing list: https://lists.mcs.anl.gov/mailman/listinfo/libensemble .. _libEnsemble Slack page: https://libensemble.slack.com +.. _manuscript: https://arxiv.org/abs/2104.08322 .. _mock: https://pypi.org/project/mock .. _mpi4py: https://bitbucket.org/mpi4py/mpi4py .. _MPICH: http://www.mpich.org/ +.. _mpmath: http://mpmath.org/ .. _NLopt documentation: http://ab-initio.mit.edu/wiki/index.php/NLopt_Installation#Shared_libraries .. _nlopt: http://ab-initio.mit.edu/wiki/index.php/NLopt .. _NumPy: http://www.numpy.org +.. _OPAL: http://amas.web.psi.ch/docs/opal/opal_user_guide-1.6.0.pdf .. _petsc4py: https://bitbucket.org/petsc/petsc4py -.. _PETSc: http://www.mcs.anl.gov/petsc +.. _PETSc/TAO: http://www.mcs.anl.gov/petsc .. _poster: https://figshare.com/articles/libEnsemble_A_Python_Library_for_Dynamic_Ensemble-Based_Computations/12559520 .. _psutil: https://pypi.org/project/psutil/ .. _PyPI: https://pypi.org @@ -46,10 +56,15 @@ We now present further information on running and testing libEnsemble. .. _Python: http://www.python.org .. _ReadtheDocs: http://libensemble.readthedocs.org/ .. _SciPy: http://www.scipy.org +.. _scipy.optimize: https://docs.scipy.org/doc/scipy/reference/optimize.html .. _Spack: https://spack.readthedocs.io/en/latest +.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest .. _Tasmanian: https://tasmanian.ornl.gov/ .. _Travis CI: https://travis-ci.org/Libensemble/libensemble .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html +.. _VTMOP: https://informs-sim.org/wsc20papers/311.pdf +.. _WarpX: https://warpx.readthedocs.io/en/latest/ +.. _WarpX + libE scripts: https://warpx.readthedocs.io/en/latest/usage/workflows/libensemble.html .. _xSDK Extreme-scale Scientific Software Development Kit: https://xsdk.info diff --git a/docs/logging.rst b/docs/logging.rst index 14f7b943cb..6dcceb0636 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -11,19 +11,19 @@ file name can also be supplied. To change the logging level to DEBUG, provide the following in the calling scripts:: - from libensemble import libE_logger - libE_logger.set_level('DEBUG') + from libensemble import logger + logger.set_level('DEBUG') Logger messages of MANAGER_WARNING level or higher are also displayed through stderr by default. This boundary can be adjusted as follows:: - from libensemble import libE_logger + from libensemble import logger # Only display messages with level >= ERROR - libE_logger.set_stderr_level('ERROR') + logger.set_stderr_level('ERROR') stderr displaying can be effectively disabled by setting the stderr level to CRITICAL. -.. automodule:: libE_logger +.. automodule:: logger :members: :no-undoc-members: diff --git a/docs/overview_usecases.rst b/docs/overview_usecases.rst index 4306fbca4a..f950133680 100644 --- a/docs/overview_usecases.rst +++ b/docs/overview_usecases.rst @@ -24,6 +24,11 @@ The default ``alloc_f`` tells each available worker to call ``sim_f`` with the highest priority unit of work from ``gen_f``. If a worker is idle and there is no ``gen_f`` output to give, the worker is told to call ``gen_f``. +.. image:: images/diagram_with_persis.png + :alt: libE component diagram + :align: center + :scale: 40 + Example Use Cases ~~~~~~~~~~~~~~~~~ .. begin_usecases_rst_tag @@ -76,7 +81,7 @@ Glossary Here we define some terms used throughout libEnsemble's code and documentation. Although many of these terms seem straight-forward, defining such terms assists with keeping confusion to a minimum when communicating about libEnsemble and -it's capabilities. +its capabilities. * **Manager**: Single libEnsemble process facilitating communication between other processes. Within libEnsemble, the *Manager* process configures and diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index eed570ed59..ec18c3c6f7 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -126,7 +126,7 @@ on Bebop is achieved by running :: sbatch myscript.sh Example submission scripts for running on Bebop in distributed and centralized mode -are also given in the examples_ directory. +are also given in the :doc:`examples`. Debugging Strategies -------------------- @@ -146,4 +146,3 @@ See the LCRC Bebop docs here_ for more information about Bebop. .. _Slurm: https://slurm.schedmd.com/ .. _here: https://www.lcrc.anl.gov/for-users/using-lcrc/running-jobs/running-jobs-on-bebop/ .. _options: https://slurm.schedmd.com/srun.html -.. _examples: https://github.com/Libensemble/libensemble/tree/develop/examples/job_submission_scripts diff --git a/docs/platforms/cori.rst b/docs/platforms/cori.rst index d4c06609ff..caa9bf6397 100644 --- a/docs/platforms/cori.rst +++ b/docs/platforms/cori.rst @@ -13,11 +13,19 @@ Configuring Python and Installation Begin by loading the Python 3 Anaconda_ module:: - module load python/3.7-anaconda-2019.07 + module load python + +Create a conda environment +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can create a conda_ environment in which to install libEnsemble and +all dependencies. If using ``mpi4py``, it is recommended that you clone +the ``lazy-mpi4py`` environment provided by NERSC:: -In many cases this may provide all the dependent packages you need (including -mpi4py). Note that these packages are installed under the ``/global/common`` -file system. This performs best for imported Python packages. + conda create --name my_env --clone lazy-mpi4py + +If you wish to build ``mpi4py``, it will need to be done using the +specific `Python instructions from NERSC`_. Installing libEnsemble ---------------------- @@ -25,61 +33,30 @@ Installing libEnsemble Having loaded the Anaconda Python module, libEnsemble can be installed by one of the following ways. -1. External pip installation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -libEnsemble can be installed locally either with:: - - pip install libensemble --user - -Or, if you have a project directory under ``/global/common/software`` it is -recommended to pip install there, for performance:: - - export PREFIX_PATH=/global/common/software//packages - pip install --install-option="--prefix=$PREFIX_PATH" libensemble - -For the latter option, to ensure you pick up from this install you will need -to prepend to your ``PYTHONPATH`` when running (check the exact ``pythonX.Y`` version):: - - export PYTHONPATH=$PREFIX_PATH/lib//site-packages:$PYTHONPATH - -If libEnsemble is not found, ensure that local paths are being used with:: - - export PYTHONNOUSERSITE=0 - -2. Create a conda environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As an alternative to using an external pip install, you can create your own -conda_ environment in which to install libEnsemble and all dependencies. -If using ``mpi4py``, installation will need to be done using the -`specific instructions from NERSC`_. libEnsemble can then be pip installed -into the environment. +1. Install via **pip** into the environment. .. code-block:: console (my_env) user@cori07:~$ pip install libensemble -Or, install via ``conda``: +2. Install via **conda**: .. code-block:: console (my_env) user@cori07:~$ conda config --add channels conda-forge (my_env) user@cori07:~$ conda install -c conda-forge libensemble -Again, it is preferable to create your conda environment under the ``common`` -file system. This can be done by modifying your ``~/.condarc`` file. -For example, add the lines:: +It is preferable to create your conda environment under the +``/global/common`` file system, which performs best for imported Python +packages. This can be done by modifying your ``~/.condarc`` file. For +example, add the lines:: envs_dirs: - /path/to/my/conda_envs env_prompt: ({name}) -The env_prompt line ensures the whole directory path is not prepended to -your prompt (The ({name}) here is literal, do not substitute). - -If highly parallel runs experience long start-up delays, consider the NERSC -documentation on `scaling Python`_. +The ``env_prompt`` line ensures the whole directory path is not prepended to +your prompt (The ``({name})`` here is literal, do not substitute). See :doc:`here<../advanced_installation>` for more information on advanced options for installing libEnsemble. @@ -90,7 +67,9 @@ Job Submission Cori uses Slurm_ for job submission and management. The two commands you'll likely use the most to initiate jobs are ``salloc`` and ``sbatch`` for running interactively and batch, respectively. libEnsemble runs on the compute nodes -on Cori using either ``multi-processing`` or ``mpi4py``. +on Cori using either ``multi-processing`` or ``mpi4py``. We recommend reading +the `Python instructions from NERSC`_ for specific guidance on using both +``multiprocessing``(used by local mode in libEnsemble) and ``mpi4py``. .. note:: While it is possible to submit jobs from the user ``$HOME`` file system, this @@ -102,7 +81,6 @@ on Cori using either ``multi-processing`` or ``mpi4py``. as this is read-only from compute nodes, but any imported codes (including libEnsemble and gen/sim functions) are best imported from there, especially when running at scale. - See instructions in `scaling Python`_ for more information. Interactive Runs ^^^^^^^^^^^^^^^^ @@ -190,7 +168,15 @@ user application. libEnsemble could be run on more than one node, but here the # leaving 256 nodes for worker launched applications. srun --overcommit --ntasks 129 --nodes=1 python calling_script.py -Example submission scripts are also given in the examples_ directory. +Example submission scripts are also given in the :doc:`examples`. + +Cori FAQ +-------- + +**Error in `/bin/python': break adjusted to free malloc space: 0x0000010000000000** + +This error has been encountered on Cori when running with an incorrect +installation of ``mpi4py``. See instructions above. Additional Information ---------------------- @@ -204,6 +190,4 @@ See the NERSC Cori docs here_ for more information about Cori. .. _Slurm: https://slurm.schedmd.com/ .. _here: https://docs.nersc.gov/jobs/ .. _options: https://slurm.schedmd.com/srun.html -.. _examples: https://github.com/Libensemble/libensemble/tree/develop/examples/job_submission_scripts -.. _specific instructions from NERSC: https://docs.nersc.gov/programming/high-level-environments/python/mpi4py/ -.. _scaling Python: https://docs.nersc.gov/programming/high-level-environments/python/scaling-up +.. _Python instructions from NERSC: https://docs.nersc.gov/development/languages/python/parallel-python/ diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index b16c82d479..23bede0bd1 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -13,22 +13,25 @@ The first mode we refer to as **central** mode, where the libEnsemble manager an are grouped on to one or more dedicated nodes. Workers' launch applications on to the remaining allocated nodes: -.. image:: ../images/centralized_new.png +.. image:: ../images/centralized_new_detailed.png :alt: centralized :scale: 30 :align: center -Alternatively, in **distributed** mode, libEnsemble is launched with the processes -spread across nodes. The worker processes will share nodes with the applications -they launch. There may be multiple nodes per worker, or multiple workers per node: +Alternatively, in **distributed** mode, the libEnsemble (manager/worker) processes +will share nodes with submitted tasks. This enables libEnsemble, using the *mpi4py* +communicator, to be run with the workers spread across nodes so as to be co-located +with their tasks. -.. image:: ../images/distributed_new.png +.. image:: ../images/distributed_new_detailed.png :alt: distributed :scale: 30 :align: center -The distributed approach allows the libEnsemble worker to read files produced by the -application on local node storage. +Configurations with multiple nodes per worker or multiple workers per node are both +common use cases. The distributed approach allows the libEnsemble worker to read files +produced by the application on local node storage. HPC systems that allow only one +application to be launched to a node at any one time prevent distributed configuration. Configuring the Run ------------------- @@ -66,7 +69,7 @@ For example:: mpirun -np 5 -ppn 1 python myscript.py -would launch libEnsemble with 5 processes across 5 nodes. However, the manager would have it's +would launch libEnsemble with 5 processes across 5 nodes. However, the manager would have its own node, which is likely wasteful. More often, a machinefile is used to add the manager to the first node. In the :doc:`examples` directory, you can find an example submission script, configured to run libensemble distributed, with multiple workers per node or multiple nodes @@ -96,7 +99,7 @@ launch nodes and launches tasks submitted by workers. Running libEnsemble on the nodes is potentially more scalable and will better manage ``sim_f`` and ``gen_f`` functions that contain considerable computational work or I/O. - .. image:: ../images/central_balsam.png + .. image:: ../images/centralized_new_detailed_balsam.png :alt: central_balsam :scale: 40 :align: center diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index 85c348aa22..964b5f2569 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -187,12 +187,14 @@ libEnsemble on Summit is achieved by running :: $ bsub myscript.sh +Example submission scripts are also given in the :doc:`examples`. + Launching User Applications from libEnsemble Workers ---------------------------------------------------- Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. This can be accomplished in user ``sim_f`` functions directly. However, it is highly -recommended that the :doc:`executor<../executor/overview>` interface +recommended that the :doc:`Executor<../executor/ex_index>` interface be used inside the ``sim_f`` or ``gen_f``, because this provides a portable interface with many advantages including automatic resource detection, portability, launch failure resilience, and ease of use. diff --git a/docs/platforms/theta.rst b/docs/platforms/theta.rst index b5bd81ddd0..ed9cf382fb 100644 --- a/docs/platforms/theta.rst +++ b/docs/platforms/theta.rst @@ -125,7 +125,7 @@ On Theta, libEnsemble can be launched to two locations: Balsam Executor interfaces with Balsam running on a MOM node for dynamic user-application submission to the compute nodes. - .. image:: ../images/central_balsam.png + .. image:: ../images/centralized_new_detailed_balsam.png :alt: central_Balsam :scale: 40 :align: center @@ -352,7 +352,7 @@ Read the documentation for Balsam here_. .. _ALCF: https://www.alcf.anl.gov/ .. _Theta: https://www.alcf.anl.gov/theta -.. _Balsam: https://www.alcf.anl.gov/support-center/theta/balsam +.. _Balsam: https://balsam.readthedocs.io .. _Cobalt: https://www.alcf.anl.gov/support-center/theta/submit-job-theta .. _`Support Center`: https://www.alcf.anl.gov/support-center/theta .. _here: https://balsam.readthedocs.io/en/latest/ diff --git a/docs/running_libE.rst b/docs/running_libE.rst index bb16321b16..ea53f1f3f5 100644 --- a/docs/running_libE.rst +++ b/docs/running_libE.rst @@ -42,7 +42,7 @@ Limitations of MPI mode If you are launching MPI applications from workers, then MPI is being nested. This is not supported with Open MPI. This can be overcome by using a proxy launcher (see :doc:`Balsam`). This nesting does work, however, -with MPICH and it's derivative MPI implementations. +with MPICH and its derivative MPI implementations. It is also unsuitable to use this mode when running on the **launch** nodes of three-tier systems (e.g. Theta/Summit). In that case ``local`` mode is recommended. @@ -90,9 +90,9 @@ systems or nodes over TCP. The necessary configuration options can be provided t The ``libE_specs`` options for TCP are:: - 'comms' [string] : + 'comms' [string]: 'tcp' - 'nworkers' [int] : + 'nworkers' [int]: Number of worker processes to spawn 'workers' list: A list of worker hostnames. diff --git a/docs/sim_gen_alloc_funcs.rst b/docs/sim_gen_alloc_funcs.rst index 832ae97209..c20c3d11a3 100644 --- a/docs/sim_gen_alloc_funcs.rst +++ b/docs/sim_gen_alloc_funcs.rst @@ -1,7 +1,8 @@ User Function API ----------------- +.. _user_api: -libEnsemble requires functions for generation, simulation and allocation. +libEnsemble requires functions for generation, simulation, and allocation. While libEnsemble provides a default allocation function, the sim and gen functions must be specified. The required API and example arguments are given here. @@ -120,7 +121,7 @@ alloc_f API The alloc_f calculations will be called by libEnsemble with the following API:: - Work, persis_info = alloc_f(W, H, sim_specs, gen_specs, alloc_specs, persis_info) + Work, persis_info, stop_flag = alloc_f(W, H, sim_specs, gen_specs, alloc_specs, persis_info) Parameters: *********** diff --git a/docs/tutorials/aposmm_tutorial.rst b/docs/tutorials/aposmm_tutorial.rst index 2a49de50f6..739304ecb5 100644 --- a/docs/tutorials/aposmm_tutorial.rst +++ b/docs/tutorials/aposmm_tutorial.rst @@ -102,7 +102,7 @@ The most recent version of APOSMM included with libEnsemble is referred to as Persistent APOSMM. Unlike most other user functions that are initiated and completed by workers multiple times based on allocation, a single worker process initiates APOSMM so that it "persists" and keeps running over the course of the -entire libEnsemble routine. APOSMM begins it's own parallel evaluations and +entire libEnsemble routine. APOSMM begins its own parallel evaluations and communicates points back and forth with the manager, which are then given to workers and evaluated by simulation routines. @@ -204,7 +204,7 @@ Also note the following: simulation will likely use ``'x'`` values. (APOSMM performs handshake to ensure that the ``x_on_cube`` that was given to be evaluated is the same the one that is given back.) - * ``'sim_id'`` in ``gen_specs['out']``. APOSMM produces points in it's + * ``'sim_id'`` in ``gen_specs['out']``. APOSMM produces points in its local History array that it will need to update later, and can best reference those points (and avoid a search) if APOSMM produces the IDs itself, instead of libEnsemble. diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst new file mode 100644 index 0000000000..e92711e0be --- /dev/null +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -0,0 +1,312 @@ +=========================================================== +Borehole Calibration with Selective Simulation Cancellation +=========================================================== + +Introduction - Calibration with libEnsemble and a Regression Model +------------------------------------------------------------------ + +This tutorial demonstrates libEnsemble's capability to selectively cancel pending +simulations based on instructions from a calibration Generator Function. +This capability is desirable, especially when evaluations are expensive, since +compute resources may then be more effectively applied towards critical evaluations. + +For a somewhat different approach than libEnsemble's :doc:`other tutorials`, +we'll emphasize the settings, functions, and data fields within the calling script, +:ref:`persistent generator`, Manager, and :ref:`sim_f` +that make this capability possible, rather than outlining a step-by-step process. + +The libEnsemble regression test ``test_persistent_surmise_calib.py`` demonstrates +cancellation of pending simulations, while the ``test_persistent_surmise_killsims.py`` +test demonstrates libEnsemble's capability to also kill running simulations that +have been marked as cancelled. + +Overview of the Calibration Problem +----------------------------------- + +The generator function featured in this tutorial can be found in +``gen_funcs/persistent_surmise_calib.py`` and uses the `surmise`_ library for its +calibration surrogate model interface. + +.. note:: + Note that this repository is a fork of + the main surmise repository, but it retains support for the "PCGPwM" emulation + method used in the generator. surmise is under active development. + +Say there is a computer model :math:`f(\theta, x)` to be calibrated. To calibrate +is to find some parameter :math:`\theta_0` such that :math:`f(\theta_0, x)` closely +resembles data collected from a physical experiment. For example, a (simple) +physical experiment may involve dropping a ball at different heights to study the +gravitational constant, and the corresponding computer model could be the set of +differential equations that governs the drop. In a case where the computation of +the computer model is relatively expensive, we employ a fast surrogate model to +approximate the model and to inform good parameters to test next. Here the computer +model :math:`f(\theta, x)` is accessible only through performing :ref:`sim_f` +evaluations. + +As a convenience for testing, the ``observed`` data values are modelled by calling the ``sim_f`` +for the known true theta, which in this case is the center of a unit hypercube. These values +are therefore stored at the start of libEnsemble's +main :doc:`History array<../history_output>` array, and have associated ``sim_id``'s. + +The generator function ``gen_f`` then samples an initial batch of parameters +:math:`(\theta_1, \ldots, \theta_n)` and constructs a surrogate model. + +For illustration, the initial batch of evaluations are arranged in the following sense: + +.. math:: + + \newcommand{\T}{\mathsf{T}} + \mathbf{f} = \begin{pmatrix} f(\theta_1)^\T \\ \vdots \\ f(\theta_n)^\T \end{pmatrix} + = \begin{pmatrix} f(\theta_1, x_1) & \ldots & f(\theta_1, x_m) \\ \vdots & \ddots & \vdots + \\ f(\theta_n, x_1) & \ldots & f(\theta_n, x_m) \end{pmatrix}. + +The surrogate then generates (suggests) new parameters for ``sim_f`` evaluations, +so the number of parameters :math:`n` grows as more evaluations are scheduled and performed. +As more evaluations are performed and received by ``gen_f``, the surrogate evolves and +suggests parameters closer to :math:`\theta_0` with uncertainty estimates. +The calibration can be terminated when either ``gen_f`` determines it has found +:math:`\theta_0` with some tolerance in the surrounding uncertainty, or computational +resource runs out. At termination, the generator exits and returns, initiating the +shutdown of the libEnsemble routine. + +The following is a pseudocode overview of the generator. Functions directly from +the calibration library used within the generator function have the ``calib:`` prefix. +Helper functions defined to improve the data received by the calibration library by +interfacing with libEnsemble have the ``libE:`` prefix. All other statements are +workflow logic or persistent generator helper functions like ``send`` or ``receive``:: + + 1 libE: calculate observation values and first batch + 2 while STOP_signal not received: + 3 receive: evaluated points + 4 unpack points into 2D Theta x Point structures + 5 if new model condition: + 6 calib: construct new model + 7 else: + 8 wait to receive more points + 9 if some condition: + 10 calib: generate new thetas from model + 11 calib: if error threshold reached: + 12 exit loop - done + 13 send: new points to be evaluated + 14 if any sent points must be obviated: + 15 libE: mark points with cancel request + 16 send: points with cancel request + +Point Cancellation Requests and Dedicated Fields +------------------------------------------------ + +While the generator loops and updates the model based on returned +points from simulations, it detects conditionally if any new Thetas should be generated +from the model, simultaneously evaluating if any *pending* simulations ought to be +cancelled ("obviated"). If so, the generator then calls ``cancel_columns()``:: + + if select_condition(pending): + new_theta, info = select_next_theta(step_add_theta, cal, emu, pending, n_explore_theta) + ... + c_obviate = info['obviatesugg'] # suggested + if len(c_obviate) > 0: + cancel_columns(obs_offset, c_obviate, n_x, pending, comm) + +``obs_offset`` is an offset that excludes the observations when mapping points in surmise +data structures to ``sim_id``'s, ``c_obviate`` is a selection +of columns to cancel, ``n_x`` is the number of ``x`` values, and ``pending`` is used +to check that points marked for cancellation have not already returned. ``comm`` is a +communicator object from :doc:`libE_info<../data_structures/work_dict>` used to send +and receive messages from the Manager. + +Within ``cancel_columns()``, each column in ``c_obviate`` is iterated over, and if a +point is ``pending`` and thus has not yet been evaluated by a simulation, +its ``sim_id`` is appended to a list to be sent to the Manager for cancellation. +A new, separate local :doc:`History array<../history_output>` is defined with the +selected ``'sim_id'`` s and the ``'cancel_requested'`` field set to ``True``. This array is +then sent to the Manager using the ``send_mgr_worker_msg`` persistent generator +helper function. Each of these helper functions is described :ref:`here`. +The entire ``cancel_columns()`` routine is listed below: + +.. code-block:: python + + def cancel_columns(obs_offset, c, n_x, pending, comm): + """Cancel columns""" + sim_ids_to_cancel = [] + columns = np.unique(c) + for c in columns: + col_offset = c*n_x + for i in range(n_x): + sim_id_cancl = obs_offset + col_offset + i + if pending[i, c]: + sim_ids_to_cancel.append(sim_id_cancl) + pending[i, c] = 0 + + # Send only these fields to existing H rows and libEnsemble will slot in the change. + H_o = np.zeros(len(sim_ids_to_cancel), dtype=[('sim_id', int), ('cancel_requested', bool)]) + H_o['sim_id'] = sim_ids_to_cancel + H_o['cancel_requested'] = True + send_mgr_worker_msg(comm, H_o) + +In future calls to the allocation function by the manager, points that would have +been distributed for simulation work but are now marked with "cancel_requested" will not +be processed. The manager will send kill signals to workers that are already processing +cancelled points. These signals can be caught and acted on by the user ``sim_f``; otherwise +they will be ignored. + +Allocation function +------------------- + +The allocation function used in this example is the *only_persistent_gens* function in the +*start_only_persistent* module. The calling script passes the following specification: + +.. code-block:: python + + alloc_specs = {'alloc_f': alloc_f, + 'out': [('given_back', bool)], + 'user': {'init_sample_size': init_sample_size, + 'async_return': True, + 'active_recv_gen': True + } + } + +**async_return** tells the allocation function to return results to the generator as soon +as they come back from evaluation (once the initial sample is complete). + +**init_sample_size** gives the size of the initial sample that is batch returned to the gen. +This is calculated from other parameters in the calling script. + +**active_recv_gen** allows the persistent generator to handle irregular communications (see below). + +By default, workers (including persistent workers), are only +allocated work when they're in an :doc:`idle or non-active state<../data_structures/worker_array>`. +However, since this generator must asynchronously update its model and +cancel pending evaluations, the worker running this generator remains +in an *active receive* state, until it becomes non-persistent. This means +both the manager and persistent worker (generator in this case) must be +prepared for irregular sending /receiving of data. + +.. Manager - Cancellation, History Updates, and Allocation +.. ------------------------------------------------------- +.. +.. Between routines to call the allocation function and distribute allocated work +.. to each Worker, the Manager selects points from the History array that are: +.. +.. 1) Marked as ``'given'`` by the allocation function +.. 2) Marked with ``'cancel_requested'`` by the generator +.. 3) *Not* been marked as ``'returned'`` by the Manager +.. 4) *Not* been marked with ``'kill_sent'`` by the Manager +.. +.. If any points match these characteristics, the Workers that are processing these +.. points are sent ``STOP`` tags and a kill signal. ``'kill_sent'`` +.. is set to ``True`` for each of these points in the Manager's History array. During +.. the subsequent :ref:`start_only_persistent` allocation +.. function calls, any points in the Manager's History array that have ``'cancel_requested'`` +.. as ``True`` are not allocated:: +.. +.. task_avail = ~H['given'] & ~H['cancel_requested'] +.. +.. This ``alloc_f`` also can prioritize allocating points that have +.. higher ``'priority'`` values from the ``gen_f`` values in the local History array:: +.. +.. # Loop through available simulation workers +.. for i in avail_worker_ids(W, persistent=False): +.. +.. if np.any(task_avail): +.. if 'priority' in H.dtype.fields: +.. priorities = H['priority'][task_avail] +.. if gen_specs['user'].get('give_all_with_same_priority'): +.. indexes = (priorities == np.max(priorities)) +.. else: +.. indexes = np.argmax(priorities) +.. else: +.. indexes = 0 + +.. Simulator - Receiving Kill Signal and Cancelling Tasks +.. ------------------------------------------------------ +.. +.. Within the Simulation Function, the :doc:`Executor<../executor/overview>` +.. is used to launch simulations based on points from the generator, +.. and then enters a routine to loop and check for signals from the Manager:: +.. +.. def subproc_borehole_func(H, subp_opts, libE_info): +.. sim_id = libE_info['H_rows'][0] +.. H_o = np.zeros(H.shape[0], dtype=sim_specs['out']) +.. ... +.. exctr = Executor.executor +.. task = exctr.submit(app_name='borehole', app_args=args, stdout='out.txt', stderr='err.txt') +.. calc_status = polling_loop(exctr, task, sim_id) +.. +.. where ``polling_loop()`` resembles the following:: +.. +.. def polling_loop(exctr, task, sim_id): +.. calc_status = UNSET_TAG +.. poll_interval = 0.01 +.. +.. # Poll task for finish and poll manager for kill signals +.. while(not task.finished): +.. exctr.manager_poll() +.. if exctr.manager_signal == 'kill': +.. task.kill() +.. calc_status = MAN_SIGNAL_KILL +.. break +.. else: +.. task.poll() +.. time.sleep(poll_interval) +.. +.. if task.state == 'FAILED': +.. calc_status = TASK_FAILED +.. +.. return calc_status +.. +.. While the launched task isn't finished, the simulator function periodically polls +.. both the task's statuses and for signals from the manager via +.. the :ref:`executor.manager_poll()` function. +.. Immediately after ``exctr.manager_signal`` is confirmed as ``'kill'``, the current +.. task is killed and the function returns with the +.. ``MAN_SIGNAL_KILL`` :doc:`calc_status<../data_structures/calc_status>`. +.. This status will be logged in ``libE_stats.txt``. + +Calling Script - Reading Results +-------------------------------- + +Within the libEnsemble calling script, once the main :doc:`libE()<../libe_module>` +function call has returned, it's a simple enough process to view the History rows +that were marked as cancelled:: + + H, persis_info, flag = libE(sim_specs, gen_specs, + exit_criteria, persis_info, + alloc_specs=alloc_specs, + libE_specs=libE_specs) + + if is_master: + print('Cancelled sims', H['cancel_requested']) + +Here's an example graph showing the relationship between scheduled, cancelled (obviated), +failed, and completed simulations requested by the ``gen_f``. Notice that for each +batch of scheduled simulations, most either complete or fail but the rest are +successfully obviated: + +.. image:: ../images/gen_v_fail_or_cancel.png + :alt: surmise_sample_graph + +Please see the ``test_persistent_surmise_calib.py`` regression test for an example +routine using the surmise calibration generator. +The associated simulation function and allocation function are included in +``sim_funcs/surmise_test_function.py`` and ``alloc_funcs/start_only_persistent.py`` respectively. + +Using cancellations to kill running simulations +------------------------------------------------ + +If a generated point is cancelled by the generator before it has been given to a worker for evaluation, +then it will never be given. If it has already returned from simulation, then results can be returned, +but the ``cancel_requested`` field remains as True. However, if the simulation is running when the manager +recevies the cancellation request, a kill signal will be sent to the worker. This can be caught and acted upon +by a user function, otherwise it will be ignored. To demonstrate this, the test ``test_persistent_surmise_killsims.py`` +captures and processes this signal from the manager. + +In order to do this, a compiled version of the borehole function is launched by ``sim_funcs/borehole_kills.py`` +via the :doc:`Executor<../executor/overview>`. As the borehole application used here is serial, we use the +:doc:`Executor base class<../executor/executor>` rather than the commonly used :doc:`MPIExecutor<../executor/mpi_executor>` +class. The base Executor submit routine simply sub-processes a serial application in-place. After the initial +sample batch of evaluations has been processed, an artificial delay is added to the sub-processed borehole to +allow time to receive the kill signal and terminate the application. Killed simulations will be reported at +the end of the test. As this is dependent on timing, the number of killed simulations will vary between runs. +This test is added simply to demonstrate the killing of running simulations and thus uses a reduced number of evaluations. + +.. _surmise: https://github.com/mosesyhc/surmise diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 5c3619d956..572fd040eb 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -39,7 +39,7 @@ then ``python`` and ``pip`` will work in place of ``python3`` and ``pip3``. .. code-block:: bash $ python3 --version - Python 3.6.0 # This should be >= 3.5 + Python 3.6.0 # This should be >= 3.6 .. _Python: https://www.python.org/ diff --git a/docs/tutorials/tutorials.rst b/docs/tutorials/tutorials.rst index efa5ccf910..cc75aabf8e 100644 --- a/docs/tutorials/tutorials.rst +++ b/docs/tutorials/tutorials.rst @@ -6,3 +6,4 @@ Tutorials local_sine_tutorial executor_forces_tutorial aposmm_tutorial + calib_cancel_tutorial diff --git a/docs/welcome.rst b/docs/welcome.rst index 632b5e58cf..3de552d734 100644 --- a/docs/welcome.rst +++ b/docs/welcome.rst @@ -10,6 +10,9 @@ .. image:: https://travis-ci.org/Libensemble/libensemble.svg?branch=master :target: https://travis-ci.org/Libensemble/libensemble + .. image:: https://github.com/Libensemble/libensemble/workflows/init-libEnsemble-CI/badge.svg?branch=develop + :target: https://github.com/Libensemble/libensemble/actions + .. image:: https://coveralls.io/repos/github/Libensemble/libensemble/badge/?maxAge=2592000/?branch=master :target: https://coveralls.io/github/Libensemble/libensemble?branch=master diff --git a/examples/tutorials/aposmm_tutorial_notebook.ipynb b/examples/tutorials/aposmm_tutorial_notebook.ipynb index ba10f0a7d6..6b7e285480 100644 --- a/examples/tutorials/aposmm_tutorial_notebook.ipynb +++ b/examples/tutorials/aposmm_tutorial_notebook.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -199,7 +199,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -216,37 +216,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[0] libensemble.libE (MANAGER_WARNING): \n", - "*******************************************************************************\n", - "User generator script will be creating sim_id.\n", - "Take care to do this sequentially.\n", - "Also, any information given back for existing sim_id values will be overwritten!\n", - "So everything in gen_specs['out'] should be in gen_specs['in']!\n", - "*******************************************************************************\n", - "\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Minima: [[ 0.08993295 -0.71265804]\n", - " [ 1.70360676 -0.79614982]\n", - " [-1.70368421 0.79606073]\n", - " [-0.08988064 0.71270945]\n", - " [-1.60699361 -0.56859108]\n", - " [ 1.60713962 0.56869567]]\n" - ] - } - ], + "outputs": [], "source": [ "H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info,\n", " alloc_specs, libE_specs)\n", diff --git a/examples/tutorials/sine_tutorial_notebook.ipynb b/examples/tutorials/sine_tutorial_notebook.ipynb index a9ac5ac527..a0f15d3c6d 100644 --- a/examples/tutorials/sine_tutorial_notebook.ipynb +++ b/examples/tutorials/sine_tutorial_notebook.ipynb @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -141,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -198,35 +198,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['allocated', 'given_time', 'given', 'gen_worker', 'returned', 'gen_time', 'y', 'sim_id', 'x', 'sim_worker']\n", - "[( True, 1.57565577e+09, True, 3, True, 1.57565577e+09, 0.3000904 , 0, [ 0.30478742], 1)\n", - " ( True, 1.57565577e+09, True, 3, True, 1.57565577e+09, 0.94863306, 1, [ 1.24888694], 2)\n", - " ( True, 1.57565577e+09, True, 3, True, 1.57565577e+09, -0.95041621, 2, [-1.25457157], 3)\n", - " ( True, 1.57565577e+09, True, 3, True, 1.57565577e+09, 0.06491994, 3, [ 0.06496563], 4)\n", - " ( True, 1.57565577e+09, True, 3, True, 1.57565577e+09, 0.70605436, 4, [ 2.35768173], 1)\n", - " ( True, 1.57565577e+09, True, 4, True, 1.57565577e+09, 0.33293422, 5, [ 2.80217903], 2)\n", - " ( True, 1.57565577e+09, True, 4, True, 1.57565577e+09, 0.27961539, 6, [ 0.2833935 ], 3)\n", - " ( True, 1.57565577e+09, True, 4, True, 1.57565577e+09, 0.30075718, 7, [ 2.83610616], 4)\n", - " ( True, 1.57565577e+09, True, 4, True, 1.57565577e+09, 0.96052853, 8, [ 1.28889596], 1)\n", - " ( True, 1.57565577e+09, True, 4, True, 1.57565577e+09, 0.92701483, 9, [ 1.18637295], 2)\n", - " ( True, 1.57565577e+09, True, 1, True, 1.57565577e+09, -0.47755342, 10, [-0.49786797], 3)\n", - " ( True, 1.57565577e+09, True, 1, True, 1.57565577e+09, 0.96919645, 11, [ 1.32194696], 4)\n", - " ( True, 1.57565577e+09, True, 1, True, 1.57565577e+09, -0.14179936, 12, [-2.99931375], 1)\n", - " ( True, 1.57565577e+09, True, 1, True, 1.57565577e+09, -0.92687662, 13, [-1.18600456], 2)\n", - " ( True, 1.57565577e+09, True, 1, True, 1.57565577e+09, -0.85321981, 14, [-2.11946466], 3)\n", - " ( True, 1.57565577e+09, True, 2, True, 1.57565577e+09, -0.37466051, 15, [-0.38403059], 4)]\n" - ] - } - ], + "outputs": [], "source": [ "print([i for i in H.dtype.fields])\n", "print(H[:16])" @@ -234,22 +210,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEWCAYAAAC9qEq5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dfXxcZZ3//9dn0jZpoARoS7hpkyErFoVyt+UeFC0IooC4uoIjFt2aLcgKK6hocL1Zo64/xfJ1t0BEBdwBZRWVIq5IoYvYKi1aKHcFLJlQbkJbaaANDW3y+f1xzrSTdCaZZO4n7+fjkUfm3Mw51zlz5nzmujnXZe6OiIhIMURKnQARERk/FHRERKRoFHRERKRoFHRERKRoFHRERKRoFHRERKRoKiLomFnMzO4udTqSzCxqZm5mE3Lczo1m9rUc3r/ZzFpyScMo92dm9iMze8XMHizWfvPBzL5gZjekTJ9rZs+F5/DIIuy/08xOLfR+CsHMTjGzdcV+bz7k+h3LYb+D7hFm9hszm5enbZ9sZmtSpvN6bZnZY2Z2Sr62N1RON818MrOTgG8BhwD9wBPAZe6+wt3jQLyU6Ss1M1sK/Le777hxuvvuRU7GScBpwAx331LkfefE3b8+ZNa3gUvc/VelSI+ML+7+7mzWMzMHDnL3Z4bZ1u+BWflIl5ndCKxz96tStn9IPradSVnkdMxsD+BO4HvA3sABwFeAvlKmS3bRDHSOJeDkmissgGbgsbG80cxq8pwWoSyvkbJTFefI3Uv+B8wBNg2z/ELggZRpBxYATwObgP8CLGX5xwlySq8AvwWah9n2ScCycDvPAReG898D/AV4NZz/5ZT3RMM0TAin9wZ+BLwQ7vOX6dKdkvY3ha9vBL4Wvt6LIPCuD7dxJ0GOAqCdIPe3FdgM/GeabTUAN4fvTwBXAZHUdBD8un8FeBZ495DzuxZ4LVwWS3Oe/incf3+Yhq+E8z8BPAP8DbgD2H/IsX4y/JyezXD+/wd4CegB7gcOGeaz6gROTZn+MkHuL/UzmQd0ARuAtqHrArVh+h3YAvw1XP4WYGl4HTwGnJ3y3huBa4G7wvecGs5bBPwm3N4fgH2BheE5fhI4cmjaw3V6gakpy44KP7eJaY75GGAlwXXYDVydzbkbY/o+DzweLv8RUBcuO4Xg13By3f2Bn4dpfhb4VMqyyeG+Xwm39ZnU96Y5vl2uEeAagu/cq8BDwMlDPsfbCK7118LPak7K8iOBP4fLfgr8hPA7luX1enGYlteAfwf+juD+8Gq430kZjqOG4Pu1geC79EkG3yOWAvPD128C/i/83DYAPw3n38/O63Iz8KHkuQc+F37WP07zeQz32V1IhnsQ0ApsA94I97d46PeM4PuykODe9kL4ujb1ugAuB14GXgQ+NuL9fqQVivEH7AFsBG4C3g3sNWT5oBMXnrQ7gT2BJoKL/4xw2TnhRfUWguLDq4BlGfbbHF5c5wMTganAESkndDZBbvAwgi/8+4bc4JIX1K8JLvC9wu28faQPPOXGkAw6U4F/AOqBKQQ3lF+mvG8p4UWbYVs3A78K3xsFngL+KSUd2wi+cDXAReEFZMBuBF+oWeG6+5Hhxp/mc3gnwZfmqPDi/B5w/5D0/Y4gKE/OsM2Ph2lOXtyrhrlOOhk56Hyf4MZ3OEFO+S1D101z7iaG18wXgEnhcb2Wck5uJLhBnBheD3XhvA3A34fT9xLcgD8anuOvAfelSztB8LooZdl3ge9lOOblwAXh692B47I5d2NM36PAzPDz+gM7r81TCG9y4fE/BPxbeK5aCG6yp4fLvwn8PtzGzHCbIwWdQdcI8BGC78MEghvaS+y8iX6Z4MfPmeFxfAP4Y7hsEsEPrn8NP9MPEFz3yePI5nr9FcH96BCC62dJeIwNBDf1eRmOYwFBIE+ev/vIHHRuBdrYeS2dlO66TDn324H/CNM8mfRBJ9NndyFZ3oMyXKtfBf4I7ANMJwjA/z4kbV8Nz/eZBD+o9kp3jnZsf7iFxfwjCBI3EkTO7QS/Qhoz3Ox8yAd1G3Bl+Po3hDfblC9JL2lyOwS/Dn6RZfoWAt8dcoObQHCTHkh3osfygaesdwTwSsr0jot26LYIvnxvAG9NWfbPwNKUdDyTsqw+fO++BEFnE0HASxsYMh0P8APgWynTuxN8yaMp6XvnKK6BPcP3NGRYvuPLEE5/mV2DzoyU5Q8C5w1dN83ncDLBjS2SsvxWwtxt+DndPCQtNwLfT5n+F+CJlOnZpOTeGfxF/hDwh/B1TbjvYzIc8/0ERc3TRnPuxpi+BSnTZ7IzF3gKO4POsUBXmu/Rj8LXawl/AIbTrYwcdIa9Rgh+vR+e8jnek7LsrcDr4eu3Ef6YSlm+jJ034Gyu1xNTlj8EfC5l+jvAwgxpvHfI+XsXmYPOzUAHKddquusy5dy/QRh0h34eWXx2F5Jb0PkrcGbKstMJitiT6Xg9eYzhvJdJ+WGU7q8s6nQA3P0Jd7/Q3WcAhxJk4RcO85aXUl73ElxAEORerjGzTWa2iSAbbQT1REPNJDipuzCzY83sPjNbb2Y9BL9kpmXYxt/c/ZVh0joiM6s3s+vNLGFmrxLcbPbMsv5gGsEvjUTKvASDj3nH+XL33vDl7h7Uz3yI4PheNLNfm9nBWSZ7/9R9uvtmghxr6n6fy/RmM6sxs2+a2V/DY+5MOZ6xynRdDGd/4Dl3H0iZN/T8pTuO7pTXr6eZzrTvXwFvNbMDCRpm9Lh7ptaA/wS8GXjSzFaY2Xsh63M32vSlHmOC4LwM1Qzsn/x+hd+xLwCN4fL902xnJIPOrZldYWZPmFlPuP0GBh/X0M+4Lqzr2B943sO7X5r9Z3O9jvUzHc1xf5bgnvRg2FLs48OsC7De3beOsE42n91YDDpnaba90d23p0yP+J0rm6CTyt2fJIjAh47h7c8B/+zue6b8TXb3ZRnW/bsM27mFILc1090bgOsILpR029jbzPZMs2wLQa4CADPbd5h0X07QIuVYd9+D4FcbKfv0tO8KbCD4xdacMq8JeH6Y9+zg7r9199MIcm1PEhRRZeOF1H2a2W4ExSKp+x0u3R8mKA49leDGEk1uKsP6g84nQU4tH14AZppZ6vdh6Pkb7jhGJbyB3EZQjHQBQTl9pnWfdvfzCYo3/gP4WXieR3vusjEz5XUTwXkZ6jmCupfU79cUdz8zXP5imu2MZMe5NbOTCW7K/0hQerAnQdFmNsf1InCAmaWum7r/bK7Xscr6uN39JXf/hLvvT1AiscjM3jTMtrO59jJ9diPdg0ba9qBzRubrImtlEXTM7GAzu9zMZoTTMwnqWf44hs1dB3zezA4Jt9VgZh/MsG4cONXM/tHMJpjZVDM7Ilw2hSAHs9XMjiH4ku/C3V8kKNJbZGZ7mdlEM0sGjIeBQ8zsCDOrIygayGQKwS+pTWa2N/ClIcu7CcqW06Whn+Am1m5mU8ysGfg0QcX5sMys0czOCb+AfQQVigMjvC3pVuBj4fHVAl8H/uTunVm+f0q4z40EX4yhzZqHWgWcF57jOQRl9vnwJ4JfaJ8Nt30KcBZBJXSh3ExQ9HE2wwQdM/uImU0Pc2GbwtkDjP7cZeOTZjYjvP7aCOoph3oQeM3MPmdmk8Mc16FmdnS4/DaC799e4ff5X0aZhikExevrgQlm9m8EdSzZWB6+91Ph5/h+goYYSbler8O5LdzvDDPbC7gy04pm9sHkvY6g6NDZ+Z3L+D0fQabPbqR70Ej7uxW4ysymm9k0grq8Ee8rwymLoENQaXss8Ccz20IQbB4l+PU/Ku7+C4JfhD8Jix0eJWickG7dLoLyz8sJiuFWEVRAQ9CK5atm9hrBib5tmN1eQJDTeJKgTPOycPtPEVSy3UPQIuaBYbaxkKCScAPB8f/vkOXXAB+w4MHM/5fm/f9C8KtmbbifW4AfDrO/pAhBgHqB4By8naChwYjc/R7giwQtmV4kyDWel817QzcTZNefJ6ikHelHxhfDfbxCUM9xyyj2lZG7v0EQZN5NcP4XAR8Nc9wF4e5/ILjR/NndhyuKOQN4zMw2E1wD57n764z+3GXjFuBugmvorwSNDYamux94L0Gd47ME5+sGgtwWBJ9LIlx2N8ME1Ax+S3DtPxVuZyvDFNEOSdsbwPsJgvnfCIqNb09Znuv1Opzvh2l/mKD13O3DrHs0wb1uM0FpyqXuvjZc9mXgprDo8h9Hsf+0n10W96AfEBT1bjKzX6bZ7tcIWk8+AqwOjy2nh21tcPGniBSLmd0L3OIpD/yWMC2dBBXd95Q6LVLdKv9BI5EKFBZHHUVQLyMybpRL8ZrIuGFmNxEUd1zm7q+VOj0ixaTiNRERKRrldEREpGiqrk5n2rRpHo1GS50MEZGK8tBDD21w9+mF3k/VBZ1oNMrKlStLnQwRkYpiZtn0HpEzFa+JiEjRKOiIiEjRKOiIiEjRKOiIiEjRKOiIiEjRKOiIiEjRKOiIiEjRlDTomNkPzexlM3s0w3Izs/9nZs+Y2SNmdlSx0ygiIvlT6pzOjQTjhWTybuCg8K8VuLYIaRKRChNfHSe6MErkKxGiC6PEV8dLnSTJoKRBx93vJxhsKZNzgJs98EdgTzPbrzipE5FKEF8dp3VxK4meBO+c7nzz4AT7bfgI99w/je5uBZ9yU+7d4BzA4FED14XzXixNckSk3LQtaaN3Wy9zp8MVs6CuJpgfGdjImjWtADQ2xkqYQklV6uK1vDCzVjNbaWYr169fX+rkiMgI4nGIRuHUU+P87GdR7rsvwvLl0THlTLp6ugCY37Iz4CQNDPSydm1bHlIs+VLuQed5YGbK9Ixw3iDu3uHuc9x9zvTpBe8kVURyEI9Dayu86U1xrriilWnTEpg5fX0J1qxpHXXgaWpoAmCf2vTL+/q6ck2y5FG5B507gI+GrdiOA3rcXUVrIhUqvjrOvFVRej8TYf6n51FX1zto+VhyJu1z26mfWM/LfemX19Y2jTW5UgAlrdMxs1uBU4BpZrYO+BIwEcDdrwPuAs4EngF6gY+VJqUikqtkhX//7kGg2Wdyf9r1Rpszic0O6mt++dClXDhz46AitkiknpaW9rElWAqipEHH3c8fYbkDnyxSckSkAOKr47QtaSPRM3i4lpf7YN+6XdcfS84kNjtGbHaM7u44a9e20dfXRW1tEy0t7WpEUGbKvfWaiFSwZO6md1vvLstuWDu4tRnknjNpbIwpyJQ5BR0RKZhkc+Z0loQNTT8RrWGfyQPU1SlnMh4o6IhIXqUWcX3zYOeGtTsDzFDLN9XzsZkdvGO2As14Ue6t10SkgnR3x1mzppW+vgTg7FsXFKHNTfMkQ3NDMx1ndexoCCDjg4KOiORFPA6//30bAwODi9PqaoIHN5PqJ9bz3+//bzov6yzvgJN8gjUSCf7H1aVOPijoiEjOkg987r13+ubOjbVgWOXkbpIHlEiAe/C/tVWBJw9UpyMiuYnHefu8Nl7r7+IPL0fo33fX52/q6poZ+FJn8dM2Vm1t0DukAURvL+vmtfF/xIiVecwsZ8rpiMjYhTmCGf0JIjhvvqGfyNbBq1TkA5pd6XNs+/d3KcOTIwUdERm7ITmCxiUw69tQ81INAwPGhg3NzJrVUXnNoJvSP6DaRRO9vcFhy9go6IjI2KXJETQugRPPH+Csswbo6+usvIAD0N4O9fWDZm2hnh/MPZNbb43ywx+OvVfs8U51OiIydk1NQSX7EC/UNNHRQeXWfYQJXzevjf37u+iiiR/MPZMTr7hpRyelyV6xQeP1jIZyOiIydmlyBNTXM+Om9soNOEmxGP93UydT6gc4kE4OmX9XXnrFHu8UdERk7GIx6OiA5mYwC/5XdBZnsNTD22ef9I0LNF7P6CjoiMjIhntQMhaDzk4YGAj+V0nASUoe3uTJ6RsXaLye0VHQEZHh6UFJAFpa2olEBhclRiL1vDjhTKILo0S+EiG6MEp89fg6L6OloCMiw8vwoOR4azfc2Bhj1qwOamubAaO2tpkNk+dxwe9uItGTwHESPQlaF7cq8AzDgnHSqsecOXN85cqVpU6GSPWIRIIczlBmQZHaOBZdGN1lcDoIOjPtvKyz+AnKgZk95O5zCr0f5XREZHgZHpTMOH8c6epJ34gg03xR0BGRkWRoFk17hXVtUwBNDekDb6b5oqAjIiOp8mbRuWif2079xMEBuX5iPe1zFZAzUdARkZFVebPosYrNjtFxVgfNDc07hm6Yt1cHbWfFNAxPBmpIICKSJ8nW5b1/F4e5bdDQhb3axIKD2ll0UXkHajUkEBGpMG1tYcA5qxX2TIA53pDguhfUjDpJQUdEJE+6ughyOJMGP9fkE3ppWzK+nmvKREFHROjujrN8eZSlS9Vlfy6amoAGNaMejoKOyDh3+4qLWfXYBfT1JQDf0WW/As/otbeDvapm1MNR0BEZx+Kr42zbeB21kcENitRl/9jEYrDgoHZsu5pRZ6KgIzKOtS1pY3pt+has6rJ/bBZdFOPH/zi4GXXHWR3EZpd367Vi0cihIuNYV08XL/fBvnW7LlOX/WMXmx1TkMlAOR2RcaypoYkb1sLW/sHz+waMlhYVB0n+KeiIjGPtc9tZvqmeb6+Bl7bCgEP3VmPLlAU0NuqXuuSfgo7IOJbsxuWZN5r58J+Mjz/czPYDfsz7j15U6qRJlVI3OCIiom5wRESk+ijoiIhI0SjoiIwD8dVxogujRL4SIbowqs4npWT0nI5IlYuvjtO6uJXebUEnlImeBK2LWwH0LIkUnXI6IlWubUnbjoCT1LtNvR5LaSjoiFSxeBwSm9TrsZQPBR2RKpUcxZIe9Xos5aOkQcfMzjCzNWb2jJldmWb5hWa23sxWhX/zS5FOkUrU1ga9vcCSdnhDvR5LeShZQwIzqwH+CzgNWAesMLM73P3xIav+1N0vKXoCRSpcV7L0bHXYWGBuWzDAWE8THR9vVyMCKYlStl47BnjG3dcCmNlPgHOAoUFHRMagqQkSiXBidWxH8Gluhth3S5cuGd9KWbx2APBcyvS6cN5Q/2Bmj5jZz8xsZroNmVmrma00s5Xr168vRFpFKk57O9QPLlWjvj6YL1Iq5d6QYDEQdffDgN8BN6Vbyd073H2Ou8+ZPn16URMoUq5iMejoCHI2ZsH/jo5gvkiplLJ47XkgNecyI5y3g7tvTJm8AfhWEdIlUjViMQUZKS+lzOmsAA4yswPNbBJwHnBH6gpmtl/K5NnAE0VMn4iI5FnJcjruvt3MLgF+C9QAP3T3x8zsq8BKd78D+JSZnQ1sB/4GXFiq9IqISO40no6IiGg8HRERqT4KOiIiZazahqXQ0AYiImWqGoelUE5HRKRMVeOwFAo6IiJlKtPwE5U8LIWCjohImco0/EQlD0uhoCNSgR64OM66CVEGLMK6CVEeuLiyK5clvfa57dRPrK5hKRR0RCrMAxfHOfLaVmb0J4jgzOhPcOS1rQo8VSg2O0bHWR00NzRjGFMnT2XyhMlccPsFFduSTQ+HilSYdROizOhP7Dq/ppkZ2zuLnyApiqEt2SDI9XSc1ZGXlmx6OFRE0tq/P30lcqb5Uh2qpSWbgo5IBYnH4TnSVyK/UFO5lcsystQWa3Onw63HwpK3wTcPTtDdXTnFbAo6IhUiHoePfTfO58/YzJaJg5dtoZ7O1sqtXJaRJVuszZ0OV8yCfesgYsH/NWtaKybwKOiIVIj/a7+Yp56+gP/+3430ToD1k2EA6Jw4lb9c1MFJiyrzCXXJTrIl2/wWqKsZvGxgoJennrq0NAkbJXWDI1IJ4nG++/R17LY9aPgz/XXYMhE+8n64tWl3/LsKONUu2VigccNH0i7v799Id3ecxsbyvhaU0xGpBG1tOwJO0m7b4OtLgAY1IBgvYrNj1NU1Z1y+dm35NypQ0BGpBF3pA0tTD0ydqAYE40lLS+a6u76+8v8BoqAjUgma0geW5xqMa85WA4LxpLExxoQJU9Mu27ChiXiZtydQ0BEpY/E4RKMQS7TTa4O7Q+mdaDx35YKK7eJexu6gg64hEhl8PWzdWs9117XT2kpZBx4FHZEyFY9DayskEnALMeZ7B13WjGPQ3Ez9j37MSVcuKnUypQQaG2PMmtXBhg3NDAwYL73UzLe/3cGSJTF6e6GtjKt21A2OSJmKRoOAM1RzM3R2Fjs1Uo4iEUh3CzeDgYHRbUvd4IiMcxnaDmScL+NPhqq+jPPLgYKOSJmqxBuKFFd7O9QPrtqhvj6YX64UdETKTLLxQCIRFJOkKvcbihRXLAYdHUGRqwVVfXR0BPPLlXokECkjycYDvWFnwu7BzcQ9uKG0t5f3DUWKLxarrGtCQUekjLS17Qw4ScmAo8YDUg1UvCZSRtR4QKqdgo5IGVHjAal2CjoiZaQSWyOJjEZWdTpmtg9wIrA/8DrwKLDS3Uf5+JGIDCdZIdzWFhSpNTWp8YBUl2GDjpm9A7gS2Bv4C/AyUAe8D/g7M/sZ8B13f7XQCRUZLyqtNZLIaIyU0zkT+IS771KNaWYTgPcCpwE/L0DaRESkygxbp+Pun0kXcMJl2939l+6ugCMyVsknQSOR4H85dw8skgdZNSQwsx+bWUPKdNTMlhQuWSLjQGo30u7B/3Lvl14kR9m2XnsA+JOZnWlmnwDuBhYWLlki40BbG93H97L8Vli6BJbfCt3Hl3m/9CI5yqr1mrtfb2aPAfcBG4Aj3f2lgqZMpMp1vynBmitgoC6Y7tsX1lwBfDtBY0lTJlI42RavXQD8EPgocCNwl5kdXsB0iVS9tQtqdgScpIG6YL5Itcq277V/AE5y95eBW83sFwTB58hCJUyk2vVN7R/VfJFqkFVOx93fFwac5PSDwLEFS5XIOFBb1zyq+SLVYNigY2ZXmdne6Za5+xtm9k4ze29hkiZS3Vpa2olEBvd5E4nU09KiPm+keo1UvLYaWGxmW4E/A+sJeiQ4CDgCuAf4+lh3bmZnANcANcAN7v7NIctrgZuBvwc2Ah9y986x7k+knDQ2Bt0OrF3bRl9fF7W1TbS0tO+YL1KNzN1HXsnsIIK+1/Yj6HvtCeB+d399zDs2qwGeIujRYB2wAjjf3R9PWedi4DB3X2Bm5wHnuvuHhtvunDlzfOXKlWNNlojIuGRmD7n7nELvJ9sm008DT5tZvbv3jviG7BwDPOPuawHM7CfAOcDjKeucA3w5fP0z4D/NzDybSCkiImUn2ybTx5vZ48CT4fThZrYox30fADyXMr0unJd2HXffDvQAU9Okr9XMVprZyvXr1+eYLJH8i6+OE10YJfKVCNGFUeKr1euAjE/Z9kiwEDidoF4Fd38YeFuhEjVa7t7h7nPcfc706dNLnRyRQeKr47QubiXRk8BxEj0JWhe3KvDIuJT1IG7u/tyQWbk+TPA8MDNlekY4L+06Ya/WDYSBT6RStC1po3fb4FLp3m29tC1Rdzcy/mQbdJ4zsxMAN7OJZnYFQWOCXKwADjKzA81sEnAecMeQde4A5oWvPwDcq/ocqTRdPWk7as84X6SaZRt0FgCfJKhjeZ6gufQnc9lxWEdzCfBbggB2m7s/ZmZfNbOzw9V+AEw1s2eATxMMKCdSUZoamkY1X6SaZdt6bQOQ94cH3P0u4K4h8/4t5fVW4IP53q9IMbXPbad1ceugIrb6ifW0z9VDoDL+ZBV0zGw68Akgmvoed/94YZIlUj1is4Pfa21L2ujq6aKpoYn2ue075ouMJ9l2+Pkr4PcEPRCoN0KRUYrNjinIiJB90Kl3988VNCUiIlL1sm1IcKeZnVnQlIiISNXLNuhcShB4XjezV83sNTN7tZAJExGR6pPteDpT3D3i7pPdfY9weo9CJ66Y4nGIRiESCf7H9bC4iEjeDVunY2YHu/uTZnZUuuXu/ufCJKu44nFobYXesEVrIhFMA8RU9ysikjfDDm1gZh3u3mpm96XM3vEGd39nIRM3FmMZ2iAaDQLNUM3N0NmZl2SJiJS1Yg1tMGzxmruHv/e5FjjH3d8B3EfQ2/MVBU5b0XRl6I0kkVAxm4hIPmXbkOAqd3/VzE4C3gncQBCIqkLTML2RtLYq8IiI5Eu2QSf5QOh7gO+7+6+BSYVJUvG1t0N9ylD15xPnWaL0E+Gx3ih/ulRRR0QkH7INOs+b2fXAh4C7zKx2FO8te7EYdHQEr88nzvdpJUqCCE6UBN/YqOyOiEg+DNuQYMdKZvXAGcBqd3/azPYDZrv73YVO4GiNpSFBUjQKSxNRoqhVgYiML2XRkCDJ3Xvd/XZ3fzqcfrEcA06u2tuhiQytCjK1NhARkaxVTRFZPsRi0Ds1Q6uC4VobiIhIVhR0htj9miGtCiCYbtfYJyIiuVLQGSrZqqC5GcyC/x0d6ppAdhFfHSe6MErkKxGiC6PEV6uxichIsh3aYHyJxRRkZFjx1fFBo4EmehK0Lg6epda4OSKZKaeTBf2ilaHalrQNGn4aoHdbL21L2kqUIpHKoKAzguQv2kRPAsdJ9CS44LZWLr5WgWc86+pJ35ox03wRCah4bQSpv2jPfwS+vgSaenrpmjKPB1bDSYtUlDIeNTU0kejZ9Xmupga1chQZjnI6I0j+cj3/Efj+Yoj2BCct+lo/R12rngrGq/a57dRPHNzKsX5iPe1z1cpRZDgKOiNI/nL9+hLYbdvgZfX0svlSleGPR7HZMTrO6qC5oRnDaG5opuOsDjUiEBlBVt3gVJJcusFJJ746zgW3tbL9a71pI/QARsQH8rY/EZFSKKtucMaz2OwYC/bvoGtKTdrlXagMX0QkWwo6WVh0UYyvD9zEFgaX4W+hnqunqgx/PFCzeZH8UNDJ0tuvj3HJxA46aWYAo5NmLpnYwbHXqAy/2qVrNt+6uFWBR2QMFHSyFIvBqT+KcUpzJxNsgFOaOzn1RzF1XFDlum+/mOYnPsKvj+vl/rfAVa8H8/UgqMjY6DmdUVDvOONL9+0Xs6b+Wgbqgun+feC0U4Cl8LXJehBUZCyU08lRPB4M/haJBP/12E71WLv9+h0BJ2mgDs44MnitB0FFRk85nRzE49DaCr1hF1yJRDANyhFVg75p6ZvCb5sO9X/Vg6AiY6GcTg7a2nYGnKTe3mC+VL7alzPP14OgImOjoPfkeTgAABUTSURBVJODTCNYa2Tr6tDyk92IbB08L7I1mK+AIzI2Cjo5yDSCdSSiup1q0PjB65m1sIbal4ABqH0JZi2sofGD15c6aSIVS3U6OWhvH1ynk9Tfr7qdqhCL0Qg0XtkWZF+bmoIPXR+qyJgp6OQgee+ZNy8INKmSdTu6P1WW7u44a9e20dfXRW1tEy2nttMY6yx1siTFtm3bWLduHVu3bh15ZdlFXV0dM2bMYOLEiSXZvzr8zINIBNKdRjMYUF+gFaP79otZs9t1DNTu/DAjkXpmzeqgsVG/HsrFs88+y5QpU5g6dSpmVurkVBR3Z+PGjbz22msceOCBg5apw88KkqluJ9N8KUPxOE/VXDso4AAMDPSydq2aI5aTrVu3KuCMkZkxderUkuYSFXTyoL0d6gf3BUp9fTBfKsNzt/4z/XukX9bXp+aI5UYBZ+xKfe5KEnTMbG8z+52ZPR3+3yvDev1mtir8u6PY6cxWLAYdHdDcHBSpNTcH06rPqQzxOKz7yBbI8F2srVWWVSRfSpXTuRJY4u4HAUvC6XRed/cjwr+zi5e80YvFoLMzqMPp7FTAqSRtbdC3T4aFDi0tyrLKTv/6r//KwoULd0yffvrpzJ8/f8f05ZdfztVXX5319jo7Ozn00EPzlr62tjZmzpzJ7rvvnrdt5lOpgs45wE3h65uA95UoHSJ0dcHEl9N/FSa8ZmpEUOHy3T/iiSeeyLJlywAYGBhgw4YNPPbYYzuWL1u2jBNOOCGrbW3fvj2ntKR7/1lnncWDDz6Y03YLqVRBp9HdXwxfvwQ0ZlivzsxWmtkfzSxjYDKz1nC9levXr897YqW6NTXB/97wz2l7Hzho+4LSJEryItk/YiIRtDBN9o+YS+A54YQTWL58OQCPPfYYhx56KFOmTOGVV16hr6+PJ554gqOOOgp35zOf+QyHHnoos2fP5qc//SkAS5cu5eSTT+bss8/mrW9966Btr127liOPPJIVK1bQ39/PZz7zGY4++mgOO+wwrr/++hHfD3Dcccex3377jf0AC6xgz+mY2T3AvmkWDWoK5O5uZpnabTe7+/Nm1gLca2ar3f2vQ1dy9w6gA4Im0zkmXcaZ4CHfRfBteNf8Dvr36WfCyzX0P9tK4+cXlTp5koPh+kccaxH4/vvvz4QJE+jq6mLZsmUcf/zxPP/88yxfvpyGhgZmz57NpEmT+PnPf86qVat4+OGH2bBhA0cffTRve9vbAPjzn//Mo48+yoEHHkhnZycAa9as4bzzzuPGG2/k8MMPp6Ojg4aGBlasWEFfXx8nnngi73rXu3Z5f6UpWNBx91MzLTOzbjPbz91fNLP9gLRdK7r78+H/tWa2FDgS2CXoiOQiefNpa1tE+4cX7ex44POlTZfkrlD9I55wwgksW7aMZcuW8elPf5rnn3+eZcuW0dDQwIknngjAAw88wPnnn09NTQ2NjY28/e1vZ8WKFeyxxx4cc8wxgwLG+vXrOeecc7j99tt35F7uvvtuHnnkEX72s58B0NPTw9NPP82kSZN2eX8lKVXx2h3AvPD1POBXQ1cws73MrDZ8PQ04EXi8aCmUcUUNQapToZ6hS9brrF69mkMPPZTjjjuO5cuXZ12fs9tuuw2abmhooKmpiQceeGDHPHfne9/7HqtWrWLVqlU8++yzO3I6Q99fSUoVdL4JnGZmTwOnhtOY2RwzuyFc5y3ASjN7GLgP+Ka7K+iISNYK9QzdCSecwJ133snee+9NTU0Ne++9N5s2bWL58uU7gs7JJ5/MT3/6U/r7+1m/fj33338/xxxzTNrtTZo0iV/84hfcfPPN3HLLLUDQKu7aa69l27ZtADz11FNs2bIlt4SXgZL0vebuG4G5aeavBOaHr5cBs4uctILapV+vlna1jCoyfQbjy86i0/z22Tp79mw2bNjAhz/84UHzNm/ezLRp0wA499xzWb58OYcffjhmxre+9S323XdfnnzyybTb3G233bjzzjs57bTT2H333Zk/fz6dnZ07GiVMnz6dX/7ylyOm7bOf/Sy33HILvb29zJgxg/nz5/PlL385twPOI/W9ViTd3XHWrGllYGBnrab69SoufQbV4YknnuAtb3lLqZNR0dKdQ/W9VmXWrm0bdLMD9etVbPoMREpPQadIMvXfpX69ikefgUjpKegUSab+u9SvV/HoMxApPQWdImlpaScSGdyMJhKpV79eRdDdHWf58ih9fQmG9uqpz0CkuBR0iqSxMcasWR3U1jYDRm1tMxs2dHDssbG89Qklu0o2HggCDoCTDDy1tc1qRCBSZBquuogaG2M7bnDJPqGSXXQk+4QCPZiYT+kaD4BTW9vM8cd3liJJIuOacjolMlyfUJI/ajwg+VbOQxv09vbynve8h4MPPphDDjmEK6/MNGpM6SjolEih+oSSwdR4QOKr40QXRol8JUJ0YZT46tzKsct9aIMrrriCJ598kr/85S/84Q9/4De/+U1O+8g3BZ0SKVSfUDKYGnCMb/HVcVoXt5LoSeA4iZ4ErYtbcwo85Ty0QX19Pe94xzuAoGudo446inXr1o35WAtBdTolEnSnP7iILR99QslgyTo0dX0zPrUtaaN32+By7N5tvbQtaSM2e2zXQKUMbbBp0yYWL17MpZdeOqbjLBQFnRIpVJ9QsqvUBhwyvnT1pC+vzjQ/W+U+tMH27ds5//zz+dSnPkVLS0tOx5pvCjolFIspyIgUUlNDE4meRNr5uRg6tMHMmTP5zne+wx577MHHPvaxEd8/3NAGyaCTHNrg9NNPH7Tu0qVLRxzaoLW1lYMOOojLLrtslEdWeKrTEZGq1T63nfqJg+v06ifW0z43t3Lsch7a4KqrrqKnp2dQC7tyopyOiFStZL1N25I2unq6aGpoon1u+5jrc5LKdWiDdevW0d7ezsEHH8xRRx0FwCWXXDKoSXepaWgDEakoGtogdxraQERExgUFHRERKRoFnUoQjwc9gqpnUBGpcGpIUO7UM6iIVBHldMqdegbdlXJ+IhVLQafcZegBdCCRyEvnhRUnmfNLJMB9Z85PgUekIijolLsMPYB2NZCXzgsrjnJ+UmLlPLQBwBlnnMHhhx/OIYccwoIFC+jv78/btvNBQafctbcHPYGm2DIRvjA3eN27rZdLf1NeHfoVlMaEkNHKc3FsuQ9tcNttt/Hwww/z6KOPsn79ev7nf/4np33km4JOuYvFoKMDmpsZADob4BNnwa2H7Vxl4+sbx09uR2NCyGgUoDi2nIc2ANhjjz2AICC98cYbmNmYj7UQ1HqtEoQ9g7YsjA7qvPD8R+DrS6CpB164Zh58j6pr0dbdHR88LMHVZ9J4wU0aE0KyM1xx7Bi/K5UwtMHpp5/Ogw8+yLvf/W4+8IEPjOk4C0U5nQqS2knh+Y/A9xdDtCf4EGe80l91Ferd3XHWrGmlry8BOH19CdZMu4nuH8+D5mYwC/53dFRdsJU8KVBxbOrQBscffzzHH3/8jumRhjYAMg5tEI/HOfzww4FgaIObb76ZI444gmOPPZaNGzfy9NNPp33/UL/97W958cUX6evr4957783pWPNNQaeCxGbHmDp5KhDkcHbbNmSFKqtQX7u2jYGBwb9SBwZ6WbvfXdDZCQMDwX8FHMmkQMWxQ4c2OO6441i+fHnW9TnDDW2QlBzaYNWqVaxatYpnn312R05npKENAOrq6jjnnHP41a9+NcqjKywFnQpzzbuvoX5iPU09GVaoogr1vr70x5Jpvsgu0jTEyUdxbLkObbB582ZefPFFIKjT+fWvf83BBx+c07Hmm+p0KkyyS/YXvj0vKFIbqooq1Gtrm8KitV3ni2SlQEP0luvQBlu2bOHss8+mr6+PgYEB3vGOd7BgwYKcjjXfNLRBpRraPQ4Ev+A6Oogflv/xQ0ohWaeTWsQWidQza1aHhp8exzS0Qe40tIGMXkpT6tQK9fhh0Lq4lURPAscr+gHSxsYYs2Z1UFvbDBi1tc0KOCIVTjmdKhMd0qw6qbmhmc7LOoufoGHEV8erIkcmxaWcTu5KmdNRnU6V6epJX8meaX6pPPDNizn5P65j7SanqwG+MDdBa2/Qe7YCj0j1UvFalWlqSF/J3tTQRHd3nOXLoyxdGmH58ijd3SUqcovHOerfrqNpkxMheNbo+4vhnId6aVtSPU2+RWRXCjpVpn1uO/UTBzcRrZ9Yz9UnnrnLg5arHruA21dcXNwExuMwbx712wYX6+62LXj2qNxyZCKSXwo6VSY2O0bHWR00NzRjGM0NzXSc1cF+2+/a5UHL2oizbeN1xWtkkGxxl6HX26aezDk1EakOCjpVKDY7RudlnQx8aYDOyzqJzY5lfKCycZLz4cM+wnabwH/axUyYABfnMfMzqEivdh7dx/dmXHfdnjaoqx+RclTuQxsknX322QXZbq4UdMaJTA9U1r4MBmyc28/f33otv7vbeNvbonzjG2PL/aQGmd//fhpPPvnxnUV60/pZcwV0z931fb0Tja7PLVAjAsm7fNdllvvQBgC33347u+++e07bLpSSBB0z+6CZPWZmA2aWsYmemZ1hZmvM7Bkzu7KYaaw2LS3t9A0M7uI8shVabgiCwJoroG9fsAjsu2+CE468gO5TbcTxR+Kr40QXRjl1kbH4nhoef+IjO4JMf/9G3N8YtP5AHaydP2QjNTXU/+jHnHTlovwcrEgobaexa1pzCjzlPrTB5s2bufrqq7nqqqvGfIyFVKom048C7weuz7SCmdUA/wWcBqwDVpjZHe7+eHGSWF0aG2Ns6foDmzZex/RaZ3J3EHAal8DyW4NgkMrrnLXzofH8cPwR2KXrkPjqOK2LWzl+z16umAV1NQNZpaVvn5SJsBcFddophZCx09i1bWN+yLjchzb44he/yOWXX0790D7nykRJcjru/oS7rxlhtWOAZ9x9rQc/l38CnFP41FWv9x+9iO0H/JiPP9zM0R8OAg4MCQIpdszP0Ht125I2erf1Mr8F6mqyT0ft32o0LIEURaE6jS3XoQ1WrVrFX//6V84999ycjq+QyrlO5wDguZTpdeG8XZhZq5mtNLOV69evL0riKlWykcGEBRftmFf7cvp1B81P03t1snnzPrXZ7z8Sqafl5Js0LIEURca6zBw7jS3XoQ2WL1/OypUriUajnHTSSTz11FOccsopYz/QAihY0DGze8zs0TR/ec+tuHuHu89x9znTp0/P9+ar06JFcNFFUFPDgTcE9TupkvU9O6TpvTrZvPnlvuF2NJEJE6aivtOkFFpa2olEBhczRSL1tLRU59AGF110ES+88AKdnZ088MADvPnNb2bp0qU5HWu+FaxOx91PzXETzwMzU6ZnhPMkXxYtgkWL2Bew5LDQWxPUvmy0fN93FL9lGn+kfW47rYtbuWFtsk5n8PKamqm8+c3XKMhIySSvvUFDnre053xNluvQBpWgpB1+mtlS4Ap336WHTjObADwFzCUINiuAD7v7Y0PXTTXeO/zMi3g86/FHkp12vmlSggVvqmHqpH7qapvz8sUWSUcdfuZu3HX4aWbnAt8DpgO/NrNV7n66me0P3ODuZ7r7djO7BPgtUAP8cKSAI3kSi2Vd1xKbHdOzNSKStZIEHXf/BfCLNPNfAM5Mmb4LuKuISRMRkQIq59ZrIiJpVds4YMVU6nOnoCMiFaWuro6NGzeW/OZZidydjRs3UldXN/LKBaJB3ESkosyYMYN169ahZ/LGpq6ujhkzZpRs/wo6IlJRJk6cmPZpfKkMKl4TEZGiUdAREZGiUdAREZGiKWmPBIVgZuuBRA6bmAZsyFNySq2ajgWq63iq6Viguo5nvB5Ls7sXvPPKqgs6uTKzlcXoCqIYqulYoLqOp5qOBarreHQshaXiNRERKRoFHRERKRoFnV11lDoBeVRNxwLVdTzVdCxQXcejYykg1emIiEjRKKcjIiJFo6AjIiJFo6CThpn9u5k9YmarzOzucHC5imRm/5+ZPRkezy/MbM9SpykXZvZBM3vMzAbMrKyagmbLzM4wszVm9oyZXVnq9OTCzH5oZi+b2aOlTkuuzGymmd1nZo+H19ilpU7TWJlZnZk9aGYPh8fylVKnKUl1OmmY2R7u/mr4+lPAW919QYmTNSZm9i7g3nAk1v8AcPfPlThZY2ZmbwEGgOvJMNR5OTOzGoJh2E8D1hEMw36+uz9e0oSNkZm9DdgM3Ozuh5Y6Pbkws/2A/dz9z2Y2BXgIeF8lfjZmZsBu7r7ZzCYCDwCXuvsfS5w05XTSSQac0G5AxUZmd7/b3beHk38ESteneR64+xPuvqbU6cjBMcAz7r7W3d8AfgKcU+I0jZm73w/8rdTpyAd3f9Hd/xy+fg14AjigtKkaGw9sDicnhn9lcR9T0MnAzNrN7DkgBvxbqdOTJx8HflPqRIxzBwDPpUyvo0JvbNXMzKLAkcCfSpuSsTOzGjNbBbwM/M7dy+JYxm3QMbN7zOzRNH/nALh7m7vPBOLAJaVN7fBGOpZwnTZgO8HxlLVsjkekUMxsd+DnwGVDSj0qirv3u/sRBKUbx5hZWRR/jttB3Nz91CxXjQN3AV8qYHJyMtKxmNmFwHuBuV4BlXij+Gwq0fPAzJTpGeE8KQNh/cfPgbi7317q9OSDu28ys/uAM4CSN/gYtzmd4ZjZQSmT5wBPliotuTKzM4DPAme7e2+p0yOsAA4yswPNbBJwHnBHidMk7Kh8/wHwhLtfXer05MLMpidbqprZZIKGK2VxH1PrtTTM7OfALIJWUglggbtX5K9RM3sGqAU2hrP+WKkt8QDM7Fzge8B0YBOwyt1PL22qRsfMzgQWAjXAD929vcRJGjMzuxU4haAL/W7gS+7+g5ImaozM7CTg98Bqgu8+wBfc/a7SpWpszOww4CaCaywC3ObuXy1tqgIKOiIiUjQqXhMRkaJR0BERkaJR0BERkaJR0BERkaJR0BERkaJR0BERkaJR0BERkaJR0BEpMDM7OhzPqM7MdgvHNymLfrBEik0Ph4oUgZl9DagDJgPr3P0bJU6SSEko6IgUQdjP2gpgK3CCu/eXOEkiJaHiNZHimArsDkwhyPGIjEvK6YgUgZndQTBK6IEEQyKX9RhNIoUybsfTESkWM/sosM3dbzGzGmCZmb3T3e8tddpEik05HRERKRrV6YiISNEo6IiISNEo6IiISNEo6IiISNEo6IiISNEo6IiISNEo6IiISNH8/67aPAK0nz4ZAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", diff --git a/install/cleanup-balsam-test.sh b/install/cleanup-balsam-test.sh new file mode 100755 index 0000000000..7fa88f6197 --- /dev/null +++ b/install/cleanup-balsam-test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +source balsamdeactivate diff --git a/install/configure-balsam-test.sh b/install/configure-balsam-test.sh index cf91a146cb..ecee8bc2a3 100755 --- a/install/configure-balsam-test.sh +++ b/install/configure-balsam-test.sh @@ -11,7 +11,7 @@ # Can't run this line in calling Python file. Balsam installation hasn't been # noticed by the Python runtime yet. python -c 'from libensemble.tests.regression_tests.common import modify_Balsam_pyCoverage; modify_Balsam_pyCoverage()' -export EXE=$PWD/libensemble/tests/regression_tests/script_test_balsam_hworld.py +export EXE=script_test_balsam_hworld.py export NUM_WORKERS=2 export WORKFLOW_NAME=libe_test-balsam export LIBE_WALLCLOCK=3 @@ -24,6 +24,9 @@ sudo chown -R postgres:postgres /var/run/postgresql sudo chmod a+w /var/run/postgresql balsam init $HOME/test-balsam sudo chmod -R 700 $HOME/test-balsam/balsamdb + +python -c 'from libensemble.tests.regression_tests.common import modify_Balsam_settings; modify_Balsam_settings()' + source balsamactivate test-balsam # Refresh DB diff --git a/install/configure_balsam_install.py b/install/configure_balsam_install.py index 83cd3e9757..fabd95e75f 100644 --- a/install/configure_balsam_install.py +++ b/install/configure_balsam_install.py @@ -12,7 +12,7 @@ def install_balsam(): here = os.getcwd() - os.chdir('../balsam/balsam-0.3.8') + os.chdir('../balsam/balsam-0.4') subprocess.check_call('pip install -e .'.split()) os.chdir(here) @@ -41,4 +41,3 @@ def configure_coverage(): install_balsam() move_test_balsam('test_balsam_hworld.py') configure_coverage() - subprocess.run('./install/configure-balsam-test.sh'.split()) diff --git a/install/run_travis_locally/quick_run.md b/install/run_travis_locally/quick_run.md index bb935b033c..7eb3565b12 100644 --- a/install/run_travis_locally/quick_run.md +++ b/install/run_travis_locally/quick_run.md @@ -12,7 +12,7 @@ Window 2: Will create and run container. Windows 1 and 2 - name container. E.g: - export CONTAINER=travis-debug-2020-07-20-py3.5 + export CONTAINER=travis-debug-2020-07-20-py3.6 Window 2: @@ -28,11 +28,11 @@ Window 1 Optional - user scripts to help navigate: docker cp ~/.bashrc $CONTAINER:/home/travis docker cp ~/.alias $CONTAINER:/home/travis -WWindow 2 (Example: Do not run tests python 3.5 - git branch feature/register_apps): +WWindow 2 (Example: Do not run tests python 3.6 - git branch feature/register_apps): chown travis:travis /home/travis/build_mpich_libE.sh su - travis - . ./build_mpich_libE.sh -p 3.5 -b feature/register_apps -i + . ./build_mpich_libE.sh -p 3.6 -b feature/register_apps -i Window 2 Optional - user scripts to help navigate: diff --git a/install/test_balsam_hworld.py b/install/test_balsam_hworld.py index d927f547f3..ee3fd45a77 100644 --- a/install/test_balsam_hworld.py +++ b/install/test_balsam_hworld.py @@ -18,6 +18,11 @@ def run_Balsam_job(): subprocess.Popen(runstr.split()) +def build_simfunc(): + buildstring = 'mpicc -o my_simtask.x ../unit_tests/simdir/my_simtask.c' + subprocess.check_call(buildstring.split()) + + def wait_for_job_dir(basedb): sleeptime = 0 limit = 15 @@ -118,9 +123,15 @@ def move_job_coverage(jobdir): # Used by Balsam Coverage config file. Dont evaluate Balsam data dir libepath = os.path.dirname(libensemble.__file__) os.environ['LIBE_PATH'] = libepath + os.environ['BALSAM_DB_PATH'] = '~/test-balsam' basedb = os.environ['HOME'] + '/test-balsam/data/libe_test-balsam' + subprocess.run('../../../install/configure-balsam-test.sh'.split()) + + if not os.path.isfile('./my_simtask.x'): + build_simfunc() + modify_Balsam_worker() modify_Balsam_JobEnv() run_Balsam_job() @@ -131,3 +142,4 @@ def move_job_coverage(jobdir): move_job_coverage(jobdir) print('Test complete.') + subprocess.run('../../../install/cleanup-balsam-test.sh'.split()) diff --git a/libensemble/__init__.py b/libensemble/__init__.py index 26c2af493a..6a4b451277 100644 --- a/libensemble/__init__.py +++ b/libensemble/__init__.py @@ -4,9 +4,9 @@ Library to coordinate the concurrent evaluation of dynamic ensembles of calculations. """ -__version__ = "0.7.1+dev" +from .version import __version__ __author__ = 'Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro' __credits__ = 'Argonne National Laboratory' -from libensemble import libE_logger +from libensemble import logger from .libE import libE, comms_abort diff --git a/libensemble/alloc_funcs/defaults.py b/libensemble/alloc_funcs/defaults.py index e644ba48a3..a5defaf6c9 100644 --- a/libensemble/alloc_funcs/defaults.py +++ b/libensemble/alloc_funcs/defaults.py @@ -1,6 +1,6 @@ from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first alloc_specs = {'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)], + 'out': [], 'user': {'batch_mode': True, 'num_active_gens': 1}} # end_alloc_specs_rst_tag diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index 22481cd00f..dc9d48b074 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -9,6 +9,11 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): is told to call the generator function, provided this wouldn't result in more than ``alloc_specs['user']['num_active_gen']`` active generators. + This fast_alloc variation of give_sim_work_first is useful for cases that + simply iterate through H, issuing evaluations in order and, in particular, + is likely to be faster if there will be many short simulation evaluations, + given that this function contains fewer column length operations. + .. seealso:: `test_fast_alloc.py `_ # noqa """ @@ -17,9 +22,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): gen_count = count_gens(W) for i in avail_worker_ids(W): + # Skip any cancelled points + while persis_info['next_to_give'] < len(H) and H[persis_info['next_to_give']]['cancel_requested']: + persis_info['next_to_give'] += 1 + + # Give sim work if possible if persis_info['next_to_give'] < len(H): - # Give sim work if possible sim_work(Work, i, sim_specs['in'], [persis_info['next_to_give']], []) persis_info['next_to_give'] += 1 @@ -28,6 +37,8 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): # Give gen work persis_info['total_gen_calls'] += 1 gen_count += 1 - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + gen_in = gen_specs.get('in', []) + return_rows = range(len(H)) if gen_in else [] + gen_work(Work, i, gen_in, return_rows, persis_info.get(i)) return Work, persis_info diff --git a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py index d5cc18b326..d556d1fd62 100644 --- a/libensemble/alloc_funcs/fast_alloc_to_aposmm.py +++ b/libensemble/alloc_funcs/fast_alloc_to_aposmm.py @@ -20,6 +20,9 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): gen_count = count_gens(W) for i in avail_worker_ids(W): + # Skip any cancelled points + while persis_info['next_to_give'] < len(H) and H[persis_info['next_to_give']]['cancel_requested']: + persis_info['next_to_give'] += 1 # Find indices of H that are not yet allocated if persis_info['next_to_give'] < len(H): diff --git a/libensemble/alloc_funcs/give_pregenerated_work.py b/libensemble/alloc_funcs/give_pregenerated_work.py index 25af30c5cc..04fa9e4f1a 100644 --- a/libensemble/alloc_funcs/give_pregenerated_work.py +++ b/libensemble/alloc_funcs/give_pregenerated_work.py @@ -18,6 +18,10 @@ def give_pregenerated_sim_work(W, H, sim_specs, gen_specs, alloc_specs, persis_i return Work, persis_info, 1 for i in avail_worker_ids(W): + # Skip any cancelled points + while persis_info['next_to_give'] < len(H) and H[persis_info['next_to_give']]['cancel_requested']: + persis_info['next_to_give'] += 1 + # Give sim work sim_work(Work, i, sim_specs['in'], [persis_info['next_to_give']], []) persis_info['next_to_give'] += 1 diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index e345eab94e..a2308a7039 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -1,6 +1,6 @@ import numpy as np -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_gens +from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_gens, all_returned def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): @@ -17,6 +17,9 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): their resources can be used for a different simulation evaluation. Can give points in highest priority, if ``'priority'`` is a field in ``H``. + If gen_specs['user']['give_all_with_same_priority'] is set to True, then + all points with the same priority value are given as a batch to the sim. + This is the default allocation function if one is not defined. @@ -29,16 +32,16 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): avail_set = set(W['worker_id'][np.logical_and(~W['blocked'], W['active'] == 0)]) + task_avail = ~H['given'] & ~H['cancel_requested'] for i in avail_worker_ids(W): if i not in avail_set: - pass - - elif not np.all(H['allocated']): + continue + if np.any(task_avail): # Pick all high priority, oldest high priority, or just oldest point if 'priority' in H.dtype.fields: - priorities = H['priority'][~H['allocated']] + priorities = H['priority'][task_avail] if gen_specs['user'].get('give_all_with_same_priority'): q_inds = (priorities == np.max(priorities)) else: @@ -46,17 +49,16 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): else: q_inds = 0 - # Get sim ids and check resources needed - sim_ids_to_send = np.nonzero(~H['allocated'])[0][q_inds] - sim_ids_to_send = np.atleast_1d(sim_ids_to_send) + # Get sim ids (indices) and check resources needed + sim_ids_to_send = np.nonzero(task_avail)[0][q_inds] # oldest point(s) nodes_needed = (np.max(H[sim_ids_to_send]['num_nodes']) if 'num_nodes' in H.dtype.names else 1) if nodes_needed > len(avail_set): break - # Assign resources and mark tasks as allocated to workers - sim_work(Work, i, sim_specs['in'], sim_ids_to_send, persis_info[i]) - H['allocated'][sim_ids_to_send] = True + # Assign points to worker and remove from task_avail list. + sim_work(Work, i, sim_specs['in'], sim_ids_to_send, persis_info.get(i)) + task_avail[sim_ids_to_send] = False # Update resource records avail_set.remove(i) @@ -71,16 +73,14 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if gen_count >= alloc_specs['user'].get('num_active_gens', gen_count+1): break - # No gen instances in batch mode if workers still working - still_working = ~H['returned'] - if alloc_specs['user'].get('batch_mode') and np.any(still_working): + # Do not start gen instances in batch mode if workers still working + if alloc_specs['user'].get('batch_mode') and not all_returned(H): break # Give gen work gen_count += 1 - if 'in' in gen_specs and len(gen_specs['in']): - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) - else: - gen_work(Work, i, [], [], persis_info[i]) + gen_in = gen_specs.get('in', []) + return_rows = range(len(H)) if gen_in else [] + gen_work(Work, i, gen_in, return_rows, persis_info.get(i)) return Work, persis_info diff --git a/libensemble/alloc_funcs/inverse_bayes_allocf.py b/libensemble/alloc_funcs/inverse_bayes_allocf.py index 4895b7650f..ac0b60b5b9 100644 --- a/libensemble/alloc_funcs/inverse_bayes_allocf.py +++ b/libensemble/alloc_funcs/inverse_bayes_allocf.py @@ -1,6 +1,7 @@ import numpy as np -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens +from libensemble.tools.alloc_support import (avail_worker_ids, sim_work, gen_work, + count_persis_gens, all_returned) def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_specs, persis_info): @@ -23,7 +24,7 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe # if > 1 persistant generator, assign the correct work to it inds_generated_by_i = (H['gen_worker'] == i) - if np.all(H['returned'][inds_generated_by_i]): + if all_returned(H, inds_generated_by_i): # Has sim_f completed everything from this persistent worker? # Then give back everything in the last batch @@ -36,10 +37,10 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe k = H['batch'][-1] H['weight'][(n*(k-1)):(n*k)] = H['weight'][(n*k):(n*(k+1))] - gen_work(Work, i, ['like'], np.atleast_1d(inds_to_send_back), - persis_info[i], persistent=True) + gen_work(Work, i, ['like'], inds_to_send_back, + persis_info.get(i), persistent=True) - task_avail = ~H['given'] + task_avail = ~H['given'] & ~H['cancel_requested'] for i in avail_worker_ids(W, persistent=False): if np.any(task_avail): @@ -47,14 +48,14 @@ def only_persistent_gens_for_inverse_bayes(W, H, sim_specs, gen_specs, alloc_spe sim_subbatches = H['subbatch'][task_avail] sim_inds = (sim_subbatches == np.min(sim_subbatches)) sim_ids_to_send = np.nonzero(task_avail)[0][sim_inds] - sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), []) + sim_work(Work, i, sim_specs['in'], sim_ids_to_send, []) task_avail[sim_ids_to_send] = False elif gen_count == 0: # Finally, generate points since there is nothing else to do. gen_count += 1 - gen_work(Work, i, gen_specs['in'], [], persis_info[i], + gen_work(Work, i, gen_specs['in'], [], persis_info.get(i), persistent=True) return Work, persis_info diff --git a/libensemble/alloc_funcs/only_one_gen_alloc.py b/libensemble/alloc_funcs/only_one_gen_alloc.py index a9f37b5091..0ac28155f0 100644 --- a/libensemble/alloc_funcs/only_one_gen_alloc.py +++ b/libensemble/alloc_funcs/only_one_gen_alloc.py @@ -1,4 +1,4 @@ -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, test_any_gen +from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, test_any_gen, all_returned def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info): @@ -15,6 +15,10 @@ def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info): gen_flag = True for i in avail_worker_ids(W): + # Skip any cancelled points + while persis_info['next_to_give'] < len(H) and H[persis_info['next_to_give']]['cancel_requested']: + persis_info['next_to_give'] += 1 + if persis_info['next_to_give'] < len(H): # Give sim work if possible @@ -23,12 +27,14 @@ def ensure_one_active_gen(W, H, sim_specs, gen_specs, alloc_specs, persis_info): elif not test_any_gen(W) and gen_flag: - if not all(H['returned']): + if not all_returned(H): break # Give gen work persis_info['total_gen_calls'] += 1 gen_flag = False - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i]) + gen_in = gen_specs.get('in', []) + return_rows = range(len(H)) if gen_in else [] + gen_work(Work, i, gen_in, return_rows, persis_info.get(i)) return Work, persis_info diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index b985c17256..d58533f0b5 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -1,6 +1,6 @@ import numpy as np -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens +from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens, all_returned def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): @@ -23,7 +23,7 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if persis_info.get('first_call', True): assert np.all(H['given']), "Initial points in H have never been given." assert np.all(H['given_back']), "Initial points in H have never been given_back." - assert np.all(H['returned']), "Initial points in H have never been returned." + assert all_returned(H), "Initial points in H have never been returned." persis_info['fields_to_give_back'] = ['f'] + [n[0] for n in gen_specs['out']] if 'grad' in [n[0] for n in sim_specs['out']]: @@ -51,22 +51,26 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info inds_to_give = np.where(returned_but_not_given)[0] gen_work(Work, i, persis_info['fields_to_give_back'], - np.atleast_1d(inds_to_give), persis_info[i], persistent=True) + inds_to_give, persis_info.get(i), persistent=True) H['given_back'][inds_to_give] = True for i in avail_worker_ids(W, persistent=False): + # Skip any cancelled points + while persis_info['next_to_give'] < len(H) and H[persis_info['next_to_give']]['cancel_requested']: + persis_info['next_to_give'] += 1 + if persis_info['next_to_give'] < len(H): # perform sim evaluations (if they exist in History). - sim_work(Work, i, sim_specs['in'], np.atleast_1d(persis_info['next_to_give']), persis_info[i]) + sim_work(Work, i, sim_specs['in'], persis_info['next_to_give'], persis_info.get(i)) persis_info['next_to_give'] += 1 elif persis_info.get('gen_started') is None: # Finally, call a persistent generator as there is nothing else to do. persis_info['gen_started'] = True - persis_info[i]['nworkers'] = len(W) + persis_info.get(i)['nworkers'] = len(W) - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i], + gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info.get(i), persistent=True) return Work, persis_info diff --git a/libensemble/alloc_funcs/start_fd_persistent.py b/libensemble/alloc_funcs/start_fd_persistent.py index 850f6bbd92..b24e8e0f73 100644 --- a/libensemble/alloc_funcs/start_fd_persistent.py +++ b/libensemble/alloc_funcs/start_fd_persistent.py @@ -41,22 +41,22 @@ def finite_diff_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): if len(inds_to_send): gen_work(Work, i, list(set(gen_specs['in'] + sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')])), - np.atleast_1d(inds_to_send), persis_info[i], persistent=True) + inds_to_send, persis_info.get(i), persistent=True) H['given_back'][inds_to_send] = True - task_avail = ~H['given'] + task_avail = ~H['given'] & ~H['cancel_requested'] for i in avail_worker_ids(W, persistent=False): if np.any(task_avail): # perform sim evaluations (if they exist in History). sim_ids_to_send = np.nonzero(task_avail)[0][0] # oldest point - sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), persis_info[i]) + sim_work(Work, i, sim_specs['in'], sim_ids_to_send, persis_info.get(i)) task_avail[sim_ids_to_send] = False elif gen_count == 0: # Finally, call a persistent generator as there is nothing else to do. gen_count += 1 - gen_work(Work, i, gen_specs['in'], [], persis_info[i], + gen_work(Work, i, gen_specs['in'], [], persis_info.get(i), persistent=True) return Work, persis_info, 0 diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 3353352ead..1493e2fafa 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -1,68 +1,92 @@ import numpy as np - -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens +from libensemble.tools.alloc_support import (avail_worker_ids, sim_work, gen_work, + count_persis_gens, all_returned) def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info): """ This allocation function will give simulation work if possible, but - otherwise start up to 1 persistent generator. If all points requested by - the persistent generator have been returned from the simulation evaluation, - then this information is given back to the persistent generator. + otherwise start up to one persistent generator. By default, evaluation + results are given back to the generator once all generated points have + been returned from the simulation evaluation. If alloc_specs['user']['async_return'] + is set to True, then any returned points are given back to the generator. + + If the single persistent generator has exited, then ensemble shutdown is triggered. + + **User options**: + + To be provided in calling script: E.g., ``alloc_specs['user']['async_return'] = True`` + + init_sample_size: int, optional + Initial sample size - always return in batch. Default: 0 + + async_return: boolean, optional + Return results to gen as they come in (after sample). Default: False (batch return). + + active_recv_gen: boolean, optional + Create gen in active receive mode. If True, the manager does not need to wait + for a return from the generator before sending further returned points. + Default: False + .. seealso:: `test_persistent_uniform_sampling.py `_ # noqa `test_persistent_uniform_sampling_async.py `_ # noqa + `test_persistent_surmise_calib.py `_ # noqa """ Work = {} gen_count = count_persis_gens(W) + # Initialize alloc_specs['user'] as user. + user = alloc_specs.get('user', {}) + active_recv_gen = user.get('active_recv_gen', False) # Persistent gen can handle irregular communications + init_sample_size = user.get('init_sample_size', 0) # Always batch return until this many evals complete + + # Asynchronous return to generator + async_return = user.get('async_return', False) and sum(H['returned']) >= init_sample_size + if persis_info.get('gen_started') and gen_count == 0: # The one persistent worker is done. Exiting return Work, persis_info, 1 - for i in avail_worker_ids(W, persistent=True): - if gen_specs['user'].get('async', False): - # If i is in persistent mode, asynchronous behavior is desired, and - # *any* of its calculated values have returned, give them back to i. - # Otherwise, give nothing to i - returned_but_not_given = np.logical_and.reduce((H['returned'], ~H['given_back'], H['gen_worker'] == i)) - if np.any(returned_but_not_given): - inds_to_give = np.where(returned_but_not_given)[0] + # Give evaluated results back to a running persistent gen + for i in avail_worker_ids(W, persistent=True, active_recv=active_recv_gen): + gen_inds = (H['gen_worker'] == i) + returned_but_not_given = np.logical_and.reduce((H['returned'], ~H['given_back'], gen_inds)) + if np.any(returned_but_not_given): + inds_since_last_gen = np.where(returned_but_not_given)[0] + if async_return or all_returned(H, gen_inds): gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], - np.atleast_1d(inds_to_give), persis_info[i], persistent=True) - - H['given_back'][inds_to_give] = True - - else: - # If i is in persistent mode, batch behavior is desired, and - # *all* of its calculated values have returned, give them back to i. - # Otherwise, give nothing to i - gen_inds = (H['gen_worker'] == i) - if np.all(H['returned'][gen_inds]): - last_time_gen_gave_batch = np.max(H['gen_time'][gen_inds]) - inds_to_give = H['sim_id'][gen_inds][H['gen_time'][gen_inds] == last_time_gen_gave_batch] - gen_work(Work, i, - sim_specs['in'] + [n[0] for n in sim_specs['out']] + [('sim_id')], - np.atleast_1d(inds_to_give), persis_info[i], persistent=True) + inds_since_last_gen, persis_info.get(i), persistent=True, + active_recv=active_recv_gen) - H['given_back'][inds_to_give] = True + H['given_back'][inds_since_last_gen] = True - task_avail = ~H['given'] + task_avail = ~H['given'] & ~H['cancel_requested'] for i in avail_worker_ids(W, persistent=False): + if np.any(task_avail): + if 'priority' in H.dtype.fields: + priorities = H['priority'][task_avail] + if gen_specs['user'].get('give_all_with_same_priority'): + q_inds = (priorities == np.max(priorities)) + else: + q_inds = np.argmax(priorities) + else: + q_inds = 0 + # perform sim evaluations (if they exist in History). - sim_ids_to_send = np.nonzero(task_avail)[0][0] # oldest point - sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), persis_info[i]) + sim_ids_to_send = np.nonzero(task_avail)[0][q_inds] # oldest point(s) + sim_work(Work, i, sim_specs['in'], sim_ids_to_send, persis_info.get(i)) task_avail[sim_ids_to_send] = False elif gen_count == 0: # Finally, call a persistent generator as there is nothing else to do. gen_count += 1 - gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i], - persistent=True) + gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info.get(i), + persistent=True, active_recv=active_recv_gen) persis_info['gen_started'] = True return Work, persis_info, 0 diff --git a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py index 0ac802fdec..8a57472b97 100644 --- a/libensemble/alloc_funcs/start_persistent_local_opt_gens.py +++ b/libensemble/alloc_funcs/start_persistent_local_opt_gens.py @@ -1,7 +1,7 @@ import numpy as np from libensemble.message_numbers import EVAL_GEN_TAG -from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens +from libensemble.tools.alloc_support import avail_worker_ids, sim_work, gen_work, count_persis_gens, all_returned from libensemble.gen_funcs.old_aposmm import initialize_APOSMM, decide_where_to_start_localopt, update_history_dist @@ -25,7 +25,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per Work = {} gen_count = count_persis_gens(W) - task_avail = ~H['given'] + task_avail = ~H['given'] & ~H['cancel_requested'] # If a persistent localopt run has just finished, use run_order to update H # and then remove other information from persis_info @@ -42,17 +42,17 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per # returned, give them back to i. Otherwise, give nothing to i for i in avail_worker_ids(W, persistent=True): gen_inds = (H['gen_worker'] == i) - if np.all(H['returned'][gen_inds]): + if all_returned(H, gen_inds): last_time_pos = np.argmax(H['given_time'][gen_inds]) last_ind = np.nonzero(gen_inds)[0][last_time_pos] gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']], - np.atleast_1d(last_ind), persis_info[i], persistent=True) + last_ind, persis_info[i], persistent=True) persis_info[i]['run_order'].append(last_ind) for i in avail_worker_ids(W, persistent=False): # Find candidates to start local opt runs if a sample has been evaluated - if np.any(np.logical_and(~H['local_pt'], H['returned'])): + if np.any(np.logical_and(~H['local_pt'], H['returned'], ~H['cancel_requested'])): n, _, _, _, r_k, mu, nu = initialize_APOSMM(H, gen_specs) update_history_dist(H, n, gen_specs['user'], c_flag=False) starting_inds = decide_where_to_start_localopt(H, r_k, mu, nu) @@ -65,7 +65,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per ind = starting_inds[np.argmin(H['f'][starting_inds])] gen_work(Work, i, sim_specs['in'] + [n[0] for n in sim_specs['out']], - np.atleast_1d(ind), persis_info[i], persistent=True) + ind, persis_info[i], persistent=True) H['started_run'][ind] = 1 H['num_active_runs'][ind] += 1 @@ -80,7 +80,7 @@ def start_persistent_local_opt_gens(W, H, sim_specs, gen_specs, alloc_specs, per if not np.any(q_inds_logical): q_inds_logical = task_avail sim_ids_to_send = np.nonzero(q_inds_logical)[0][0] # oldest point - sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), []) + sim_work(Work, i, sim_specs['in'], sim_ids_to_send, []) task_avail[sim_ids_to_send] = False elif (gen_count == 0 diff --git a/libensemble/executors/balsam_executor.py b/libensemble/executors/balsam_executor.py index a16a765d45..adf2f77b55 100644 --- a/libensemble/executors/balsam_executor.py +++ b/libensemble/executors/balsam_executor.py @@ -274,7 +274,7 @@ def submit(self, calc_type=None, app_name=None, num_procs=None, num_nodes=None, ranks_per_node=None, machinefile=None, app_args=None, stdout=None, stderr=None, stage_inout=None, hyperthreads=False, dry_run=False, wait_on_run=False, - extra_args=None): + extra_args=''): """Creates a new task, and either executes or schedules to execute in the executor diff --git a/libensemble/executors/executor.py b/libensemble/executors/executor.py index 3ad5adb9c0..e80065b2b2 100644 --- a/libensemble/executors/executor.py +++ b/libensemble/executors/executor.py @@ -1,11 +1,13 @@ """ -This module contains an -``executor`` and ``task``. The class ``Executor`` is a base class and not -intended for direct use. Instead one of the inherited classes should be used. Inherited -classes include MPI and Balsam variants. A ``executor`` can create and manage -multiple ``tasks``. The worker or user-side code can issue and manage ``tasks`` using the submit, -poll and kill functions. ``Task`` attributes are queried to determine status. Functions are -also provided to access and interrogate files in the ``task``'s working directory. +This module contains the classes +``Executor`` and ``Task``. The class ``Executor`` can be used to subprocess an application +locally. For MPI programs, or any program using non-local compute resources, one of the +inherited classes should be used. Inherited classes include MPI and Balsam variants. +An ``executor`` can create and manage multiple ``tasks``. The user function +can issue and manage ``tasks`` using the submit, poll, wait, and kill functions. +``Task`` attributes are queried to determine status. Functions are +also provided to access and interrogate files in the ``task``'s working directory. A +``manager_poll`` function can be used to poll for STOP signals from the manager. """ @@ -15,7 +17,11 @@ import itertools import time -from libensemble.message_numbers import STOP_TAG, MAN_SIGNAL_FINISH, MAN_SIGNAL_KILL +from libensemble.message_numbers import (UNSET_TAG, MAN_SIGNAL_FINISH, + MAN_SIGNAL_KILL, WORKER_DONE, + TASK_FAILED, WORKER_KILL_ON_TIMEOUT, + STOP_TAG) + import libensemble.utils.launcher as launcher from libensemble.utils.timer import TaskTimer @@ -282,13 +288,11 @@ class Executor: **Class Attributes:** - :cvar Executor: executor: The default executor. + :cvar Executor: executor: The executor object is stored here and can be retrieved in user functions. **Object Attributes:** - :ivar int wait_time: Timeout period for hard kill :ivar list list_of_tasks: A list of tasks created in this executor - :ivar int workerID: The workerID associated with this executor """ @@ -305,7 +309,7 @@ def _wait_on_run(self, task, fail_time=None): task.timer.start() # To ensure a start time before poll - will be overwritten unless finished by poll. task.submit_time = task.timer.tstart while task.state in NOT_STARTED_STATES: - time.sleep(0.2) + time.sleep(0.02) task.poll() logger.debug("Task {} polled as {} after {} seconds".format(task.name, task.state, time.time()-start)) if not task.finished: @@ -314,7 +318,7 @@ def _wait_on_run(self, task, fail_time=None): if fail_time: remaining = fail_time - task.timer.elapsed while task.state not in END_STATES and remaining > 0: - time.sleep(min(1.0, remaining)) + time.sleep(min(0.2, remaining)) task.poll() remaining = fail_time - task.timer.elapsed logger.debug("After {} seconds: task {} polled as {}".format(task.timer.elapsed, task.name, task.state)) @@ -322,12 +326,9 @@ def _wait_on_run(self, task, fail_time=None): def __init__(self): """Instantiate a new Executor instance. - A new Executor object is created with an application - registry and configuration attributes. + A new Executor object is created. + This is typically created in the user calling script. - This is typically created in the user calling script. If - auto_resources is True, an evaluation of system resources is - performance during this call. """ self.top_level_dir = os.getcwd() self.manager_signal = 'none' @@ -337,11 +338,15 @@ def __init__(self): self.wait_time = 60 self.list_of_tasks = [] self.workerID = None + self.comm = None Executor.executor = self def _serial_setup(self): pass # To be overloaded + def add_comm_info(self, libE_nodes, serial_setup): + pass # To be overloaded + @property def sim_default_app(self): """Returns the default simulation app""" @@ -371,18 +376,23 @@ def default_app(self, calc_type): return app def register_calc(self, full_path, app_name=None, calc_type=None, desc=None): - """Registers a user application to libEnsemble + """Registers a user application to libEnsemble. + + The ``full_path`` of the application must be supplied. Either + ``app_name`` or ``calc_type`` can be used to identify the + application in user scripts (in the **submit** function). + ``app_name`` is recommended. Parameters ---------- - app_name: String - Name to identify this application. - full_path: String The full path of the user application to be registered - calc_type: String + app_name: String, optional + Name to identify this application. + + calc_type: String, optional Calculation type: Set this application as the default 'sim' or 'gen' function. @@ -400,23 +410,27 @@ def register_calc(self, full_path, app_name=None, calc_type=None, desc=None): "Unrecognized calculation type", calc_type) self.default_apps[calc_type] = self.apps[app_name] - def manager_poll(self, comm): - """ Polls for a manager signal + def manager_poll(self): + """ + .. _manager_poll_label: + + Polls for a manager signal The executor manager_signal attribute will be updated. """ + self.manager_signal = 'none' # Reset # Check for messages; disregard anything but a stop signal - if not comm.mail_flag(): + if not self.comm.mail_flag(): return - mtag, man_signal = comm.recv() + mtag, man_signal = self.comm.recv() if mtag != STOP_TAG: return # Process the signal and push back on comm (for now) - logger.info('Manager probe hit true') + logger.info('Worker received kill signal {} from manager'.format(man_signal)) if man_signal == MAN_SIGNAL_FINISH: self.manager_signal = 'finish' elif man_signal == MAN_SIGNAL_KILL: @@ -424,7 +438,69 @@ def manager_poll(self, comm): else: logger.warning("Received unrecognized manager signal {} - " "ignoring".format(man_signal)) - comm.push_to_buffer(mtag, man_signal) + self.comm.push_to_buffer(mtag, man_signal) + return man_signal + + def polling_loop(self, task, timeout=None, delay=0.1, poll_manager=False): + """ Optional, blocking, generic task status polling loop. Operates until the task + finishes, times out, or is optionally killed via a manager signal. On completion, returns a + presumptive :ref:`calc_status` integer. Potentially useful + for running an application via the Executor until it stops without monitoring + its intermediate output. + + Parameters + ---------- + + task: object + a Task object returned by the executor on submission + + timeout: int, optional + Maximum number of seconds for the polling loop to run. Tasks that run + longer than this limit are killed. Default: No timeout + + delay: int, optional + Sleep duration between polling loop iterations. Default: 0.1 seconds + + poll_manager: bool, optional + Whether to also poll the manager for 'finish' or 'kill' signals. + If detected, the task is killed. Default: False. + + Returns + ------- + calc_status: int + presumptive integer attribute describing the final status of a launched task + + """ + + calc_status = UNSET_TAG + + while not task.finished: + task.poll() + + if poll_manager: + man_signal = self.manager_poll() + if self.manager_signal != 'none': + task.kill() + calc_status = man_signal + break + + if timeout is not None and task.runtime > timeout: + task.kill() + calc_status = WORKER_KILL_ON_TIMEOUT + break + + time.sleep(delay) + + if calc_status == UNSET_TAG: + if task.state == 'FINISHED': + calc_status = WORKER_DONE + elif task.state == 'FAILED': + calc_status = TASK_FAILED + else: + logger.warning("Warning: Task {} in unknown state {}. Error code {}" + .format(self.name, self.state, self.errcode)) + + return calc_status def get_task(self, taskid): """ Returns the task object for the supplied task ID """ @@ -437,9 +513,86 @@ def set_workerID(self, workerid): """Sets the worker ID for this executor""" self.workerID = workerid - def set_worker_info(self, workerid=None): + def set_worker_info(self, comm, workerid=None): """Sets info for this executor""" self.workerID = workerid + self.comm = comm + + def submit(self, calc_type=None, app_name=None, app_args=None, + stdout=None, stderr=None, dry_run=False, wait_on_run=False): + """Create a new task and run as a local serial subprocess. + + The created task object is returned. + + Parameters + ---------- + + calc_type: String, optional + The calculation type: 'sim' or 'gen' + Only used if app_name is not supplied. Uses default sim or gen application. + + app_name: String, optional + The application name. Must be supplied if calc_type is not. + + app_args: string, optional + A string of the application arguments to be added to task + submit command line + + stdout: string, optional + A standard output filename + + stderr: string, optional + A standard error filename + + dry_run: boolean, optional + Whether this is a dry_run - no task will be launched; instead + runline is printed to logger (at INFO level) + + wait_on_run: boolean, optional + Whether to wait for task to be polled as RUNNING (or other + active/end state) before continuing + + Returns + ------- + + task: obj: Task + The lauched task object + + """ + + if app_name is not None: + app = self.get_app(app_name) + elif calc_type is not None: + app = self.default_app(calc_type) + else: + raise ExecutorException("Either app_name or calc_type must be set") + + default_workdir = os.getcwd() + task = Task(app, app_args, default_workdir, stdout, stderr, self.workerID) + runline = task.app.full_path.split() + if task.app_args is not None: + runline.extend(task.app_args.split()) + + if dry_run: + logger.info('Test (No submit) Runline: {}'.format(' '.join(runline))) + else: + # Launch Task + logger.info("Launching task {}: {}". + format(task.name, " ".join(runline))) + with open(task.stdout, 'w') as out, open(task.stderr, 'w') as err: + task.process = launcher.launch(runline, cwd='./', + stdout=out, + stderr=err, + start_new_session=False) + if (wait_on_run): + self._wait_on_run(task, 0) # No fail time as no re-starts in-place + + if not task.timer.timing: + task.timer.start() + task.submit_time = task.timer.tstart # Time not date - may not need if using timer. + + self.list_of_tasks.append(task) + return task def poll(self, task): "Polls a task" diff --git a/libensemble/executors/mpi_executor.py b/libensemble/executors/mpi_executor.py index e3354c855b..623de5bbd9 100644 --- a/libensemble/executors/mpi_executor.py +++ b/libensemble/executors/mpi_executor.py @@ -152,10 +152,11 @@ def _launch_with_retries(self, task, runline, subgroup_launch, wait_on_run): logger.info("Launching task {}{}: {}". format(task.name, retry_string, " ".join(runline))) task.run_attempts += 1 - task.process = launcher.launch(runline, cwd='./', - stdout=open(task.stdout, 'w'), - stderr=open(task.stderr, 'w'), - start_new_session=subgroup_launch) + with open(task.stdout, 'w') as out, open(task.stderr, 'w') as err: + task.process = launcher.launch(runline, cwd='./', + stdout=out, + stderr=err, + start_new_session=subgroup_launch) except Exception as e: logger.warning('task {} submit command failed on ' 'try {} with error {}' @@ -303,6 +304,6 @@ def submit(self, calc_type=None, app_name=None, num_procs=None, def set_worker_info(self, comm, workerid=None): """Sets info for this executor""" - self.workerID = workerid + super().set_worker_info(comm, workerid) if self.workerID and self.auto_resources: self.resources.set_worker_resources(self.workerID, comm) diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index 1ef4d27b1b..42b47c26a1 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -94,7 +94,7 @@ def get_mpi_specs(self, task, num_procs, num_nodes, ranks_per_node, hyperthreads) # Use hostlist if full nodes, otherwise machinefile - full_node = resources.worker_resources.workers_per_node == 1 + full_node = resources.worker_resources.workers_on_node == 1 if full_node or not self.mfile_support: hostlist = resources.get_hostlist(num_nodes) else: diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index d277550dd1..71fd580973 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -6,6 +6,7 @@ 'run_local_dfols', 'run_local_scipy_opt', 'run_external_localopt'] import psutil +from libensemble.tools.tools import osx_set_mp_method import numpy as np from libensemble.message_numbers import STOP_TAG, EVAL_GEN_TAG # Only used to simulate receiving from manager from multiprocessing import Event, Process, Queue @@ -14,6 +15,9 @@ optimizer_list = ['petsc', 'nlopt', 'dfols', 'scipy', 'external'] optimizers = libensemble.gen_funcs.rc.aposmm_optimizers +# Resolves multiprocessing issues with Python 3.8+ on macOS +osx_set_mp_method() + if optimizers is None: from petsc4py import PETSc import nlopt @@ -193,7 +197,7 @@ def run_local_nlopt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_r # Care must be taken here because a too-large initial step causes nlopt to move the starting point! dist_to_bound = min(min(ub-x0), min(x0-lb)) - assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" + assert dist_to_bound > np.finfo(np.float64).eps, "The distance to the boundary is too small for NLopt to handle" if 'dist_to_bound_multiple' in user_specs: opt.set_initial_step(dist_to_bound*user_specs['dist_to_bound_multiple']) @@ -349,7 +353,7 @@ def run_local_dfols(user_specs, comm_queue, x0, f0, child_can_read, parent_can_r # Care must be taken here because a too-large initial step causes DFO-LS to move the starting point! dist_to_bound = min(min(ub-x0), min(x0-lb)) - assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small" + assert dist_to_bound > np.finfo(np.float64).eps, "The distance to the boundary is too small" assert 'bounds' not in user_specs.get('dfols_kwargs', {}), "APOSMM must set the bounds for DFO-LS" assert 'rhobeg' not in user_specs.get('dfols_kwargs', {}), "APOSMM must set rhobeg for DFO-LS" assert 'x0' not in user_specs.get('dfols_kwargs', {}), "APOSMM must set x0 for DFO-LS" diff --git a/libensemble/gen_funcs/old_aposmm.py b/libensemble/gen_funcs/old_aposmm.py index 4ee0204f78..3eee28b2a7 100644 --- a/libensemble/gen_funcs/old_aposmm.py +++ b/libensemble/gen_funcs/old_aposmm.py @@ -631,7 +631,7 @@ def nlopt_obj_fun(x, grad, Run_H): # Care must be taken here because a too-large initial step causes nlopt to move the starting point! dist_to_bound = min(min(ub-x0), min(x0-lb)) - assert dist_to_bound > np.finfo(np.float32).eps, "The distance to the boundary is too small for NLopt to handle" + assert dist_to_bound > np.finfo(np.float64).eps, "The distance to the boundary is too small for NLopt to handle" if 'dist_to_bound_multiple' in user_specs: opt.set_initial_step(dist_to_bound*user_specs['dist_to_bound_multiple']) diff --git a/libensemble/gen_funcs/persistent_deap_nsga2.py b/libensemble/gen_funcs/persistent_deap_nsga2.py index 4e70aa6c2e..d59bef9371 100644 --- a/libensemble/gen_funcs/persistent_deap_nsga2.py +++ b/libensemble/gen_funcs/persistent_deap_nsga2.py @@ -68,7 +68,7 @@ def evaluate_pop(g, deap_object, Out, comm): Out['generation'][index] = g # Sending work to sim_f, which is defined in main call script # A fitness value will be returned in calc_in - tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, Out) + tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, Out[['individual', 'generation']]) if tag not in [STOP_TAG, PERSIS_STOP]: for i, ind in enumerate(deap_object): @@ -157,12 +157,13 @@ def deap_nsga2(H, persis_info, gen_specs, libE_info): # Don't update population pass - fits = [ind.fitness.values[0] for ind in pop] + fits = np.array(np.array([ind.fitness.values for ind in pop])) if tag in [STOP_TAG, PERSIS_STOP]: # Min value when exiting print('Met exit criteria. Current best fitness is:', np.min(fits)) else: - print('Current fitness minimum:', np.min(fits)) + print('Current fitness minimum:', np.min(fits, axis=0)) print('Sum of fit values at end of loop', sum(fits)) + Out['last_points'] = 1 return Out, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_surmise_calib.py b/libensemble/gen_funcs/persistent_surmise_calib.py new file mode 100644 index 0000000000..f62dff7cc1 --- /dev/null +++ b/libensemble/gen_funcs/persistent_surmise_calib.py @@ -0,0 +1,228 @@ +""" +This module contains a simple calibration example using the Surmise package. +""" + +__all__ = ['surmise_calib'] + +import numpy as np +from libensemble.gen_funcs.surmise_calib_support import gen_xs, gen_thetas, gen_observations, gen_true_theta, \ + thetaprior, select_next_theta +from surmise.calibration import calibrator +from surmise.emulation import emulator +from libensemble.message_numbers import STOP_TAG, PERSIS_STOP, FINISHED_PERSISTENT_GEN_TAG +from libensemble.tools.gen_support import sendrecv_mgr_worker_msg, get_mgr_worker_msg, send_mgr_worker_msg + + +def build_emulator(theta, x, fevals): + """Build the emulator.""" + print(x.shape, theta.shape, fevals.shape) + emu = emulator(x, theta, fevals, method='PCGPwM', + options={'xrmnan': 'all', + 'thetarmnan': 'never', + 'return_grad': True}) + emu.fit() + return emu + + +def select_condition(pending, n_remaining_theta=5): + n_x = pending.shape[0] + return False if np.sum(pending) > n_remaining_theta * n_x else True + + +def rebuild_condition(pending, prev_pending, n_theta=5): # needs changes + n_x = pending.shape[0] + if np.sum(prev_pending) - np.sum(pending) >= n_x * n_theta or np.sum(pending) == 0: + return True + else: + return False + + +def create_arrays(calc_in, n_thetas, n_x): + """Create 2D (point * rows) arrays fevals, pending and complete""" + fevals = np.reshape(calc_in['f'], (n_x, n_thetas)) + pending = np.full(fevals.shape, False) + prev_pending = pending.copy() + complete = np.full(fevals.shape, True) + + return fevals, pending, prev_pending, complete + + +def pad_arrays(n_x, thetanew, theta, fevals, pending, prev_pending, complete): + """Extend arrays to appropriate sizes.""" + n_thetanew = len(thetanew) + + theta = np.vstack((theta, thetanew)) + fevals = np.hstack((fevals, np.full((n_x, n_thetanew), np.nan))) + pending = np.hstack((pending, np.full((n_x, n_thetanew), True))) + prev_pending = np.hstack((prev_pending, np.full((n_x, n_thetanew), True))) + complete = np.hstack((complete, np.full((n_x, n_thetanew), False))) + + # print('after:', fevals.shape, theta.shape, pending.shape, complete.shape) + return theta, fevals, pending, prev_pending, complete + + +def update_arrays(fevals, pending, complete, calc_in, obs_offset, n_x): + """Unpack from calc_in into 2D (point * rows) fevals""" + sim_id = calc_in['sim_id'] + c, r = divmod(sim_id - obs_offset, n_x) # r, c are arrays if sim_id is an array + + fevals[r, c] = calc_in['f'] + pending[r, c] = False + complete[r, c] = True + return + + +def cancel_columns(obs_offset, c, n_x, pending, comm): + """Cancel columns""" + sim_ids_to_cancel = [] + columns = np.unique(c) + for c in columns: + col_offset = c*n_x + for i in range(n_x): + sim_id_cancl = obs_offset + col_offset + i + if pending[i, c]: + sim_ids_to_cancel.append(sim_id_cancl) + pending[i, c] = 0 + + # Send only these fields to existing H rows and libEnsemble will slot in the change. + H_o = np.zeros(len(sim_ids_to_cancel), dtype=[('sim_id', int), ('cancel_requested', bool)]) + H_o['sim_id'] = sim_ids_to_cancel + H_o['cancel_requested'] = True + send_mgr_worker_msg(comm, H_o) + + +def assign_priority(n_x, n_thetas): + """Assign priorities to points.""" + # Arbitrary priorities + priority = np.arange(n_x*n_thetas) + np.random.shuffle(priority) + return priority + + +def load_H(H, xs, thetas, offset=0, set_priorities=False): + """Fill inputs into H0. + + There will be num_points x num_thetas entries + """ + n_thetas = len(thetas) + for i, x in enumerate(xs): + start = (i+offset)*n_thetas + H['x'][start:start+n_thetas] = x + H['thetas'][start:start+n_thetas] = thetas + + if set_priorities: + n_x = len(xs) + H['priority'] = assign_priority(n_x, n_thetas) + + +def gen_truevals(x, gen_specs): + """Generate true values using libE.""" + n_x = len(x) + H_o = np.zeros((1) * n_x, dtype=gen_specs['out']) + + # Generate true theta and load into H + true_theta = gen_true_theta() + H_o['x'][0:n_x] = x + H_o['thetas'][0:n_x] = true_theta + return H_o + + +def surmise_calib(H, persis_info, gen_specs, libE_info): + """Generator to select and obviate parameters for calibration.""" + comm = libE_info['comm'] + rand_stream = persis_info['rand_stream'] + n_thetas = gen_specs['user']['n_init_thetas'] + n_x = gen_specs['user']['num_x_vals'] # Num of x points + step_add_theta = gen_specs['user']['step_add_theta'] # No. of thetas to generate per step + n_explore_theta = gen_specs['user']['n_explore_theta'] # No. of thetas to explore + obsvar_const = gen_specs['user']['obsvar'] # Constant for generator + priorloc = gen_specs['user']['priorloc'] + priorscale = gen_specs['user']['priorscale'] + prior = thetaprior(priorloc, priorscale) + + # Create points at which to evaluate the sim + x = gen_xs(n_x, rand_stream) + + H_o = gen_truevals(x, gen_specs) + obs_offset = len(H_o) + + tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, H_o) + if tag in [STOP_TAG, PERSIS_STOP]: + return H, persis_info, FINISHED_PERSISTENT_GEN_TAG + + returned_fevals = np.reshape(calc_in['f'], (1, n_x)) + true_fevals = returned_fevals + obs, obsvar = gen_observations(true_fevals, obsvar_const, rand_stream) + + # Generate a batch of inputs and load into H + H_o = np.zeros(n_x*(n_thetas), dtype=gen_specs['out']) + theta = gen_thetas(prior, n_thetas) + load_H(H_o, x, theta, set_priorities=True) + tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, H_o) + # ------------------------------------------------------------------------- + + fevals = None + prev_pending = None + + while tag not in [STOP_TAG, PERSIS_STOP]: + if fevals is None: # initial batch + fevals, pending, prev_pending, complete = create_arrays(calc_in, n_thetas, n_x) + emu = build_emulator(theta, x, fevals) + # Refer to surmise package for additional options + cal = calibrator(emu, obs, x, prior, obsvar, method='directbayes') + + print('quantiles:', np.round(np.quantile(cal.theta.rnd(10000), (0.01, 0.99), axis=0), 3)) + update_model = False + else: + # Update fevals from calc_in + update_arrays(fevals, pending, complete, calc_in, + obs_offset, n_x) + update_model = rebuild_condition(pending, prev_pending) + if not update_model: + tag, Work, calc_in = get_mgr_worker_msg(comm) + if tag in [STOP_TAG, PERSIS_STOP]: + break + + if update_model: + print('Percentage Cancelled: %0.2f ( %d / %d)' % (100*np.round(np.mean(1-pending-complete), 4), + np.sum(1-pending-complete), + np.prod(pending.shape))) + print('Percentage Pending: %0.2f ( %d / %d)' % (100*np.round(np.mean(pending), 4), + np.sum(pending), + np.prod(pending.shape))) + print('Percentage Complete: %0.2f ( %d / %d)' % (100*np.round(np.mean(complete), 4), + np.sum(complete), + np.prod(pending.shape))) + + emu.update(theta=theta, f=fevals) + cal.fit() + + samples = cal.theta.rnd(2500) + print(np.mean(np.sum((samples - np.array([0.5]*4))**2, 1))) + print(np.round(np.quantile(cal.theta.rnd(10000), (0.01, 0.99), axis=0), 3)) + + step_add_theta += 2 + prev_pending = pending.copy() + update_model = False + + # Conditionally generate new thetas from model + if select_condition(pending): + new_theta, info = select_next_theta(step_add_theta, cal, emu, pending, n_explore_theta) + + # Add space for new thetas + theta, fevals, pending, prev_pending, complete = \ + pad_arrays(n_x, new_theta, theta, fevals, pending, prev_pending, complete) + + # n_thetas = step_add_theta + H_o = np.zeros(n_x*(len(new_theta)), dtype=gen_specs['out']) + load_H(H_o, x, new_theta, set_priorities=True) + tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, H_o) + + # Determine evaluations to cancel + c_obviate = info['obviatesugg'] + if len(c_obviate) > 0: + print('columns sent for cancel is: {}'.format(c_obviate), flush=True) + cancel_columns(obs_offset, c_obviate, n_x, pending, comm) + pending[:, c_obviate] = False + + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_uniform_sampling.py b/libensemble/gen_funcs/persistent_uniform_sampling.py index 768f1b3dc7..1eeac429ed 100644 --- a/libensemble/gen_funcs/persistent_uniform_sampling.py +++ b/libensemble/gen_funcs/persistent_uniform_sampling.py @@ -27,7 +27,13 @@ def persistent_uniform(H, persis_info, gen_specs, libE_info): H_o = np.zeros(b, dtype=gen_specs['out']) H_o['x'] = persis_info['rand_stream'].uniform(lb, ub, (b, n)) tag, Work, calc_in = sendrecv_mgr_worker_msg(libE_info['comm'], H_o) - if calc_in is not None: + if hasattr(calc_in, '__len__'): b = len(calc_in) + if gen_specs['user'].get('replace_final_fields', 0): + # This is only to test libE ability to accept History after a + # PERSIS_STOP. This history is returned in Work. + H_o = Work + H_o['x'] = -1.23 + return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index adb560e2bf..1bea1c06ba 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -38,8 +38,8 @@ def uniform_random_sample_with_different_nodes_and_ranks(H, persis_info, gen_spe else: H_o = np.zeros(1, dtype=gen_specs['out']) H_o['x'] = len(H)*np.ones(n) - H_o['num_nodes'] = np.random.randint(1, gen_specs['user']['max_num_nodes']+1) - H_o['ranks_per_node'] = np.random.randint(1, gen_specs['user']['max_ranks_per_node']+1) + H_o['num_nodes'] = persis_info['rand_stream'].randint(1, gen_specs['user']['max_num_nodes']+1) + H_o['ranks_per_node'] = persis_info['rand_stream'].randint(1, gen_specs['user']['max_ranks_per_node']+1) H_o['priority'] = 10*H_o['num_nodes'] return H_o, persis_info @@ -95,6 +95,28 @@ def uniform_random_sample(H, persis_info, gen_specs, _): return H_o, persis_info +def uniform_random_sample_cancel(H, persis_info, gen_specs, _): + """ + Similar to uniform_random_sample but with immediate cancellation of + selected points for testing. + + """ + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] + + n = len(lb) + b = gen_specs['user']['gen_batch_size'] + + H_o = np.zeros(b, dtype=gen_specs['out']) + for i in range(b): + if i % 10 == 0: + H_o[i]['cancel_requested'] = True + + H_o['x'] = persis_info['rand_stream'].uniform(lb, ub, (b, n)) + + return H_o, persis_info + + def latin_hypercube_sample(H, persis_info, gen_specs, _): """ Generates ``gen_specs['user']['gen_batch_size']`` points in a Latin @@ -113,18 +135,18 @@ def latin_hypercube_sample(H, persis_info, gen_specs, _): H_o = np.zeros(b, dtype=gen_specs['out']) - A = lhs_sample(n, b) + A = lhs_sample(n, b, persis_info['rand_stream']) H_o['x'] = A*(ub-lb)+lb return H_o, persis_info -def lhs_sample(n, k): +def lhs_sample(n, k, stream): # Generate the intervals and random values intervals = np.linspace(0, 1, k+1) - rand_source = np.random.uniform(0, 1, (k, n)) + rand_source = stream.uniform(0, 1, (k, n)) rand_pts = np.zeros((k, n)) sample = np.zeros((k, n)) @@ -136,6 +158,6 @@ def lhs_sample(n, k): # Randomly perturb for j in range(n): - sample[:, j] = rand_pts[np.random.permutation(k), j] + sample[:, j] = rand_pts[stream.permutation(k), j] return sample diff --git a/libensemble/gen_funcs/surmise_calib_support.py b/libensemble/gen_funcs/surmise_calib_support.py new file mode 100644 index 0000000000..155553d424 --- /dev/null +++ b/libensemble/gen_funcs/surmise_calib_support.py @@ -0,0 +1,74 @@ +"""Contains supplemental methods for gen function in persistent_surmise_calib.py.""" + +import numpy as np +import scipy.stats as sps + + +class thetaprior: + """Define the class instance of priors provided to the methods.""" + def __init__(self, loc, scale): + self._loc = loc + self._scale = scale + + def lpdf(self, theta): + """Return log prior density.""" + loc = self._loc + scale = self._scale + return (np.sum(sps.norm.logpdf(theta, loc, scale), 1)).reshape((len(theta), 1)) + + def rnd(self, n): + """Return random draws from prior.""" + loc = self._loc + scale = self._scale + return np.vstack((sps.norm.rvs(loc, scale, size=(n, 4)))) + + +def gen_true_theta(): + """Generate one parameter to be the true parameter for calibration.""" + theta0 = np.atleast_2d(np.array([0.5] * 4)) + + return theta0 + + +def gen_thetas(prior, n): + """Generate and return n parameters for the test function.""" + thetas = prior.rnd(n) + return thetas + + +def gen_xs(nx, randstream): + """Generate and returns n inputs for the modified Borehole function.""" + xs = randstream.uniform(0, 1, (nx, 3)) + xs[:, 2] = xs[:, 2] > 0.5 + + return xs + + +def gen_observations(fevals, obsvar_const, randstream): + """Generate observations.""" + n_x = len(np.squeeze(fevals)) + obsvar = np.maximum(obsvar_const*fevals, 5) + obsvar = np.squeeze(obsvar) + obs = fevals + randstream.normal(0, np.sqrt(obsvar), n_x).reshape((n_x)) + + obs = np.squeeze(obs) + return obs, obsvar + + +def select_next_theta(numnewtheta, cal, emu, pending, numexplore): + # numnewtheta += 2 + thetachoices = cal.theta(numexplore) + choicescost = np.ones(thetachoices.shape[0]) + thetaneworig, info = emu.supplement(size=numnewtheta, thetachoices=thetachoices, + choicescost=choicescost, + cal=cal, overwrite=True, + args={'includepending': True, + 'costpending': 0.01 + 0.99 * np.mean(pending, 0), + 'pending': pending}) + thetaneworig = thetaneworig[:numnewtheta, :] + thetanew = thetaneworig + return thetanew, info + + +def obviate_pend_theta(info, pending): + return pending, info['obviatesugg'] diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index d6a9d45805..f0b471f249 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -57,7 +57,7 @@ def nlopt_obj_fun(x, grad): if np.array_equiv(x, H['x']): if gen_specs['user']['localopt_method'] in ['LD_MMA']: grad[:] = H['grad'] - return np.float(H['f']) + return float(H['f']) # Send back x to the manager, then receive info or stop tag H_o = add_to_Out(np.zeros(1, dtype=gen_specs['out']), x, 0, diff --git a/libensemble/history.py b/libensemble/history.py index 826193ee28..279544ded5 100644 --- a/libensemble/history.py +++ b/libensemble/history.py @@ -31,10 +31,10 @@ class History: :ivar int given_count: Number of points given to sim fuctions (according to H) - :ivar int sim_count: + :ivar int returned_count: Number of points evaluated (according to H) - Note that index, given_count and sim_count reflect the total number of points + Note that index, given_count and returned_count reflect the total number of points in H and therefore include those prepended to H in addition to the current run. """ @@ -76,6 +76,7 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): H['sim_id'][-L:] = -1 H['given_time'][-L:] = np.inf + H['last_given_time'][-L:] = np.inf self.H = H # self.offset = 0 @@ -84,9 +85,9 @@ def __init__(self, alloc_specs, sim_specs, gen_specs, exit_criteria, H0): self.given_count = np.sum(H['given']) - self.sim_count = np.sum(H['returned']) + self.returned_count = np.sum(H['returned']) - def update_history_f(self, D): + def update_history_f(self, D, safe_mode): """ Updates the history after points have been evaluated """ @@ -96,7 +97,8 @@ def update_history_f(self, D): for j, ind in enumerate(new_inds): for field in returned_H.dtype.names: - assert field not in protected_libE_fields, "The field '" + field + "' is protected" + if safe_mode: + assert field not in protected_libE_fields, "The field '" + field + "' is protected" if np.isscalar(returned_H[field][j]): self.H[field][ind] = returned_H[field][j] else: @@ -111,7 +113,8 @@ def update_history_f(self, D): self.H[field][ind][:H0_size] = returned_H[field][j] # Slice View self.H['returned'][ind] = True - self.sim_count += 1 + self.H['returned_time'][ind] = time.time() + self.returned_count += 1 def update_history_x_out(self, q_inds, sim_worker): """ @@ -125,16 +128,18 @@ def update_history_x_out(self, q_inds, sim_worker): sim_worker: integer Worker ID """ + q_inds = np.atleast_1d(q_inds) + first_given_inds = ~self.H['given'][q_inds] + t = time.time() + self.H['given'][q_inds] = True - self.H['given_time'][q_inds] = time.time() + self.H['given_time'][q_inds[first_given_inds]] = t + self.H['last_given_time'][q_inds] = t self.H['sim_worker'][q_inds] = sim_worker - if np.isscalar(q_inds): - self.given_count += 1 - else: - self.given_count += len(q_inds) + self.given_count += len(q_inds) - def update_history_x_in(self, gen_worker, D): + def update_history_x_in(self, gen_worker, D, safe_mode): """ Updates the history (in place) when new points have been returned from a gen @@ -175,11 +180,15 @@ def update_history_x_in(self, gen_worker, D): update_inds = D['sim_id'] for field in D.dtype.names: - assert field not in protected_libE_fields, "The field '" + field + "' is protected" + if safe_mode: + assert field not in protected_libE_fields, "The field '" + field + "' is protected" self.H[field][update_inds] = D[field] - self.H['gen_time'][update_inds] = time.time() - self.H['gen_worker'][update_inds] = gen_worker + first_gen_inds = update_inds[self.H['gen_time'][update_inds] == 0] + t = time.time() + self.H['gen_time'][first_gen_inds] = t + self.H['last_gen_time'][update_inds] = t + self.H['gen_worker'][first_gen_inds] = gen_worker self.index += num_new def grow_H(self, k): @@ -195,6 +204,7 @@ def grow_H(self, k): H_1 = np.zeros(k, dtype=self.H.dtype) H_1['sim_id'] = -1 H_1['given_time'] = np.inf + H_1['last_given_time'] = np.inf self.H = np.append(self.H, H_1) # Could be arguments here to return different truncations eg. all done, given etc... diff --git a/libensemble/libE.py b/libensemble/libE.py index 310c15fe1e..7affa25f76 100644 --- a/libensemble/libE.py +++ b/libensemble/libE.py @@ -16,7 +16,6 @@ __all__ = ['libE'] import os -import platform import logging import random import socket @@ -27,14 +26,14 @@ from libensemble.utils import launcher from libensemble.utils.timer import Timer from libensemble.history import History -from libensemble.libE_manager import manager_main, ManagerException -from libensemble.libE_worker import worker_main +from libensemble.manager import manager_main, report_worker_exc, WorkerException, LoggedException +from libensemble.worker import worker_main from libensemble.alloc_funcs import defaults as alloc_defaults from libensemble.comms.comms import QCommProcess, Timeout from libensemble.comms.logs import manager_logging_config from libensemble.comms.tcp_mgr import ServerQCommManager, ClientQCommManager from libensemble.executors.executor import Executor -from libensemble.tools.tools import _USER_SIM_ID_WARNING +from libensemble.tools.tools import _USER_SIM_ID_WARNING, osx_set_mp_method from libensemble.tools.check_inputs import check_inputs logger = logging.getLogger(__name__) @@ -140,32 +139,39 @@ def libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs, H0) -def libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, persis_info, - alloc_specs, libE_specs, hist, - on_abort=None, on_cleanup=None): +def manager(wcomms, sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs, hist, + on_abort=None, on_cleanup=None): "Generic manager routine run." + logger.info('Logger initializing: [workerID] precedes each line. [0] = Manager') + if 'out' in gen_specs and ('sim_id', int) in gen_specs['out']: logger.manager_warning(_USER_SIM_ID_WARNING) save_H = libE_specs.get('save_H_and_persis_on_abort', True) try: - persis_info, exit_flag, elapsed_time = \ - manager_main(hist, libE_specs, alloc_specs, sim_specs, gen_specs, - exit_criteria, persis_info, wcomms) - logger.info("libE_manager total time: {}".format(elapsed_time)) - - except ManagerException as e: - _report_manager_exception(hist, persis_info, e, save_H=save_H) - if libE_specs.get('abort_on_exception', True) and on_abort is not None: - on_abort() - raise - except Exception: - _report_manager_exception(hist, persis_info, save_H=save_H) + try: + persis_info, exit_flag, elapsed_time = \ + manager_main(hist, libE_specs, alloc_specs, sim_specs, gen_specs, + exit_criteria, persis_info, wcomms) + logger.info("Manager total time: {}".format(elapsed_time)) + except LoggedException: + # Exception already logged in manager + raise + except WorkerException as e: + report_worker_exc(e) + raise LoggedException(e.args[0], e.args[1]) from None + except Exception as e: + logger.error(traceback.format_exc()) + raise LoggedException(e.args) from None + except Exception as e: + exit_flag = 1 # Only exits if no abort/raise + _dump_on_abort(hist, persis_info, save_H=save_H) if libE_specs.get('abort_on_exception', True) and on_abort is not None: on_abort() - raise + raise LoggedException(*e.args, 'See error details above and in ensemble.log') from None else: logger.debug("Manager exiting") logger.debug("Exiting with {} workers.".format(len(wcomms))) @@ -182,8 +188,8 @@ def libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, persis_info, class DupComm: """Duplicate MPI communicator for use with a with statement""" - def __init__(self, comm): - self.parent_comm = comm + def __init__(self, mpi_comm): + self.parent_comm = mpi_comm def __enter__(self): self.dup_comm = self.parent_comm.Dup() @@ -193,9 +199,9 @@ def __exit__(self, etype, value, traceback): self.dup_comm.Free() -def comms_abort(comm): +def comms_abort(mpi_comm): "Abort all MPI ranks" - comm.Abort(1) # Exit code 1 to represent an abort + mpi_comm.Abort(1) # Exit code 1 to represent an abort def libE_mpi_defaults(libE_specs): @@ -261,17 +267,17 @@ def on_abort(): comms_abort(mpi_comm) # Run generic manager - return libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs, hist, - on_abort=on_abort) + return manager(wcomms, sim_specs, gen_specs, exit_criteria, + persis_info, alloc_specs, libE_specs, hist, + on_abort=on_abort) def libE_mpi_worker(libE_comm, sim_specs, gen_specs, libE_specs): "Worker routine run at ranks > 0." from libensemble.comms.mpi import MainMPIComm - mpi_comm = MainMPIComm(libE_comm) - worker_main(mpi_comm, sim_specs, gen_specs, libE_specs, log_comm=True) + comm = MainMPIComm(libE_comm) + worker_main(comm, sim_specs, gen_specs, libE_specs, log_comm=True) logger.debug("Worker {} exiting".format(libE_comm.Get_rank())) @@ -314,10 +320,8 @@ def libE_local(sim_specs, gen_specs, exit_criteria, # switched to 'spawn' by default due to 'fork' potentially causing crashes. # These crashes haven't yet been observed with libE, but with 'spawn' runs, # warnings about leaked semaphore objects are displayed instead. - # The next several statements enforce 'fork' on macOS (Python 3.8) - if platform.system() == 'Darwin': - from multiprocessing import set_start_method - set_start_method('fork', force=True) + # This function enforces 'fork' on macOS (Python 3.8) + osx_set_mp_method() # Launch worker team and set up logger wcomms = start_proc_team(nworkers, sim_specs, gen_specs, libE_specs) @@ -331,9 +335,9 @@ def cleanup(): kill_proc_team(wcomms, timeout=libE_specs.get('worker_timeout', 1)) # Run generic manager - return libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs, hist, - on_cleanup=cleanup) + return manager(wcomms, sim_specs, gen_specs, exit_criteria, + persis_info, alloc_specs, libE_specs, hist, + on_cleanup=cleanup) # ==================== TCP version ================================= @@ -434,11 +438,13 @@ def libE_tcp_mgr(sim_specs, gen_specs, exit_criteria, port = libE_specs.get('port', 0) authkey = libE_specs.get('authkey', libE_tcp_authkey()) - with ServerQCommManager(port, authkey.encode('utf-8')) as manager: + osx_set_mp_method() + + with ServerQCommManager(port, authkey.encode('utf-8')) as tcp_manager: # Get port if needed because of auto-assignment if port == 0: - _, port = manager.address + _, port = tcp_manager.address if not libE_specs.get('disable_log_files', False): manager_logging_config() @@ -447,7 +453,7 @@ def libE_tcp_mgr(sim_specs, gen_specs, exit_criteria, # Launch worker team and set up logger worker_procs, wcomms =\ - libE_tcp_start_team(manager, nworkers, workers, + libE_tcp_start_team(tcp_manager, nworkers, workers, ip, port, authkey, launchf) def cleanup(): @@ -456,9 +462,9 @@ def cleanup(): launcher.cancel(wp, timeout=libE_specs.get('worker_timeout')) # Run generic manager - return libE_manager(wcomms, sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs, hist, - on_cleanup=cleanup) + return manager(wcomms, sim_specs, gen_specs, exit_criteria, + persis_info, alloc_specs, libE_specs, hist, + on_cleanup=cleanup) def libE_tcp_worker(sim_specs, gen_specs, libE_specs): @@ -478,20 +484,12 @@ def libE_tcp_worker(sim_specs, gen_specs, libE_specs): # ==================== Additional Internal Functions =========================== -def _report_manager_exception(hist, persis_info, mgr_exc=None, save_H=True): - "Write out exception manager exception to log." - if mgr_exc is not None: - from_line, msg, exc = mgr_exc.args - logger.error("---- {} ----".format(from_line)) - logger.error("Message: {}".format(msg)) - logger.error(exc) - else: - logger.error(traceback.format_exc()) +def _dump_on_abort(hist, persis_info, save_H=True): logger.error("Manager exception raised .. aborting ensemble:") logger.error("Dumping ensemble history with {} sims evaluated:". - format(hist.sim_count)) + format(hist.returned_count)) if save_H: - np.save('libE_history_at_abort_' + str(hist.sim_count) + '.npy', hist.trim_H()) - with open('libE_persis_info_at_abort_' + str(hist.sim_count) + '.pickle', "wb") as f: + np.save('libE_history_at_abort_' + str(hist.returned_count) + '.npy', hist.trim_H()) + with open('libE_persis_info_at_abort_' + str(hist.returned_count) + '.pickle', "wb") as f: pickle.dump(persis_info, f) diff --git a/libensemble/libE_logger.py b/libensemble/logger.py similarity index 100% rename from libensemble/libE_logger.py rename to libensemble/logger.py diff --git a/libensemble/libE_manager.py b/libensemble/manager.py similarity index 73% rename from libensemble/libE_manager.py rename to libensemble/manager.py index 0d93ea1ba5..40c47c7a1f 100644 --- a/libensemble/libE_manager.py +++ b/libensemble/manager.py @@ -8,6 +8,7 @@ import glob import logging import socket +import traceback import numpy as np from libensemble.utils.timer import Timer @@ -19,9 +20,11 @@ TASK_FAILED, WORKER_DONE, \ MAN_SIGNAL_FINISH, MAN_SIGNAL_KILL from libensemble.comms.comms import CommFinishedException -from libensemble.libE_worker import WorkerErrMsg +from libensemble.worker import WorkerErrMsg +from libensemble.output_directory import EnsembleDirectory from libensemble.tools.tools import _USER_CALC_DIR_WARNING -from libensemble.tools.fields_keys import libE_spec_calc_dir_combined, protected_libE_fields +from libensemble.tools.tools import _PERSIS_RETURN_WARNING +from libensemble.tools.fields_keys import protected_libE_fields import cProfile import pstats import copy @@ -35,7 +38,24 @@ class ManagerException(Exception): - "Exception at manager, raised on abort signal from worker" + """Exception raised by the Manager""" + + +class WorkerException(Exception): + """Exception raised on abort signal from worker""" + + +class LoggedException(Exception): + """Raise exception for handling without re-logging""" + + +def report_worker_exc(wrk_exc=None): + """Write worker exception to log""" + if wrk_exc is not None: + from_line, msg, exc = wrk_exc.args + logger.error("---- {} ----".format(from_line)) + logger.error("Message: {}".format(msg)) + logger.error(exc) def manager_main(hist, libE_specs, alloc_specs, @@ -69,7 +89,7 @@ def manager_main(hist, libE_specs, alloc_specs, wcomms: :obj:`list`, optional A list of comm type objects for each worker. Default is an empty list. """ - if sim_specs.get('profile'): + if libE_specs.get('profile'): pr = cProfile.Profile() pr.enable() @@ -77,8 +97,13 @@ def manager_main(hist, libE_specs, alloc_specs, gen_specs['in'] = [] # Send dtypes to workers - dtypes = {EVAL_SIM_TAG: hist.H[sim_specs['in']].dtype, - EVAL_GEN_TAG: hist.H[gen_specs['in']].dtype} + if 'repack_fields' in globals(): + dtypes = {EVAL_SIM_TAG: repack_fields(hist.H[sim_specs['in']]).dtype, + EVAL_GEN_TAG: repack_fields(hist.H[gen_specs['in']]).dtype} + else: + dtypes = {EVAL_SIM_TAG: hist.H[sim_specs['in']].dtype, + EVAL_GEN_TAG: hist.H[gen_specs['in']].dtype} + for wcomm in wcomms: wcomm.send(0, dtypes) @@ -87,7 +112,7 @@ def manager_main(hist, libE_specs, alloc_specs, sim_specs, gen_specs, exit_criteria, wcomms) result = mgr.run(persis_info) - if sim_specs.get('profile'): + if libE_specs.get('profile'): pr.disable() profile_stats_fname = 'manager.prof' @@ -122,6 +147,7 @@ class Manager: worker_dtype = [('worker_id', int), ('active', int), ('persis_state', int), + ('active_recv', int), ('blocked', bool)] def __init__(self, hist, libE_specs, alloc_specs, @@ -151,27 +177,12 @@ def __init__(self, hist, libE_specs, alloc_specs, (1, 'gen_max', self.term_test_gen_max), (1, 'stop_val', self.term_test_stop_val)] - if any([setting in self.libE_specs for setting in libE_spec_calc_dir_combined]): - self.check_ensemble_dir(libE_specs) - if libE_specs.get('ensemble_copy_back', False): - Manager.make_copyback_dir(libE_specs) + temp_EnsembleDirectory = EnsembleDirectory(libE_specs=libE_specs) - @staticmethod - def make_copyback_dir(libE_specs): - ensemble_dir_path = libE_specs.get('ensemble_dir_path', './ensemble') - copybackdir = os.path.basename(ensemble_dir_path) # Current directory, same basename - if os.path.relpath(ensemble_dir_path) == os.path.relpath(copybackdir): - copybackdir += '_back' - os.makedirs(copybackdir) - - def check_ensemble_dir(self, libE_specs): - prefix = libE_specs.get('ensemble_dir_path', './ensemble') try: - os.rmdir(prefix) - except FileNotFoundError: # Ensemble dir doesn't exist. - pass + temp_EnsembleDirectory.make_copyback_check() except OSError as e: # Ensemble dir exists and isn't empty. - logger.manager_warning(_USER_CALC_DIR_WARNING.format(prefix)) + logger.manager_warning(_USER_CALC_DIR_WARNING.format(temp_EnsembleDirectory.prefix)) self._kill_workers() raise ManagerException('Manager errored on initialization', 'Ensemble directory already existed and wasn\'t empty.', e) @@ -184,7 +195,7 @@ def term_test_wallclock(self, max_elapsed): def term_test_sim_max(self, sim_max): """Checks against max simulations""" - return self.hist.given_count >= sim_max + self.hist.offset + return self.hist.returned_count >= sim_max + self.hist.offset def term_test_gen_max(self, gen_max): """Checks against max generator calls""" @@ -196,6 +207,18 @@ def term_test_stop_val(self, stop_val): H = self.hist.H return np.any(filter_nans(H[key][H['returned']]) <= val) + def work_giving_term_test(self, logged=True): + b = self.term_test() + if b: + return b + elif ('sim_max' in self.exit_criteria + and self.hist.given_count >= self.exit_criteria['sim_max'] + self.hist.offset): + # To avoid starting more sims if sim_max is an exit criteria + logger.info("Ignoring the alloc_f request for more sims than sim_max.") + return 1 + else: + return 0 + def term_test(self, logged=True): """Checks termination criteria""" for retval, key, testf in self.term_tests: @@ -227,7 +250,7 @@ def _save_every_k(self, fname, count, k): def _save_every_k_sims(self): "Saves history every kth sim step" self._save_every_k('libE_history_for_run_starting_{}_after_sim_{}.npy', - self.hist.sim_count, + self.hist.returned_count, self.libE_specs['save_every_k_sims']) def _save_every_k_gens(self): @@ -242,9 +265,14 @@ def _check_work_order(self, Work, w): """Checks validity of an allocation function order """ assert w != 0, "Can't send to worker 0; this is the manager." - assert self.W[w-1]['active'] == 0, \ - "Allocation function requested work be sent to to worker %d, an "\ - "already active worker." % w + if self.W[w-1]['active_recv']: + assert 'active_recv' in Work['libE_info'], \ + "Messages to a worker in active_recv mode should have active_recv"\ + "set to True in libE_info. Work['libE_info'] is {}".format(Work['libE_info']) + else: + assert self.W[w-1]['active'] == 0, \ + "Allocation function requested work be sent to worker %d, an "\ + "already active worker." % w work_rows = Work['libE_info']['H_rows'] if len(work_rows): work_fields = set(Work['H_fields']) @@ -264,8 +292,13 @@ def _send_work_order(self, Work, w): self.wcomms[w-1].send(Work['tag'], Work) work_rows = Work['libE_info']['H_rows'] if len(work_rows): - if 'repack_fields' in dir(): - self.wcomms[w-1].send(0, repack_fields(self.hist.H[Work['H_fields']][work_rows])) + if 'repack_fields' in globals(): + new_dtype = [(name, self.hist.H.dtype.fields[name][0]) for name in Work['H_fields']] + H_to_be_sent = np.empty(len(work_rows), dtype=new_dtype) + for i, row in enumerate(work_rows): + H_to_be_sent[i] = repack_fields(self.hist.H[Work['H_fields']][row]) + # H_to_be_sent = repack_fields(self.hist.H[Work['H_fields']])[work_rows] + self.wcomms[w-1].send(0, H_to_be_sent) else: self.wcomms[w-1].send(0, self.hist.H[Work['H_fields']][work_rows]) @@ -273,9 +306,14 @@ def _update_state_on_alloc(self, Work, w): """Updates a workers' active/idle status following an allocation order""" self.W[w-1]['active'] = Work['tag'] - if 'libE_info' in Work and 'persistent' in Work['libE_info']: - self.W[w-1]['persis_state'] = Work['tag'] - + if 'libE_info' in Work: + if 'persistent' in Work['libE_info']: + self.W[w-1]['persis_state'] = Work['tag'] + if Work['libE_info'].get('active_recv', False): + self.W[w-1]['active_recv'] = Work['tag'] + else: + assert 'active_recv' not in Work['libE_info'], \ + "active_recv worker must also be persistent" if 'blocking' in Work['libE_info']: for w_i in Work['libE_info']['blocking']: assert self.W[w_i-1]['active'] == 0, \ @@ -338,21 +376,31 @@ def _update_state_on_worker_msg(self, persis_info, D_recv, w): calc_type = D_recv['calc_type'] calc_status = D_recv['calc_status'] Manager._check_received_calc(D_recv) - - if w not in self.persis_pending: + if w not in self.persis_pending and not self.W[w-1]['active_recv']: self.W[w-1]['active'] = 0 - if calc_status in [FINISHED_PERSISTENT_SIM_TAG, FINISHED_PERSISTENT_GEN_TAG]: + final_data = D_recv.get('calc_out', None) + if isinstance(final_data, np.ndarray): + if self.libE_specs.get('use_persis_return', False): + if calc_status is FINISHED_PERSISTENT_GEN_TAG: + self.hist.update_history_x_in(w, final_data, self.safe_mode) + else: + self.hist.update_history_f(D_recv, self.safe_mode) + else: + logger.info(_PERSIS_RETURN_WARNING) self.W[w-1]['persis_state'] = 0 + if self.W[w-1]['active_recv']: + self.W[w-1]['active'] = 0 + self.W[w-1]['active_recv'] = 0 if w in self.persis_pending: self.persis_pending.remove(w) self.W[w-1]['active'] = 0 else: if calc_type == EVAL_SIM_TAG: - self.hist.update_history_f(D_recv) + self.hist.update_history_f(D_recv, self.safe_mode) if calc_type == EVAL_GEN_TAG: - self.hist.update_history_x_in(w, D_recv['calc_out']) + self.hist.update_history_x_in(w, D_recv['calc_out'], self.safe_mode) assert len(D_recv['calc_out']) or np.any(self.W['active']) or self.W[w-1]['persis_state'], \ "Gen must return work when is is the only thing active and not persistent." if 'libE_info' in D_recv and 'persistent' in D_recv['libE_info']: @@ -378,19 +426,31 @@ def _handle_msg_from_worker(self, persis_info, w): except CommFinishedException: logger.debug("Finalizing message from Worker {}".format(w)) return - if isinstance(D_recv, WorkerErrMsg): self.W[w-1]['active'] = 0 if not self.WorkerExc: self.WorkerExc = True self._kill_workers() - raise ManagerException('Received error message from {}'.format(w), - D_recv.msg, D_recv.exc) + raise WorkerException('Received error message from worker {}'.format(w), + D_recv.msg, D_recv.exc) elif isinstance(D_recv, logging.LogRecord): logging.getLogger(D_recv.name).handle(D_recv) else: self._update_state_on_worker_msg(persis_info, D_recv, w) + def _kill_cancelled_sims(self): + kill_sim = self.hist.H['given'] & self.hist.H['cancel_requested'] \ + & ~self.hist.H['returned'] & ~self.hist.H['kill_sent'] + + if np.any(kill_sim): + logger.debug('Manager sending kill signals to H indices {}'.format(np.where(kill_sim))) + kill_ids = self.hist.H['sim_id'][kill_sim] + kill_on_workers = self.hist.H['sim_worker'][kill_sim] + for w in kill_on_workers: + self.wcomms[w-1].send(STOP_TAG, MAN_SIGNAL_KILL) + self.hist.H['kill_sent'][kill_ids] = True + # SH*** Still expecting return? Currrently yes.... else set returned and inactive sim here. + # --- Handle termination def _final_receive_and_kill(self, persis_info): @@ -406,7 +466,13 @@ def _final_receive_and_kill(self, persis_info): if any(self.W['persis_state']): for w in self.W['worker_id'][self.W['persis_state'] > 0]: logger.debug("Manager sending PERSIS_STOP to worker {}".format(w)) - self.wcomms[w-1].send(PERSIS_STOP, MAN_SIGNAL_KILL) + if 'final_fields' in self.libE_specs: + rows_to_send = self.hist.trim_H()['returned'] + fields_to_send = self.libE_specs['final_fields'] + H_to_send = self.hist.trim_H()[rows_to_send][fields_to_send] + self.wcomms[w-1].send(PERSIS_STOP, H_to_send) + else: + self.wcomms[w-1].send(PERSIS_STOP, MAN_SIGNAL_KILL) if not self.W[w-1]['active']: # Re-activate if necessary self.W[w-1]['active'] = self.W[w-1]['persis_state'] @@ -437,7 +503,10 @@ def _alloc_work(self, H, persis_info): fields before the alloc_f call and ensures they weren't modified """ if self.safe_mode: - saveH = copy.deepcopy(H[protected_libE_fields]) + if 'repack_fields' in globals(): + saveH = repack_fields(H[protected_libE_fields], recurse=True) + else: + saveH = copy.deepcopy(H[protected_libE_fields]) alloc_f = self.alloc_specs['alloc_f'] output = alloc_f(self.W, H, self.sim_specs, self.gen_specs, self.alloc_specs, persis_info) @@ -463,6 +532,7 @@ def run(self, persis_info): # Continue receiving and giving until termination test is satisfied try: while not self.term_test(): + self._kill_cancelled_sims() persis_info = self._receive_from_workers(persis_info) if self.anything_new: self.anything_new = False @@ -472,14 +542,19 @@ def run(self, persis_info): break for w in Work: - if self.term_test(): + if self.work_giving_term_test(): break self._check_work_order(Work[w], w) self._send_work_order(Work[w], w) self._update_state_on_alloc(Work[w], w) assert self.term_test() or any(self.W['active'] != 0), \ "alloc_f did not return any work, although all workers are idle." - + except WorkerException as e: + report_worker_exc(e) + raise LoggedException(e.args[0], e.args[1]) from None + except Exception as e: + logger.error(traceback.format_exc()) + raise LoggedException(e.args) from None finally: # Return persis_info, exit_flag, elapsed time result = self._final_receive_and_kill(persis_info) diff --git a/libensemble/output_directory.py b/libensemble/output_directory.py new file mode 100644 index 0000000000..617d42e2a8 --- /dev/null +++ b/libensemble/output_directory.py @@ -0,0 +1,222 @@ +import os +import re +import shutil +from itertools import groupby +from operator import itemgetter + +from libensemble.utils.loc_stack import LocationStack +from libensemble.tools.fields_keys import libE_spec_sim_dir_keys, libE_spec_gen_dir_keys, \ + libE_spec_calc_dir_misc +from libensemble.message_numbers import EVAL_SIM_TAG, calc_type_strings + + +class EnsembleDirectory: + """ + The EnsembleDirectory class provides methods for workers to initialize and + manipulate the optional output directories where workers can change to before + calling user functions. + + The top-level ensemble directory typically stores unique sub-directories containing results + for each libEnsemble user function call. This can be a separate location + on other filesystems or directories (like scratch spaces). + + When libEnsemble is initialized in a Distributed fashion, each worker can + initiate its own ensemble directory on the local node, and copy + back its results on completion or exception into the directory that libEnsemble + was originally launched from. + + Ensemble directory behavior can be configured via separate libE_specs + dictionary entries or defining an EnsembleDirectory object within libE_specs. + + Parameters + ---------- + libE_specs: dict + Parameters/information for libE operations. EnsembleDirectory only extracts + values specific for ensemble directory operations. Can technically contain + a different set of settings then the libE_specs passed to libE(). + + loc_stack: object + A LocationStack object from libEnsemble's internal libensemble.utils.loc_stack module. + """ + + def __init__(self, libE_specs=None, loc_stack=None): + + self.specs = libE_specs + self.loc_stack = loc_stack + + if self.specs is not None: + self.prefix = self.specs.get('ensemble_dir_path', './ensemble') + self.use_worker_dirs = self.specs.get('use_worker_dirs', False) + self.sim_input_dir = self.specs.get('sim_input_dir', '').rstrip('/') + self.sim_dirs_make = self.specs.get('sim_dirs_make', False) + self.sim_dir_copy_files = self.specs.get('sim_dir_copy_files', []) + self.sim_dir_symlink_files = self.specs.get('sim_dir_symlink_files', []) + self.gen_input_dir = self.specs.get('gen_input_dir', '').rstrip('/') + self.gen_dirs_make = self.specs.get('gen_dirs_make', False) + self.gen_dir_copy_files = self.specs.get('gen_dir_copy_files', []) + self.gen_dir_symlink_files = self.specs.get('gen_dir_symlink_files', []) + self.ensemble_copy_back = self.specs.get('ensemble_copy_back', False) + self.sim_use = any([i in self.specs for i in libE_spec_sim_dir_keys + libE_spec_calc_dir_misc]) + self.gen_use = any([i in self.specs for i in libE_spec_gen_dir_keys + libE_spec_calc_dir_misc]) + + def _make_copyback_dir(self): + """Make copyback directory, adding suffix if identical to ensemble dir""" + copybackdir = os.path.basename(self.prefix) # Current directory, same basename + if os.path.relpath(self.prefix) == os.path.relpath(copybackdir): + copybackdir += '_back' + os.makedirs(copybackdir) + + def make_copyback_check(self): + """Check for existing copyback, make copyback if doesn't exist""" + try: + os.rmdir(self.prefix) + except FileNotFoundError: + pass + except Exception: + raise + if self.ensemble_copy_back: + self._make_copyback_dir() + + def use_calc_dirs(self, type): + """Determines calc_dirs enabling for each calc type""" + + if type == EVAL_SIM_TAG: + return self.sim_use + else: + return self.gen_use + + @staticmethod + def extract_H_ranges(Work): + """Convert received H_rows into ranges for labeling """ + work_H_rows = Work['libE_info']['H_rows'] + if len(work_H_rows) == 1: + return str(work_H_rows[0]) + else: + # From https://stackoverflow.com/a/30336492 + ranges = [] + for diff, group in groupby(enumerate(work_H_rows.tolist()), lambda x: x[0]-x[1]): + group = list(map(itemgetter(1), group)) + if len(group) > 1: + ranges.append(str(group[0]) + '-' + str(group[-1])) + else: + ranges.append(str(group[0])) + return '_'.join(ranges) + + def _make_calc_dir(self, workerID, H_rows, calc_str, locs): + """Create calc dirs and intermediate dirs, copy inputs, based on libE_specs""" + + if calc_str == 'sim': + input_dir = self.sim_input_dir + do_calc_dirs = self.sim_dirs_make + copy_files = self.sim_dir_copy_files + symlink_files = self.sim_dir_symlink_files + else: # calc_str is 'gen' + input_dir = self.gen_input_dir + do_calc_dirs = self.gen_dirs_make + copy_files = self.gen_dir_copy_files + symlink_files = self.gen_dir_symlink_files + + # If 'use_worker_dirs' only calc_dir option. Use worker dirs, but no calc dirs + if self.use_worker_dirs and not self.sim_dirs_make and not self.gen_dirs_make: + do_calc_dirs = False + + # If using input_dir, set of files to copy is contents of provided dir + if input_dir: + copy_files = set(copy_files + [os.path.join(input_dir, i) for i in os.listdir(input_dir)]) + + # If identical paths to copy and symlink, remove those paths from symlink_files + if len(symlink_files): + symlink_files = [i for i in symlink_files if i not in copy_files] + + # Cases where individual sim_dirs or gen_dirs not created. + if not do_calc_dirs: + if self.use_worker_dirs: # Each worker does work in worker dirs + key = workerID + dir = "worker" + str(workerID) + prefix = self.prefix + else: # Each worker does work in prefix (ensemble_dir) + key = self.prefix + dir = self.prefix + prefix = None + + locs.register_loc(key, dir, prefix=prefix, copy_files=copy_files, + symlink_files=symlink_files, ignore_FileExists=True) + return key + + # All cases now should involve sim_dirs or gen_dirs + # ensemble_dir/worker_dir registered here, set as parent dir for calc dirs + if self.use_worker_dirs: + worker_dir = "worker" + str(workerID) + worker_path = os.path.abspath(os.path.join(self.prefix, worker_dir)) + calc_dir = calc_str + str(H_rows) + locs.register_loc(workerID, worker_dir, prefix=self.prefix) + calc_prefix = worker_path + + # Otherwise, ensemble_dir set as parent dir for sim dirs + else: + calc_dir = "{}{}_worker{}".format(calc_str, H_rows, workerID) + if not os.path.isdir(self.prefix): + os.makedirs(self.prefix, exist_ok=True) + calc_prefix = self.prefix + + # Register calc dir with adjusted parent dir and source-file location + locs.register_loc(calc_dir, calc_dir, # Dir name also label in loc stack dict + prefix=calc_prefix, + copy_files=copy_files, + symlink_files=symlink_files) + + return calc_dir + + def prep_calc_dir(self, Work, calc_iter, workerID, calc_type): + """Determines choice for calc_dir structure, then performs calculation.""" + + if not self.loc_stack: + self.loc_stack = LocationStack() + + if calc_type == EVAL_SIM_TAG: + H_rows = self.extract_H_ranges(Work) + else: + H_rows = str(calc_iter[calc_type]) + + calc_str = calc_type_strings[calc_type] + + calc_dir = self._make_calc_dir(workerID, H_rows, calc_str, self.loc_stack) + + return self.loc_stack, calc_dir + + def copy_back(self): + """Copy back all ensemble dir contents to launch location""" + if os.path.isdir(self.prefix) and self.ensemble_copy_back: + + no_calc_dirs = not self.sim_dirs_make or \ + not self.gen_dirs_make + + copybackdir = os.path.basename(self.prefix) + + if os.path.relpath(self.prefix) == os.path.relpath(copybackdir): + copybackdir += '_back' + + for dir in self.loc_stack.dirs.values(): + dest_path = os.path.join(copybackdir, os.path.basename(dir)) + if dir == self.prefix: # occurs when no_calc_dirs is True + continue # otherwise, entire ensemble dir copied into copyback dir + + shutil.copytree(dir, dest_path, symlinks=True) + if os.path.basename(dir).startswith('worker'): + return # Worker dir (with all contents) has been copied. + + # If not using calc dirs, likely miscellaneous files to copy back + if no_calc_dirs: + p = re.compile(r"((^sim)|(^gen))\d+_worker\d+") + for file in [i for i in os.listdir(self.prefix) if not p.match(i)]: # each non-calc_dir file + source_path = os.path.join(self.prefix, file) + dest_path = os.path.join(copybackdir, file) + try: + if os.path.isdir(source_path): + shutil.copytree(source_path, dest_path, symlinks=True) + else: + shutil.copy(source_path, dest_path, follow_symlinks=False) + except FileExistsError: + continue + except shutil.SameFileError: # creating an identical symlink + continue diff --git a/libensemble/resources/env_resources.py b/libensemble/resources/env_resources.py index 22608a5a06..28a97c1904 100644 --- a/libensemble/resources/env_resources.py +++ b/libensemble/resources/env_resources.py @@ -3,6 +3,7 @@ """ import os +import re import logging from collections import OrderedDict @@ -126,13 +127,13 @@ def _range_split(s): return a, b, nnum_len @staticmethod - def _noderange_append(prefix, nidstr): + def _noderange_append(prefix, nidstr, suffix): """Formats and appends nodes to overall nodelist""" nidlst = [] for nidgroup in nidstr.split(','): a, b, nnum_len = EnvResources._range_split(nidgroup) for nid in range(a, b): - nidlst.append(prefix + str(nid).zfill(nnum_len)) + nidlst.append(prefix + str(nid).zfill(nnum_len) + suffix) return nidlst @staticmethod @@ -141,24 +142,22 @@ def get_slurm_nodelist(node_list_env): fullstr = os.environ[node_list_env] if not fullstr: return [] - part_splitstr = fullstr.split('],') - if len(part_splitstr) == 1: # Single Partition - splitstr = fullstr.split('[', 1) - if len(splitstr) == 1: # Single Node - return splitstr - prefix = splitstr[0] - nidstr = splitstr[1].strip("]") - nidlst = EnvResources._noderange_append(prefix, nidstr) - else: # Multiple Partitions - splitgroups = [str.split('[', 1) for str in part_splitstr] - prefixgroups = [group[0] for group in splitgroups] - nodegroups = [group[1].strip(']') for group in splitgroups] - nidlst = [] - for i in range(len(prefixgroups)): - prefix = prefixgroups[i] - nidstr = nodegroups[i] - nidlst.extend(EnvResources._noderange_append(prefix, nidstr)) - + # Split at commas outside of square brackets + r = re.compile(r'(?:[^,\[]|\[[^\]]*\])+') + part_splitstr = r.findall(fullstr) + nidlst = [] + for i in range(len(part_splitstr)): + part = part_splitstr[i] + splitstr = part.split('[', 1) + if len(splitstr) == 1: + nidlst.append(splitstr[0]) + else: + prefix = splitstr[0] + remainder = splitstr[1] + splitstr = remainder.split(']', 1) + nidstr = splitstr[0] + suffix = splitstr[1] + nidlst.extend(EnvResources._noderange_append(prefix, nidstr, suffix)) return sorted(nidlst) @staticmethod diff --git a/libensemble/resources/mpi_resources.py b/libensemble/resources/mpi_resources.py index f92fdac04c..c27dd5de6b 100644 --- a/libensemble/resources/mpi_resources.py +++ b/libensemble/resources/mpi_resources.py @@ -67,14 +67,12 @@ def get_resources(self, num_procs=None, num_nodes=None, raised if these are infeasible. """ node_list = self.worker_resources.local_nodelist - num_workers = self.worker_resources.num_workers local_node_count = self.worker_resources.local_node_count cores_avail_per_node = \ (self.logical_cores_avail_per_node if hyperthreads else self.physical_cores_avail_per_node) - workers_per_node = \ - (self.worker_resources.workers_per_node if num_workers > local_node_count else 1) + workers_per_node = self.worker_resources.workers_on_node cores_avail_per_node_per_worker = cores_avail_per_node//workers_per_node rassert(node_list, "Node list is empty - aborting") diff --git a/libensemble/resources/resources.py b/libensemble/resources/resources.py index 249da52207..e993826f8d 100644 --- a/libensemble/resources/resources.py +++ b/libensemble/resources/resources.py @@ -6,9 +6,7 @@ import os import socket import logging -import itertools import subprocess -# from collections import OrderedDict from libensemble.resources import node_resources from libensemble.resources.env_resources import EnvResources @@ -122,13 +120,19 @@ def __init__(self, top_level_dir=None, nodelist_env_lsf=nodelist_env_lsf, nodelist_env_lsf_shortform=nodelist_env_lsf_shortform) - # This is global nodelist avail to workers - may change to global_worker_nodelist - self.local_host = self.env_resources.shortnames([socket.gethostname()])[0] if node_file is None: node_file = Resources.DEFAULT_NODEFILE + self.global_nodelist = Resources.get_global_nodelist(node_file=node_file, rundir=self.top_level_dir, env_resources=self.env_resources) + + self.shortnames = Resources.is_nodelist_shortnames(self.global_nodelist) + if self.shortnames: + self.local_host = self.env_resources.shortnames([socket.gethostname()])[0] + else: + self.local_host = socket.gethostname() + self.launcher = launcher remote_detect = False if self.local_host not in self.global_nodelist: @@ -150,7 +154,10 @@ def add_comm_info(self, libE_nodes): Removes libEnsemble nodes from nodelist if in central_mode. """ - self.libE_nodes = self.env_resources.shortnames(libE_nodes) + if self.shortnames: + self.libE_nodes = self.env_resources.shortnames(libE_nodes) + else: + self.libE_nodes = libE_nodes libE_nodes_in_list = list(filter(lambda x: x in self.libE_nodes, self.global_nodelist)) if libE_nodes_in_list: if self.central_mode and len(self.global_nodelist) > 1: @@ -203,6 +210,14 @@ def get_MPI_variant(): # --------------------------------------------------------------------------- + @staticmethod + def is_nodelist_shortnames(nodelist): + """Returns True if any entry contains a '.', else False""" + for item in nodelist: + if '.' in item: + return False + return True + # This is for central mode where libE nodes will not share with app nodes @staticmethod def remove_nodes(global_nodelist_in, remove_list): @@ -237,8 +252,9 @@ def get_global_nodelist(node_file=DEFAULT_NODEFILE, with open(node_filepath, 'r') as f: for line in f: global_nodelist.append(line.rstrip()) - if env_resources: - global_nodelist = env_resources.shortnames(global_nodelist) + # Expect correct format - if anything - could have an option to truncate. + # if env_resources: + # global_nodelist = env_resources.shortnames(global_nodelist) else: logger.debug("No node_file found - searching for nodelist in environment") if env_resources: @@ -247,7 +263,8 @@ def get_global_nodelist(node_file=DEFAULT_NODEFILE, if not global_nodelist: # Assume a standalone machine logger.info("Can not find nodelist from environment. Assuming standalone") - global_nodelist.append(env_resources.shortnames([socket.gethostname()])[0]) + # global_nodelist.append(env_resources.shortnames([socket.gethostname()])[0]) + global_nodelist.append(socket.gethostname()) if global_nodelist: return global_nodelist @@ -289,17 +306,10 @@ def __init__(self, workerID, comm, resources): """ self.num_workers = comm.get_num_workers() self.workerID = workerID - self.local_nodelist = WorkerResources.get_local_nodelist(self.num_workers, self.workerID, resources) + self.local_nodelist, self.workers_on_node = \ + WorkerResources.get_local_nodelist(self.num_workers, self.workerID, resources) self.local_node_count = len(self.local_nodelist) - self.workers_per_node = WorkerResources.get_workers_on_a_node(self.num_workers, resources) - - @staticmethod - def get_workers_on_a_node(num_workers, resources): - """Returns the number of workers that can be placed on each node""" - num_nodes = len(resources.global_nodelist) - # Round up if theres a remainder - workers_per_node = num_workers//num_nodes + (num_workers % num_nodes > 0) - return workers_per_node + self.num_workers_2assign2 = WorkerResources.get_workers2assign2(self.num_workers, resources) @staticmethod def map_workerid_to_index(num_workers, workerID, zero_resource_list): @@ -314,6 +324,34 @@ def map_workerid_to_index(num_workers, workerID, zero_resource_list): raise ResourcesException("Error mapping workerID {} to nodelist index {}".format(workerID, index)) return index + @staticmethod + def get_workers2assign2(num_workers, resources): + """Returns workers to assign resources to""" + zero_resource_list = resources.zero_resource_workers + return num_workers - len(zero_resource_list) + + @staticmethod + def even_assignment(nnodes, nworkers): + """Returns True if workers are evenly distributied to nodes, else False""" + return nnodes % nworkers == 0 or nworkers % nnodes == 0 + + @staticmethod + def expand_list(nnodes, nworkers, nodelist): + """Duplicates each element of ``nodelist`` to best map workers to nodes. + + Returns node list with duplicates, and a list of local (on-node) worker + counts, both indexed by worker. + """ + k, m = divmod(nworkers, nnodes) + dup_list = [] + local_workers_list = [] + for i, x in enumerate(nodelist): + repeats = k + 1 if i < m else k + for j in range(repeats): + dup_list.append(x) + local_workers_list.append(repeats) + return dup_list, local_workers_list + @staticmethod def get_local_nodelist(num_workers, workerID, resources): """Returns the list of nodes available to the current worker @@ -321,31 +359,22 @@ def get_local_nodelist(num_workers, workerID, resources): Assumes that self.global_nodelist has been calculated (in __init__). Also self.global_nodelist will have already removed non-application nodes """ - global_nodelist = resources.global_nodelist num_nodes = len(global_nodelist) zero_resource_list = resources.zero_resource_workers - num_workers_2assign2 = num_workers - len(zero_resource_list) + num_workers_2assign2 = WorkerResources.get_workers2assign2(num_workers, resources) - # Check if current host in nodelist - if it is then in distributed mode. - distrib_mode = resources.local_host in global_nodelist + if not WorkerResources.even_assignment(num_nodes, num_workers_2assign2): + logger.warning('Workers with assigned resources ({}) are not distributed evenly to available nodes ({})' + .format(num_workers_2assign2, num_nodes)) # If multiple workers per node - create global node_list with N duplicates (for N workers per node) sub_node_workers = (num_workers_2assign2 >= num_nodes) if sub_node_workers: - workers_per_node = num_workers_2assign2//num_nodes - dup_list = itertools.chain.from_iterable(itertools.repeat(x, workers_per_node) for x in global_nodelist) - global_nodelist = list(dup_list) - - # Currently require even split for distrib mode - to match machinefile - throw away remainder - if distrib_mode and not sub_node_workers: - nodes_per_worker, remainder = divmod(num_nodes, num_workers_2assign2) - if remainder != 0: - # Worker node may not be at head of list after truncation - should perhaps be warning or enforced - logger.warning("Nodes to workers not evenly distributed. Wasted nodes. " - "{} workers and {} nodes".format(num_workers_2assign2, num_nodes)) - num_nodes = num_nodes - remainder - global_nodelist = global_nodelist[0:num_nodes] + global_nodelist, local_workers_list = \ + WorkerResources.expand_list(num_nodes, num_workers_2assign2, global_nodelist) + else: + local_workers_list = [1] * num_workers_2assign2 # Divide global list between workers split_list = list(Resources.best_split(global_nodelist, num_workers_2assign2)) @@ -356,10 +385,12 @@ def get_local_nodelist(num_workers, workerID, resources): if workerID in zero_resource_list: local_nodelist = [] + workers_on_node = 0 logger.debug("Worker is a zero-resource worker") else: index = WorkerResources.map_workerid_to_index(num_workers, workerID, zero_resource_list) local_nodelist = split_list[index] + workers_on_node = local_workers_list[index] logger.debug("Worker's local_nodelist is {}".format(local_nodelist)) - return local_nodelist + return local_nodelist, workers_on_node diff --git a/libensemble/sim_funcs/borehole.py b/libensemble/sim_funcs/borehole.py index 528fe6a555..bc7409bf2b 100644 --- a/libensemble/sim_funcs/borehole.py +++ b/libensemble/sim_funcs/borehole.py @@ -5,7 +5,7 @@ [990, 1110], [700, 820], [0, np.inf], # Not sure if the physics have a more meaningful upper bound - [1, 1.2], # Very low probability of being outside of this range + [0.05, 0.15], # Very low probability of being outside of this range [9855, 12045], [1120, 1680]]) @@ -79,7 +79,7 @@ def gen_borehole_input(n): Hu = np.random.uniform(990, 1110, n) Hl = np.random.uniform(700, 820, n) r = np.random.lognormal(7.71, 1.0056, n) - rw = np.random.normal(1.1, 0.0161812, n) + rw = np.random.normal(0.1, 0.0161812, n) Kw = np.random.uniform(9855, 12045, n) L = np.random.uniform(1120, 1680, n) diff --git a/libensemble/sim_funcs/borehole_kills.py b/libensemble/sim_funcs/borehole_kills.py new file mode 100644 index 0000000000..8e4840fc80 --- /dev/null +++ b/libensemble/sim_funcs/borehole_kills.py @@ -0,0 +1,57 @@ +import numpy as np +from libensemble.executors.executor import Executor +from libensemble.sim_funcs.surmise_test_function import borehole_true +from libensemble.message_numbers import TASK_FAILED, MAN_SIGNAL_KILL, UNSET_TAG + + +def subproc_borehole(H, delay): + """This evaluates the Borehole function using a subprocess + running compiled code. + + Note that the Executor base class submit runs a + serial process in-place. This should work on compute nodes + so long as there are free contexts. + + """ + with open('input', 'w') as f: + H['thetas'][0].tofile(f) + H['x'][0].tofile(f) + + exctr = Executor.executor + args = 'input' + ' ' + str(delay) + + task = exctr.submit(app_name='borehole', app_args=args, stdout='out.txt', stderr='err.txt') + calc_status = exctr.polling_loop(task, delay=0.01, poll_manager=True) + + if calc_status in [MAN_SIGNAL_KILL, TASK_FAILED]: + f = np.inf + else: + f = float(task.read_stdout()) + return f, calc_status + + +def borehole(H, persis_info, sim_specs, libE_info): + """ + Wraps the borehole function + Subprocess to test receiving kill signals from manager + """ + calc_status = UNSET_TAG # Calc_status gets printed in libE_stats.txt + H_o = np.zeros(H['x'].shape[0], dtype=sim_specs['out']) + + # Add a delay so subprocessed borehole takes longer + sim_id = libE_info['H_rows'][0] + delay = 0 + if sim_id > sim_specs['user']['init_sample_size']: + delay = 2 + np.random.normal(scale=0.5) + + f, calc_status = subproc_borehole(H, delay) + + # Failure model (excluding observations) + if sim_id > sim_specs['user']['num_obs']: + if (f / borehole_true(H['x'])) > 1.25: + f = np.inf + calc_status = TASK_FAILED + print('Failure of sim_id {}'.format(sim_id), flush=True) + + H_o['f'] = f + return H_o, persis_info, calc_status diff --git a/libensemble/sim_funcs/executor_hworld.py b/libensemble/sim_funcs/executor_hworld.py index 079d3eb214..f17df5e848 100644 --- a/libensemble/sim_funcs/executor_hworld.py +++ b/libensemble/sim_funcs/executor_hworld.py @@ -7,10 +7,10 @@ __all__ = ['executor_hworld'] # Alt send values through X -sim_count = 0 +returned_count = 0 -def polling_loop(comm, exctr, task, timeout_sec=3.0, delay=0.3): +def custom_polling_loop(exctr, task, timeout_sec=3.0, delay=0.3): import time calc_status = UNSET_TAG # Sim func determines status of libensemble calc - returned to worker @@ -19,7 +19,7 @@ def polling_loop(comm, exctr, task, timeout_sec=3.0, delay=0.3): time.sleep(delay) # print('Probing manager at time: ', task.runtime) - exctr.manager_poll(comm) + exctr.manager_poll() if exctr.manager_signal == 'finish': exctr.kill(task) calc_status = MAN_SIGNAL_FINISH # Worker will pick this up and close down @@ -69,31 +69,29 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): """ Tests launching and polling task and exiting on task finish""" exctr = MPIExecutor.executor cores = sim_specs['user']['cores'] - comm = libE_info['comm'] - use_balsam = 'balsam_test' in sim_specs['user'] args_for_sim = 'sleep 1' # pref send this in X as a sim_in from calling script - global sim_count - sim_count += 1 + global returned_count + returned_count += 1 timeout = 6.0 wait = False launch_shc = False - if sim_count == 1: + if returned_count == 1: args_for_sim = 'sleep 1' # Should finish - elif sim_count == 2: + elif returned_count == 2: args_for_sim = 'sleep 1 Error' # Worker kill on error - if sim_count == 3: + if returned_count == 3: wait = True args_for_sim = 'sleep 1' # Should finish launch_shc = True - elif sim_count == 4: + elif returned_count == 4: args_for_sim = 'sleep 3' # Worker kill on timeout timeout = 1.0 - elif sim_count == 5: + elif returned_count == 5: args_for_sim = 'sleep 1 Fail' # Manager kill - if signal received else completes - elif sim_count == 6: + elif returned_count == 6: args_for_sim = 'sleep 60' # Manager kill - if signal received else completes timeout = 65.0 @@ -114,7 +112,13 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): calc_status = TASK_FAILED else: - task, calc_status = polling_loop(comm, exctr, task, timeout) + if returned_count >= 2: + calc_status = exctr.polling_loop(task, timeout=timeout) + if returned_count == 2 and task.stdout_exists() and 'Error' in task.read_stdout(): + calc_status = WORKER_KILL_ON_ERR + + else: + task, calc_status = custom_polling_loop(exctr, task, timeout) if use_balsam: task.read_file_in_workdir('ensemble.log') @@ -123,9 +127,6 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): except ValueError: pass - # assert task.finished, "task.finished should be True. Returned " + str(task.finished) - # assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) - # This is temp - return something - so doing six_hump_camel_func again... batch = len(H['x']) H_o = np.zeros(batch, dtype=sim_specs['out']) @@ -142,10 +143,6 @@ def executor_hworld(H, persis_info, sim_specs, libE_info): # This is just for testing at calling script level - status of each task H_o['cstat'] = calc_status - # v = np.random.uniform(0, 10) - # print('About to sleep for :' + str(v)) - # time.sleep(v) - return H_o, persis_info, calc_status diff --git a/libensemble/sim_funcs/run_line_check.py b/libensemble/sim_funcs/run_line_check.py new file mode 100644 index 0000000000..57b1641357 --- /dev/null +++ b/libensemble/sim_funcs/run_line_check.py @@ -0,0 +1,95 @@ +from libensemble.message_numbers import WORKER_DONE +from libensemble.executors.executor import Executor +import numpy as np + + +def exp_nodelist_for_worker(exp_list, workerID, nodes_per_worker, persis_gens): + """Modify expected node-lists based on workerID""" + comps = exp_list.split() + new_line = [] + for comp in comps: + if comp.startswith('node-'): + new_node_list = [] + node_list = comp.split(',') + for node in node_list: + node_name, node_num = node.split('-') + offset = workerID - (1 + persis_gens) + new_num = int(node_num) + int(nodes_per_worker*offset) + new_node = '-'.join([node_name, str(new_num)]) + new_node_list.append(new_node) + new_list = ','.join(new_node_list) + new_line.append(new_list) + else: + new_line.append(comp) + return ' '.join(new_line) + + +def runline_check(H, persis_info, sim_specs, libE_info): + """Check run-lines produced by executor provided by a list of tests""" + calc_status = 0 + x = H['x'][0][0] + exctr = Executor.executor + test_list = sim_specs['user']['tests'] + exp_list = sim_specs['user']['expect'] + npw = sim_specs['user']['nodes_per_worker'] + p_gens = sim_specs['user'].get('persis_gens', 0) + + for i, test in enumerate(test_list): + task = exctr.submit(calc_type='sim', + num_procs=test.get('nprocs', None), + num_nodes=test.get('nnodes', None), + ranks_per_node=test.get('ppn', None), + extra_args=test.get('e_args', None), + app_args='--testid ' + test.get('testid', None), + stdout='out.txt', + stderr='err.txt', + hyperthreads=test.get('ht', None), + dry_run=True) + + outline = task.runline + new_exp_list = exp_nodelist_for_worker(exp_list[i], libE_info['workerID'], npw, p_gens) + + if outline != new_exp_list: + print('outline is: {}\nexp is: {}'.format(outline, new_exp_list), flush=True) + + assert(outline == new_exp_list) + + calc_status = WORKER_DONE + output = np.zeros(1, dtype=sim_specs['out']) + output['f'][0] = np.linalg.norm(x) + return output, persis_info, calc_status + + +def runline_check_by_worker(H, persis_info, sim_specs, libE_info): + """Check run-lines produced by executor provided by a list of lines per worker""" + calc_status = 0 + x = H['x'][0][0] + exctr = Executor.executor + test = sim_specs['user']['tests'][0] + exp_list = sim_specs['user']['expect'] + p_gens = sim_specs['user'].get('persis_gens', 0) + + task = exctr.submit(calc_type='sim', + num_procs=test.get('nprocs', None), + num_nodes=test.get('nnodes', None), + ranks_per_node=test.get('ppn', None), + extra_args=test.get('e_args', None), + app_args='--testid ' + test.get('testid', None), + stdout='out.txt', + stderr='err.txt', + hyperthreads=test.get('ht', None), + dry_run=True) + + outline = task.runline + wid = libE_info['workerID'] + new_exp_list = exp_list[wid-1-p_gens] + + if outline != new_exp_list: + print('Worker {}:\n outline is: {}\n exp is: {}'.format(wid, outline, new_exp_list), flush=True) + + assert(outline == new_exp_list) + + calc_status = WORKER_DONE + output = np.zeros(1, dtype=sim_specs['out']) + output['f'][0] = np.linalg.norm(x) + return output, persis_info, calc_status diff --git a/libensemble/sim_funcs/surmise_test_function.py b/libensemble/sim_funcs/surmise_test_function.py new file mode 100644 index 0000000000..4b08d9fc63 --- /dev/null +++ b/libensemble/sim_funcs/surmise_test_function.py @@ -0,0 +1,99 @@ +""" +Created on Tue Feb 9 10:27:23 2021 + +@author: mosesyhc +""" +import numpy as np + + +def borehole(H, persis_info, sim_specs, libE_info): + """ + Wraps the borehole function + """ + + H_o = np.zeros(H['x'].shape[0], dtype=sim_specs['out']) + + # If observation do not use failure model + sim_id = libE_info['H_rows'][0] + if sim_id > sim_specs['user']['num_obs']: + H_o['f'] = borehole_failmodel(H['x'], H['thetas']) + else: + H_o['f'] = borehole_model(H['x'], H['thetas']) + return H_o, persis_info + + +def borehole_failmodel(x, theta): + """Given x and theta, return matrix of [row x] times [row theta] of values.""" + f = borehole_model(x, theta) + wheretoobig = np.where((f / borehole_true(x)) > 1.25) + f[wheretoobig[0], wheretoobig[1]] = np.inf + return f + + +def borehole_model(x, theta): + """Given x and theta, return matrix of [row x] times [row theta] of values.""" + theta = tstd2theta(theta) + x = xstd2x(x) + p = x.shape[0] + n = theta.shape[0] + + theta_stacked = np.repeat(theta, repeats=p, axis=0) + x_stacked = np.tile(x.astype(float), (n, 1)) + + f = borehole_vec(x_stacked, theta_stacked).reshape((n, p)) + return f.T + + +def borehole_true(x): + """Given x, return matrix of [row x] times 1 of values.""" + # assume true theta is [0.5]^d + theta0 = np.atleast_2d(np.array([0.5] * 4)) + f0 = borehole_model(x, theta0) + + return f0 + + +def borehole_vec(x, theta): + """Given x and theta, return vector of values.""" + (Hu, Ld_Kw, Treff, powparam) = np.split(theta, theta.shape[1], axis=1) + (rw, Hl) = np.split(x[:, :-1], 2, axis=1) + numer = 2 * np.pi * (Hu - Hl) + denom1 = 2 * Ld_Kw / rw ** 2 + denom2 = Treff + + f = ((numer / ((denom1 + denom2))) * np.exp(powparam * rw)).reshape(-1) + + return f + + +def tstd2theta(tstd, hard=True): + """Given standardized theta in [0, 1]^d, return non-standardized theta.""" + if tstd.ndim < 1.5: + tstd = tstd[:, None].T + (Treffs, Hus, LdKw, powparams) = np.split(tstd, tstd.shape[1], axis=1) + + Treff = (0.5-0.05) * Treffs + 0.05 + Hu = Hus * (1110 - 990) + 990 + if hard: + Ld_Kw = LdKw * (1680/1500 - 1120/15000) + 1120/15000 + else: + Ld_Kw = LdKw*(1680/9855 - 1120/12045) + 1120/12045 + + powparam = powparams * (0.5 - (- 0.5)) + (-0.5) + + theta = np.hstack((Hu, Ld_Kw, Treff, powparam)) + return theta + + +def xstd2x(xstd): + """Given standardized x in [0, 1]^2 x {0, 1}, return non-standardized x.""" + if xstd.ndim < 1.5: + xstd = xstd[:, None].T + (rws, Hls, labels) = np.split(xstd, xstd.shape[1], axis=1) + + rw = rws * (np.log(0.5) - np.log(0.05)) + np.log(0.05) + rw = np.exp(rw) + Hl = Hls * (820 - 700) + 700 + + x = np.hstack((rw, Hl, labels)) + return x diff --git a/libensemble/tests/.coveragerc b/libensemble/tests/.coveragerc index bed6acb7bb..aa2a404c1d 100644 --- a/libensemble/tests/.coveragerc +++ b/libensemble/tests/.coveragerc @@ -19,11 +19,14 @@ omit = */unit_tests_logger/* */regression_tests/* */sim_funcs/helloworld.py + */sim_funcs/executor_hworld.py */balsam_executor.py */forkable_pdb.py */parse_args.py */sim_funcs/mop_funcs.py */gen_funcs/vtmop.py + */gen_funcs/old_aposmm.py + */alloc_funcs/fast_alloc_to_aposmm.py */regression_tests/test_vtmop.py exclude_lines = if __name__ == .__main__.: diff --git a/libensemble/tests/unit_tests/test_old_aposmm_logic.py b/libensemble/tests/deprecated_tests/test_old_aposmm_logic.py similarity index 100% rename from libensemble/tests/unit_tests/test_old_aposmm_logic.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_logic.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_one_residual_at_a_time.py b/libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_one_residual_at_a_time.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_one_residual_at_a_time.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_pounders.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_pounders.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_pounders.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_pounders_splitcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_pounders_splitcomm.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_pounders_splitcomm.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_pounders_subcomm.py b/libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_pounders_subcomm.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_pounders_subcomm.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_sim_dirs.py b/libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_sim_dirs.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_sim_dirs.py diff --git a/libensemble/tests/regression_tests/test_old_aposmm_with_gradients.py b/libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py similarity index 100% rename from libensemble/tests/regression_tests/test_old_aposmm_with_gradients.py rename to libensemble/tests/deprecated_tests/test_old_aposmm_with_gradients.py diff --git a/libensemble/tests/regression_tests/common.py b/libensemble/tests/regression_tests/common.py index c2d3b5c934..814420c648 100644 --- a/libensemble/tests/regression_tests/common.py +++ b/libensemble/tests/regression_tests/common.py @@ -3,9 +3,21 @@ """ import os +import json import os.path +def create_node_file(num_nodes, name='node_list'): + """Create a nodelist file""" + if os.path.exists(name): + os.remove(name) + with open(name, 'w') as f: + for i in range(1, num_nodes + 1): + f.write('node-' + str(i) + '\n') + f.flush() + os.fsync(f) + + def mpi_comm_excl(exc=[0], comm=None): "Exlude ranks from a communicator for MPI comms." from mpi4py import MPI @@ -38,6 +50,12 @@ def build_simfunc(): subprocess.check_call(buildstring.split()) +def build_borehole(): + import subprocess + buildstring = 'gcc -o borehole.x ../unit_tests/simdir/borehole.c -lm' + subprocess.check_call(buildstring.split()) + + def modify_Balsam_worker(): # Balsam is meant for HPC systems that commonly distribute jobs across many # nodes. Due to the nature of testing Balsam on local or CI systems which @@ -93,9 +111,22 @@ def modify_Balsam_pyCoverage(): f.write(line) +def modify_Balsam_settings(): + # Set $HOME/.balsam/settings.json to DEFAULT instead of Theta worker setup + settingsfile = os.path.join(os.environ.get('HOME'), '.balsam/settings.json') + with open(settingsfile, 'r') as f: + lines = json.load(f) + + lines['MPI_RUN_TEMPLATE'] = "MPICHCommand" + lines['WORKER_DETECTION_TYPE'] = "DEFAULT" + + with open(settingsfile, 'w') as f: + json.dump(lines, f) + + def modify_Balsam_JobEnv(): # If Balsam detects that the system on which it is running contains the string - # 'cc' in it's hostname, then it thinks it's on Cooley! Travis hostnames are + # 'cc' in its hostname, then it thinks it's on Cooley! Travis hostnames are # randomly generated and occasionally may contain that offending string. This # modifies Balsam's JobEnvironment class to not check for 'cc'. import balsam diff --git a/libensemble/tests/regression_tests/script_test_balsam_hworld.py b/libensemble/tests/regression_tests/script_test_balsam_hworld.py index d58ea89ac8..1addcb2402 100644 --- a/libensemble/tests/regression_tests/script_test_balsam_hworld.py +++ b/libensemble/tests/regression_tests/script_test_balsam_hworld.py @@ -1,7 +1,6 @@ # This script is submitted as an app and job to Balsam. The job submission is # via 'balsam launch' executed in the test_balsam_hworld.py script. -import os import numpy as np import mpi4py from mpi4py import MPI @@ -16,15 +15,6 @@ mpi4py.rc.recv_mprobe = False # Disable matching probes - -# Slighty different due to working directory not being /regression_tests -def build_simfunc(): - import subprocess - print('Balsam job launched in: {}'.format(os.getcwd())) - buildstring = 'mpicc -o my_simtask.x libensemble/tests/unit_tests/simdir/my_simtask.c' - subprocess.check_call(buildstring.split()) - - libE_specs = {'mpi_comm': MPI.COMM_WORLD, 'comms': 'mpi', 'save_every_k_sims': 400, @@ -37,8 +27,6 @@ def build_simfunc(): cores_per_task = 1 sim_app = './my_simtask.x' -if not os.path.isfile(sim_app): - build_simfunc() sim_app2 = six_hump_camel.__file__ exctr = BalsamMPIExecutor(auto_resources=False, central_mode=False, custom_info={'not': 'used'}) diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index bda85398fe..e1f8d40937 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -23,6 +23,7 @@ nworkers, is_manager, libE_specs, _ = parse_args() libE_specs['save_every_k_gens'] = 300 +libE_specs['safe_mode'] = False sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} @@ -34,7 +35,7 @@ } } -persis_info = add_unique_random_streams({}, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1, seed=1234) exit_criteria = {'gen_max': 501} diff --git a/libensemble/tests/regression_tests/test_1d_with_profile_sampling.py b/libensemble/tests/regression_tests/test_1d_with_profile_sampling.py index 1024ef6346..06e3d5df0e 100644 --- a/libensemble/tests/regression_tests/test_1d_with_profile_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_with_profile_sampling.py @@ -24,7 +24,7 @@ nworkers, is_manager, libE_specs, _ = parse_args() -libE_specs['profile_worker'] = True +libE_specs['profile'] = True sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} @@ -48,6 +48,9 @@ assert len(H) >= 501 print("\nlibEnsemble with random sampling has generated enough points") + assert 'manager.prof' in os.listdir(), 'Expected manager profile not found after run' + os.remove('manager.prof') + prof_files = ['worker_{}.prof'.format(i+1) for i in range(nworkers)] # Ensure profile writes complete before checking @@ -57,8 +60,8 @@ assert file in os.listdir(), 'Expected profile {} not found after run'.format(file) with open(file, 'r') as f: data = f.read().split() - num_worker_funcs_profiled = sum(['libE_worker' in i for i in data]) + num_worker_funcs_profiled = sum(['worker' in i for i in data]) assert num_worker_funcs_profiled >= 8, 'Insufficient number of ' + \ - 'libE_worker functions profiled: ' + str(num_worker_funcs_profiled) + 'worker functions profiled: ' + str(num_worker_funcs_profiled) os.remove(file) diff --git a/libensemble/tests/regression_tests/test_calc_exception.py b/libensemble/tests/regression_tests/test_calc_exception.py index 1b88122a07..5714d1ff6e 100644 --- a/libensemble/tests/regression_tests/test_calc_exception.py +++ b/libensemble/tests/regression_tests/test_calc_exception.py @@ -16,7 +16,7 @@ import numpy as np from libensemble.libE import libE -from libensemble.libE_manager import ManagerException +from libensemble.manager import LoggedException from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tools import parse_args, add_unique_random_streams @@ -49,7 +49,7 @@ def six_hump_camel_err(H, persis_info, sim_specs, _): try: H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) -except ManagerException as e: +except LoggedException as e: print("Caught deliberate exception: {}".format(e)) return_flag = 0 diff --git a/libensemble/tests/regression_tests/test_deap_nsga2.py b/libensemble/tests/regression_tests/test_deap_nsga2.py index 5f73f799a2..55f31b516c 100644 --- a/libensemble/tests/regression_tests/test_deap_nsga2.py +++ b/libensemble/tests/regression_tests/test_deap_nsga2.py @@ -35,15 +35,15 @@ def deap_six_hump(H, persis_info, sim_specs, _): assert nworkers >= 2, "Cannot run with a persistent gen_f if only one worker." # Number of generations, population size, indiviual size, and objectives -ngen = 100 -pop_size = 80 +ngen = 125 +pop_size = 100 ind_size = 2 num_obj = 2 # Variable Bounds (deap requires lists, not arrays!!!) lb = [-3.0, -2.0] ub = [3.0, 2.0] -w = (-1.0,) # Must be a tuple +w = (-1.0, -1.0) # Must be a tuple # State the objective function, its arguments, output, and necessary parameters (and their sizes) sim_specs = {'sim_f': deap_six_hump, # This is the function whose output is being minimized @@ -54,12 +54,13 @@ def deap_six_hump(H, persis_info, sim_specs, _): # State the generating function, its arguments, output, and necessary parameters. gen_specs = {'gen_f': gen_f, 'in': ['sim_id', 'generation', 'individual', 'fitness_values'], - 'out': [('individual', float, ind_size), ('generation', int)], + 'out': [('individual', float, ind_size), ('generation', int), ('last_points', bool)], 'user': {'lb': lb, 'ub': ub, 'weights': w, 'pop_size': pop_size, 'indiv_size': ind_size, + 'give_all_with_same_priority': True, 'cxpb': 0.8, # probability two individuals are crossed 'eta': 20.0, # large eta = low variation in children 'indpb': 0.8/ind_size} # end user @@ -103,6 +104,9 @@ def deap_six_hump(H, persis_info, sim_specs, _): H_dummy['individual'] = x objs = deap_six_hump(H_dummy, {}, sim_specs, {}) H0['fitness_values'][i] = objs[0] + + # Testing use_persis_return capabilities + libE_specs['use_persis_return'] = True else: H0 = None @@ -110,8 +114,15 @@ def deap_six_hump(H, persis_info, sim_specs, _): H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs, H0=H0) if is_manager: + if run == 0: + assert np.sum(H['last_points']) == 0, ("The last_points shouldn't be marked (even though " + "they were marked in the gen) as 'use_persis_return' was false.") + elif run == 1: + assert np.sum(H['last_points']) == pop_size, ("The last_points should be marked as true because they " + "were marked in the manager and 'use_persis_return' is true.") + script_name = os.path.splitext(os.path.basename(__file__))[0] assert flag == 0, script_name + " didn't exit correctly" assert sum(H['returned']) >= exit_criteria['sim_max'], script_name + " didn't evaluate the sim_max points." - assert min(H['fitness_values'][:, 0]) <= 1e-3, script_name + " didn't find the minimum for objective 0." + assert min(H['fitness_values'][:, 0]) <= 4e-3, script_name + " didn't find the minimum for objective 0." assert min(H['fitness_values'][:, 1]) <= -1.0, script_name + " didn't find the minimum for objective 1." diff --git a/libensemble/tests/regression_tests/test_elapsed_time_abort.py b/libensemble/tests/regression_tests/test_elapsed_time_abort.py index 525c01dc3d..80ddba3244 100644 --- a/libensemble/tests/regression_tests/test_elapsed_time_abort.py +++ b/libensemble/tests/regression_tests/test_elapsed_time_abort.py @@ -40,7 +40,7 @@ } alloc_specs = {'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)], + 'out': [], 'user': {'batch_mode': False, 'num_active_gens': 2}} diff --git a/libensemble/tests/regression_tests/test_fast_alloc.py b/libensemble/tests/regression_tests/test_fast_alloc.py index d810d256ee..b40ca03aca 100644 --- a/libensemble/tests/regression_tests/test_fast_alloc.py +++ b/libensemble/tests/regression_tests/test_fast_alloc.py @@ -14,6 +14,7 @@ # TESTSUITE_NPROCS: 2 4 import sys +import gc import numpy as np # Import libEnsemble items for this test @@ -26,7 +27,7 @@ nworkers, is_manager, libE_specs, _ = parse_args() -num_pts = 30*(nworkers - 1) +num_pts = 30*(nworkers) sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float), ('large', float, 1000000)], 'user': {}} @@ -49,6 +50,7 @@ sys.exit("Cannot run with tcp when repeated calls to libE -- aborting...") for time in np.append([0], np.logspace(-5, -1, 5)): + print("Starting for time: ", time, flush=True) if time == 0: alloc_specs = {'alloc_f': alloc_f2, 'out': []} else: @@ -70,3 +72,6 @@ if is_manager: assert flag == 0 assert len(H) == 2*num_pts + + del H + gc.collect() # If doing multiple libE calls, users might need to clean up their memory space. diff --git a/libensemble/tests/regression_tests/test_mpi_runners.py b/libensemble/tests/regression_tests/test_mpi_runners.py index 69094a4ded..9b6ec52714 100644 --- a/libensemble/tests/regression_tests/test_mpi_runners.py +++ b/libensemble/tests/regression_tests/test_mpi_runners.py @@ -9,82 +9,22 @@ # The number of concurrent evaluations of the objective function will be 4-1=3. # """ -import os import numpy as np -from libensemble.message_numbers import WORKER_DONE from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tools import parse_args, add_unique_random_streams from libensemble.executors.mpi_executor import MPIExecutor -from libensemble import libE_logger +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger -# libE_logger.set_level('DEBUG') # For testing the test -libE_logger.set_level('INFO') +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 2 4 -nodes_per_worker = 2 - - -def exp_nodelist_for_worker(exp_list, workerID): - """Modify expected node-lists based on workerID""" - comps = exp_list.split() - new_line = [] - for comp in comps: - if comp.startswith('node-'): - new_node_list = [] - node_list = comp.split(',') - for node in node_list: - node_name, node_num = node.split('-') - new_num = int(node_num) + nodes_per_worker*(workerID - 1) - new_node = '-'.join([node_name, str(new_num)]) - new_node_list.append(new_node) - new_list = ','.join(new_node_list) - new_line.append(new_list) - else: - new_line.append(comp) - return ' '.join(new_line) - - -def runline_check(H, persis_info, sim_specs, libE_info): - """Check run-lines produced by executor provided by a list""" - calc_status = 0 - x = H['x'][0][0] - exctr = MPIExecutor.executor - test_list = sim_specs['user']['tests'] - exp_list = sim_specs['user']['expect'] - - for i, test in enumerate(test_list): - task = exctr.submit(calc_type='sim', - num_procs=test.get('nprocs', None), - num_nodes=test.get('nnodes', None), - ranks_per_node=test.get('ppn', None), - extra_args=test.get('e_args', None), - app_args='--testid ' + test.get('testid', None), - stdout='out.txt', - stderr='err.txt', - hyperthreads=test.get('ht', None), - dry_run=True) - - outline = task.runline - new_exp_list = exp_nodelist_for_worker(exp_list[i], libE_info['workerID']) - - if outline != new_exp_list: - print('outline is: {}'.format(outline), flush=True) - print('exp is: {}'.format(new_exp_list), flush=True) - - assert(outline == new_exp_list) - - calc_status = WORKER_DONE - output = np.zeros(1, dtype=sim_specs['out']) - output['f'][0] = np.linalg.norm(x) - return output, persis_info, calc_status - -# -------------------------------------------------------------------- - - nworkers, is_manager, libE_specs, _ = parse_args() rounds = 1 sim_app = '/path/to/fakeapp.x' @@ -92,18 +32,17 @@ def runline_check(H, persis_info, sim_specs, libE_info): # To allow visual checking - log file not used in test log_file = 'ensemble_mpi_runners_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' -libE_logger.set_filename(log_file) +logger.set_filename(log_file) + +nodes_per_worker = 2 # For varying size test - relate node count to nworkers -node_file = 'nodelist_mpi_runnerscomms_' + str(comms) + '_wrks_' + str(nworkers) +node_file = 'nodelist_mpi_runners_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = nworkers*nodes_per_worker + if is_manager: - if os.path.exists(node_file): - os.remove(node_file) - with open(node_file, 'w') as f: - for i in range(1, nworkers*nodes_per_worker+1): - f.write('node-' + str(i) + '\n') - f.flush() - os.fsync(f) + create_node_file(num_nodes=nnodes, name=node_file) + if comms == 'mpi': libE_specs['mpi_comm'].Barrier() @@ -112,7 +51,7 @@ def runline_check(H, persis_info, sim_specs, libE_info): # TODO: May move specs, inputs and expected outputs to a data_set module. -sim_specs = {'sim_f': runline_check, +sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)], } @@ -254,11 +193,11 @@ def run_tests(mpi_runner, runner_name, test_list_exargs, exp_list): 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) 'node_file': node_file} # Name of file containing a node-list - exctr = MPIExecutor(central_mode=True, auto_resources=True, custom_info=customizer) + exctr = MPIExecutor(central_mode=True, auto_resources=True, allow_oversubscribe=False, custom_info=customizer) exctr.register_calc(full_path=sim_app, calc_type='sim') test_list = test_list_base + test_list_exargs - sim_specs['user'] = {'tests': test_list, 'expect': exp_list} + sim_specs['user'] = {'tests': test_list, 'expect': exp_list, 'nodes_per_worker': nodes_per_worker, 'persis_gens': 0} # Perform the run H, pinfo, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) diff --git a/libensemble/tests/regression_tests/test_mpi_runners_subnode.py b/libensemble/tests/regression_tests/test_mpi_runners_subnode.py new file mode 100644 index 0000000000..9f6c85a000 --- /dev/null +++ b/libensemble/tests/regression_tests/test_mpi_runners_subnode.py @@ -0,0 +1,111 @@ +# """ +# Runs libEnsemble testing the MPI Runners command creation with 2 workers per node. +# +# This test must be run on an even number of workers >= 2 (e.g. odd no. of procs when using mpi4py). +# +# Execute via one of the following commands (e.g. 4 workers): +# mpiexec -np 5 python3 test_mpi_runners_subnode.py +# python3 test_mpi_runners_subnode.py --nworkers 4 --comms local +# python3 test_mpi_runners_subnode.py --nworkers 4 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +import sys +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check as sim_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 3 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] + +# To allow visual checking - log file not used in test +log_file = 'ensemble_mpi_runners_subnode_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +nodes_per_worker = 0.5 + +# For varying size test - relate node count to nworkers +nsim_workers = nworkers + +if not (nsim_workers*nodes_per_worker).is_integer(): + sys.exit("Sim workers ({}) must divide evenly into nodes".format(nsim_workers)) + +comms = libE_specs['comms'] +node_file = 'nodelist_mpi_runners_subnode_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = int(nsim_workers*nodes_per_worker) + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(central_mode=True, auto_resources=True, + allow_oversubscribe=False, custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +# Each worker has 2 nodes. Basic test list for portable options +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + {'testid': 'base2', 'nprocs': 5}, + {'testid': 'base3', 'nnodes': 1}, + {'testid': 'base4', 'ppn': 6}, + ] + +exp_srun = \ + ['srun -w node-1 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1', + 'srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base2', + 'srun -w node-1 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base3', + 'srun -w node-1 --ntasks 6 --nodes 1 --ntasks-per-node 6 /path/to/fakeapp.x --testid base4', + ] + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list, + 'nodes_per_worker': nodes_per_worker} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + +# All asserts are in sim func diff --git a/libensemble/tests/regression_tests/test_mpi_runners_subnode_uneven.py b/libensemble/tests/regression_tests/test_mpi_runners_subnode_uneven.py new file mode 100644 index 0000000000..8ffc338db2 --- /dev/null +++ b/libensemble/tests/regression_tests/test_mpi_runners_subnode_uneven.py @@ -0,0 +1,126 @@ +# """ +# Runs libEnsemble testing the MPI Runners command creation with uneven workers per node. +# +# This test must be run on an odd number of workers >= 3 and <= 31 (e.g. even no. of procs when using mpi4py). +# +# Execute via one of the following commands (e.g. 5 workers): +# mpiexec -np 6 python3 test_mpi_runners_subnode_uneven.py +# python3 test_mpi_runners_subnode_uneven.py --nworkers 5 --comms local +# python3 test_mpi_runners_subnode_uneven.py --nworkers 5 --comms tcp +# """ + +import sys +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check_by_worker as sim_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 4 6 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] + +# To allow visual checking - log file not used in test +log_file = 'ensemble_mpi_runners_subnode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +# For varying size test - relate node count to nworkers +nsim_workers = nworkers + +if (nsim_workers % 2 == 0): + sys.exit("This test must be run with an odd of workers >= 3 and <= 31. There are {} workers." + .format(nsim_workers)) + +comms = libE_specs['comms'] +node_file = 'nodelist_mpi_runners_subnode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = 2 + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(central_mode=True, auto_resources=True, + allow_oversubscribe=False, custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + ] + +# Example: On 5 workers, runlines should be ... +# [w1]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w2]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w3]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w4]: srun -w node-2 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1 +# [w5]: srun -w node-2 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1 + +srun_p1 = 'srun -w ' +srun_p2 = ' --ntasks ' +srun_p3 = ' --nodes 1 --ntasks-per-node ' +srun_p4 = ' /path/to/fakeapp.x --testid base1' + +exp_tasks = [] +exp_srun = [] + +# Hard coding an example for 2 nodes to avoid replicating general logic in libEnsemble. +low_wpn = nsim_workers//nnodes +high_wpn = nsim_workers//nnodes + 1 + +for i in range(nsim_workers): + if i < (nsim_workers//nnodes + 1): + nodename = 'node-1' + ntasks = 16//high_wpn + else: + nodename = 'node-2' + ntasks = 16//low_wpn + exp_tasks.append(ntasks) + exp_srun.append(srun_p1 + str(nodename) + srun_p2 + str(ntasks) + srun_p3 + str(ntasks) + srun_p4) + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + + +# All asserts are in sim func diff --git a/libensemble/tests/regression_tests/test_mpi_runners_supernode_uneven.py b/libensemble/tests/regression_tests/test_mpi_runners_supernode_uneven.py new file mode 100644 index 0000000000..fb6b21a5c2 --- /dev/null +++ b/libensemble/tests/regression_tests/test_mpi_runners_supernode_uneven.py @@ -0,0 +1,127 @@ +# """ +# Runs libEnsemble testing the MPI Runners command creation with multiple and uneven nodes per worker. +# +# This test must be run on a number of workers >= 2. +# +# Execute via one of the following commands (e.g. 5 workers): +# mpiexec -np 6 python3 test_mpi_runners_supernode_uneven.py +# python3 test_mpi_runners_supernode_uneven.py --nworkers 5 --comms local +# """ + +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check_by_worker as sim_f +from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 3 4 5 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] + +# To allow visual checking - log file not used in test +log_file = 'ensemble_mpi_runners_supernode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +nodes_per_worker = 2.5 + +# For varying size test - relate node count to nworkers +nsim_workers = nworkers +comms = libE_specs['comms'] +node_file = 'nodelist_mpi_runners_supernode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = int(nsim_workers*nodes_per_worker) + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(central_mode=True, auto_resources=True, + allow_oversubscribe=False, custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +# Each worker has either 3 or 2 nodes. Basic test list for portable options +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + ] + +# Example: On 2 workers, runlines should be ... +# (one workers has 3 nodes, the other 2 - does not split 2.5 nodes each). +# [w1]: srun -w node-1,node-2,node-3 --ntasks 48 --nodes 3 --ntasks-per-node 16 /path/to/fakeapp.x --testid base1 +# [w2]: srun -w node-4,node-5 --ntasks 32 --nodes 2 --ntasks-per-node 16 /path/to/fakeapp.x --testid base1 + +srun_p1 = 'srun -w ' +srun_p2 = ' --ntasks ' +srun_p3 = ' --nodes ' +srun_p4 = ' --ntasks-per-node 16 /path/to/fakeapp.x --testid base1' + +exp_tasks = [] +exp_srun = [] + +# Hard coding an example for 2 nodes to avoid replicating general logic in libEnsemble. +low_npw = nnodes//nsim_workers +high_npw = nnodes//nsim_workers + 1 + +nodelist = [] +for i in range(1, nnodes + 1): + nodelist.append('node-' + str(i)) + +inode = 0 +for i in range(nsim_workers): + if i < (nsim_workers//2): + npw = high_npw + else: + npw = low_npw + nodename = ','.join(nodelist[inode:inode+npw]) + inode += npw + ntasks = 16*npw + loc_nodes = npw + exp_tasks.append(ntasks) + exp_srun.append(srun_p1 + str(nodename) + srun_p2 + str(ntasks) + srun_p3 + str(loc_nodes) + srun_p4) + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + + +# All asserts are in sim func diff --git a/libensemble/tests/regression_tests/test_mpi_runners_zrw_subnode_uneven.py b/libensemble/tests/regression_tests/test_mpi_runners_zrw_subnode_uneven.py new file mode 100644 index 0000000000..107ec14e0e --- /dev/null +++ b/libensemble/tests/regression_tests/test_mpi_runners_zrw_subnode_uneven.py @@ -0,0 +1,133 @@ +# """ +# Runs libEnsemble testing the MPI Runners command creation with uneven workers per node. +# +# This test must be run on an even number of workers >= 4 and <= 32 (e.g. odd no. of procs when using mpi4py). +# +# Execute via one of the following commands (e.g. 6 workers - one is zero resource): +# mpiexec -np 7 python3 test_mpi_runners_zrw_subnode_uneven.py +# python3 test_mpi_runners_zrw_subnode_uneven.py --nworkers 6 --comms local +# python3 test_mpi_runners_zrw_subnode_uneven.py --nworkers 6 --comms tcp +# """ + +import sys +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check_by_worker as sim_f +from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 5 7 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] +libE_specs['zero_resource_workers'] = [1] + +# To allow visual checking - log file not used in test +log_file = 'ensemble_mpi_runners_zrw_subnode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +# For varying size test - relate node count to nworkers +in_place = libE_specs['zero_resource_workers'] +n_gens = len(in_place) +nsim_workers = nworkers-n_gens + +if (nsim_workers % 2 == 0): + sys.exit("This test must be run with an odd number of sim workers >= 3 and <= 31. There are {} sim workers." + .format(nsim_workers)) + +comms = libE_specs['comms'] +node_file = 'nodelist_mpi_runners_zrw_subnode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = 2 + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(zero_resource_workers=in_place, central_mode=True, + auto_resources=True, allow_oversubscribe=False, + custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + ] + +# Example: On 5 workers, runlines should be ... +# [w1]: Gen only +# [w2]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w3]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w4]: srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base1 +# [w5]: srun -w node-2 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1 +# [w6]: srun -w node-2 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1 + +srun_p1 = 'srun -w ' +srun_p2 = ' --ntasks ' +srun_p3 = ' --nodes 1 --ntasks-per-node ' +srun_p4 = ' /path/to/fakeapp.x --testid base1' + +exp_tasks = [] +exp_srun = [] + +# Hard coding an example for 2 nodes to avoid replicating general logic in libEnsemble. +low_wpn = nsim_workers//nnodes +high_wpn = nsim_workers//nnodes + 1 + +for i in range(nsim_workers): + if i < (nsim_workers//nnodes + 1): + nodename = 'node-1' + ntasks = 16//high_wpn + else: + nodename = 'node-2' + ntasks = 16//low_wpn + exp_tasks.append(ntasks) + exp_srun.append(srun_p1 + str(nodename) + srun_p2 + str(ntasks) + srun_p3 + str(ntasks) + srun_p4) + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list, 'persis_gens': n_gens} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +# All asserts are in sim func diff --git a/libensemble/tests/regression_tests/test_mpi_runners_zrw_supernode_uneven.py b/libensemble/tests/regression_tests/test_mpi_runners_zrw_supernode_uneven.py new file mode 100644 index 0000000000..d7cc37743e --- /dev/null +++ b/libensemble/tests/regression_tests/test_mpi_runners_zrw_supernode_uneven.py @@ -0,0 +1,135 @@ +# """ +# Runs libEnsemble testing the MPI Runners command creation with multiple and uneven nodes per worker. +# +# This test must be run on a number of workers >= 3. +# +# Execute via one of the following commands (e.g. 6 workers - one is zero resource): +# mpiexec -np 7 python3 test_mpi_runners_zrw_supernode_uneven.py +# python3 test_mpi_runners_zrw_supernode_uneven.py --nworkers 6 --comms local +# """ + +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check_by_worker as sim_f +from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 4 5 6 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] +libE_specs['zero_resource_workers'] = [1] + +# To allow visual checking - log file not used in test +log_file = 'ensemble_mpi_runners_zrw_supernode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +nodes_per_worker = 2.5 + +# For varying size test - relate node count to nworkers +in_place = libE_specs['zero_resource_workers'] +n_gens = len(in_place) +nsim_workers = nworkers-n_gens +comms = libE_specs['comms'] +node_file = 'nodelist_mpi_runners_zrw_supernode_uneven_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = int(nsim_workers*nodes_per_worker) + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(zero_resource_workers=in_place, central_mode=True, + auto_resources=True, allow_oversubscribe=False, + custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +# Each worker has either 3 or 2 nodes. Basic test list for portable options +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + ] + +# Example: On 3 workers, runlines should be ... +# (one workers has 3 nodes, the other 2 - does not split 2.5 nodes each). +# [w1]: Gen only +# [w2]: srun -w node-1,node-2,node-3 --ntasks 48 --nodes 3 --ntasks-per-node 16 /path/to/fakeapp.x --testid base1 +# [w3]: srun -w node-4,node-5 --ntasks 32 --nodes 2 --ntasks-per-node 16 /path/to/fakeapp.x --testid base1 + +srun_p1 = 'srun -w ' +srun_p2 = ' --ntasks ' +srun_p3 = ' --nodes ' +srun_p4 = ' --ntasks-per-node 16 /path/to/fakeapp.x --testid base1' + +exp_tasks = [] +exp_srun = [] + +# Hard coding an example for 2 nodes to avoid replicating general logic in libEnsemble. +low_npw = nnodes//nsim_workers +high_npw = nnodes//nsim_workers + 1 + +nodelist = [] +for i in range(1, nnodes + 1): + nodelist.append('node-' + str(i)) + +inode = 0 +for i in range(nsim_workers): + if i < (nsim_workers//2): + npw = high_npw + else: + npw = low_npw + nodename = ','.join(nodelist[inode:inode+npw]) + inode += npw + ntasks = 16*npw + loc_nodes = npw + exp_tasks.append(ntasks) + exp_srun.append(srun_p1 + str(nodename) + srun_p2 + str(ntasks) + srun_p3 + str(loc_nodes) + srun_p4) + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list, 'persis_gens': n_gens} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + + +# All asserts are in sim func diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py index 38c56c1f67..d5e6ab2ae7 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py @@ -66,7 +66,11 @@ persis_info = add_unique_random_streams({}, nworkers + 1) -exit_criteria = {'sim_max': 1000} +# Tell libEnsemble when to stop (stop_val key must be in H) +exit_criteria = {'sim_max': 1000, + 'elapsed_wallclock_time': 100, + 'stop_val': ('f', 3000)} +# end_exit_criteria_rst_tag # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, @@ -75,16 +79,15 @@ if is_manager: assert persis_info[1].get('run_order'), "Run_order should have been given back" assert flag == 0 - assert len(H) >= budget + assert np.min(H['f'][H['returned']]) <= 3000, "Didn't find a value below 3000" save_libE_output(H, persis_info, __file__, nworkers) # # Calculating the Jacobian at local_minima (though this information was not used by DFO-LS) - # from libensemble.sim_funcs.chwirut1 import EvaluateFunction, EvaluateJacobian - # for i in np.where(H['local_min'])[0]: - - # F = EvaluateFunction(H['x'][i]) - # J = EvaluateJacobian(H['x'][i]) + from libensemble.sim_funcs.chwirut1 import EvaluateFunction, EvaluateJacobian + for i in np.where(H['local_min'])[0]: + F = EvaluateFunction(H['x'][i]) + J = EvaluateJacobian(H['x'][i]) # u = gen_specs['user']['ub']-H['x'][i] # l = H['x'][i]-gen_specs['user']['lb'] # if np.any(u <= 1e-7) or np.any(l <= 1e-7): diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py index d9e5e5c27d..767a2f4ccf 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py @@ -80,7 +80,7 @@ def assertion(passed): alloc_specs, libE_specs) except Exception as e: if is_manager: - if e.args[1] == 'NLopt roundoff-limited': + if e.args[1].endswith('NLopt roundoff-limited'): assertion(True) else: assertion(False) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py index 52d1c32eed..eaef442b3b 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py @@ -75,8 +75,9 @@ exit_criteria = {'sim_max': 500} sample_points = np.zeros((0, n)) +rand_stream = np.random.RandomState(0) for i in range(ceil(exit_criteria['sim_max']/gen_specs['user']['lhs_divisions'])): - sample_points = np.append(sample_points, lhs_sample(n, gen_specs['user']['lhs_divisions']), axis=0) + sample_points = np.append(sample_points, lhs_sample(n, gen_specs['user']['lhs_divisions'], rand_stream), axis=0) gen_specs['user']['sample_points'] = sample_points*(ub-lb) + lb diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py index 676d0e6979..053e865d5a 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py @@ -2,9 +2,9 @@ # Runs libEnsemble testing timeout and wait for persistent worker. # # Execute via one of the following commands (e.g. 3 workers): -# mpiexec -np 4 python3 test_6-test_persistent_aposmm_timeout.py.py -# python3 test_6-test_persistent_aposmm_timeout.py.py --nworkers 3 --comms local -# python3 test_6-test_persistent_aposmm_timeout.py.py --nworkers 3 --comms tcp +# mpiexec -np 4 python3 test_6-test_persistent_aposmm_timeout.py +# python3 test_6-test_persistent_aposmm_timeout.py --nworkers 3 --comms local +# python3 test_6-test_persistent_aposmm_timeout.py --nworkers 3 --comms tcp # # The number of concurrent evaluations of the objective function will be 4-1=3. # """ diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py new file mode 100644 index 0000000000..6cf731df0b --- /dev/null +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -0,0 +1,109 @@ +# """ +# Runs libEnsemble with Surmise calibration test. +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_persistent_surmise_calib.py +# python3 test_persistent_surmise_calib.py --nworkers 3 --comms local +# python3 test_persistent_surmise_calib.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# +# This test uses the Surmise package to perform a Borehole Calibration with +# selective simulation cancellation. Initial observations are modeled using +# a theta at the center of a unit hypercube. The initial function values for +# these are run first. As the model is updated, the generator selects previously +# issued evaluations to cancel. +# +# See more information, see tutorial: +# "Borehole Calibration with Selective Simulation Cancellation" +# in the libEnsemble documentation. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_NPROCS: 3 4 + +# Requires: +# Install Surmise package + +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.sim_funcs.surmise_test_function import borehole as sim_f +from libensemble.tools import parse_args, save_libE_output, add_unique_random_streams + +# from libensemble import logger +# logger.set_level('DEBUG') # To get debug logging in ensemble.log + +if __name__ == '__main__': + + nworkers, is_manager, libE_specs, _ = parse_args() + + n_init_thetas = 15 # Initial batch of thetas + n_x = 25 # No. of x values + nparams = 4 # No. of theta params + ndims = 3 # No. of x co-ordinates. + max_add_thetas = 50 # Max no. of thetas added for evaluation + step_add_theta = 10 # No. of thetas to generate per step, before emulator is rebuilt + n_explore_theta = 200 # No. of thetas to explore while selecting the next theta + obsvar = 10 ** (-1) # Constant for generating noise in obs + + # Batch mode until after init_sample_size (add one theta to batch for observations) + init_sample_size = (n_init_thetas + 1) * n_x + + # Stop after max_emul_runs runs of the emulator + max_evals = init_sample_size + max_add_thetas*n_x + + sim_specs = {'sim_f': sim_f, + 'in': ['x', 'thetas'], + 'out': [('f', float)], + 'user': {'num_obs': n_x} + } + + gen_out = [('x', float, ndims), ('thetas', float, nparams), + ('priority', int), ('obs', float, n_x), ('obsvar', float, n_x)] + + gen_specs = {'gen_f': gen_f, + 'in': [o[0] for o in gen_out]+['f', 'returned'], + 'out': gen_out, + 'user': {'n_init_thetas': n_init_thetas, # Num thetas in initial batch + 'num_x_vals': n_x, # Num x points to create + 'step_add_theta': step_add_theta, # No. of thetas to generate per step + 'n_explore_theta': n_explore_theta, # No. of thetas to explore each step + 'obsvar': obsvar, # Variance for generating noise in obs + 'init_sample_size': init_sample_size, # Initial batch size inc. observations + 'priorloc': 1, # Prior location in the unit cube + 'priorscale': 0.5, # Standard deviation of prior + } + } + + alloc_specs = {'alloc_f': alloc_f, + 'out': [('given_back', bool)], + 'user': {'init_sample_size': init_sample_size, + 'async_return': True, # True = Return results to gen as they come in (after sample) + 'active_recv_gen': True # Persistent gen can handle irregular communications + } + } + + persis_info = add_unique_random_streams({}, nworkers + 1) + + # Currently just allow gen to exit if mse goes below threshold value + # exit_criteria = {'sim_max': max_evals, 'stop_val': ('mse', mse_exit)} + + exit_criteria = {'sim_max': max_evals} # Now just a set number of sims. + + # Perform the run + H, persis_info, flag = libE(sim_specs, gen_specs, + exit_criteria, persis_info, + alloc_specs=alloc_specs, + libE_specs=libE_specs) + + if is_manager: + print('Cancelled sims', H['sim_id'][H['cancel_requested']]) + sims_done = np.count_nonzero(H['returned']) + save_libE_output(H, persis_info, __file__, nworkers) + assert sims_done == max_evals, \ + 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py new file mode 100644 index 0000000000..c52debfb4b --- /dev/null +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -0,0 +1,125 @@ +# """ +# Tests killing of cancelled simulation that are in progress.. +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_persistent_surmise_killsims.py +# python3 test_persistent_surmise_killsims.py --nworkers 3 --comms local +# python3 test_persistent_surmise_killsims.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# +# This test is a smaller variant of test_persistent_surmise_calib.py, but which +# subprocesses a compiled version of the borehole simulation. A delay is +# added to simulations after the initial batch, so that the killing of running +# simulations can be tested. This will only affect simulations that have already +# been issued to a worker when the cancel request is registesred by the manger. +# +# See more information, see tutorial: +# "Borehole Calibration with Selective Simulation Cancellation" +# in the libEnsemble documentation. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_NPROCS: 3 4 + +# Requires: +# Install Surmise package + +import os +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.sim_funcs.borehole_kills import borehole as sim_f +from libensemble.tests.regression_tests.common import build_borehole # current location +from libensemble.executors.executor import Executor +from libensemble.tools import parse_args, save_libE_output, add_unique_random_streams + +# from libensemble import logger +# logger.set_level('DEBUG') # To get debug logging in ensemble.log + + +if __name__ == '__main__': + + nworkers, is_manager, libE_specs, _ = parse_args() + + n_init_thetas = 15 # Initial batch of thetas + n_x = 5 # No. of x values + nparams = 4 # No. of theta params + ndims = 3 # No. of x co-ordinates. + max_add_thetas = 20 # Max no. of thetas added for evaluation + step_add_theta = 10 # No. of thetas to generate per step, before emulator is rebuilt + n_explore_theta = 200 # No. of thetas to explore while selecting the next theta + obsvar = 10 ** (-1) # Constant for generating noise in obs + + # Batch mode until after init_sample_size (add one theta to batch for observations) + init_sample_size = (n_init_thetas + 1) * n_x + + # Stop after max_emul_runs runs of the emulator + max_evals = init_sample_size + max_add_thetas*n_x + + sim_app = os.path.join(os.getcwd(), "borehole.x") + if not os.path.isfile(sim_app): + build_borehole() + + exctr = Executor() # Run serial sub-process in place + exctr.register_calc(full_path=sim_app, app_name='borehole') + + # Subprocess variant creates input and output files for each sim + libE_specs['sim_dirs_make'] = True # To keep all - make sim dirs + # libE_specs['use_worker_dirs'] = True # To overwrite - make worker dirs only + + # Rename ensemble dir for non-inteference with other regression tests + libE_specs['ensemble_dir_path'] = 'ensemble_calib_kills' + + sim_specs = {'sim_f': sim_f, + 'in': ['x', 'thetas'], + 'out': [('f', float)], + 'user': {'num_obs': n_x, + 'init_sample_size': init_sample_size} + } + + gen_out = [('x', float, ndims), ('thetas', float, nparams), + ('priority', int), ('obs', float, n_x), ('obsvar', float, n_x)] + + gen_specs = {'gen_f': gen_f, + 'in': [o[0] for o in gen_out]+['f', 'returned'], + 'out': gen_out, + 'user': {'n_init_thetas': n_init_thetas, # Num thetas in initial batch + 'num_x_vals': n_x, # Num x points to create + 'step_add_theta': step_add_theta, # No. of thetas to generate per step + 'n_explore_theta': n_explore_theta, # No. of thetas to explore each step + 'obsvar': obsvar, # Variance for generating noise in obs + 'init_sample_size': init_sample_size, # Initial batch size inc. observations + 'priorloc': 1, # Prior location in the unit cube. + 'priorscale': 0.2, # Standard deviation of prior + } + } + + alloc_specs = {'alloc_f': alloc_f, + 'out': [('given_back', bool)], + 'user': {'init_sample_size': init_sample_size, + 'async_return': True, # True = Return results to gen as they come in (after sample) + 'active_recv_gen': True # Persistent gen can handle irregular communications + } + } + + persis_info = add_unique_random_streams({}, nworkers + 1) + exit_criteria = {'sim_max': max_evals} + + # Perform the run + H, persis_info, flag = libE(sim_specs, gen_specs, + exit_criteria, persis_info, + alloc_specs=alloc_specs, + libE_specs=libE_specs) + + if is_manager: + print('Cancelled sims', H['sim_id'][H['cancel_requested']]) + print('Killed sims', H['sim_id'][H['kill_sent']]) + sims_done = np.count_nonzero(H['returned']) + save_libE_output(H, persis_info, __file__, nworkers) + assert sims_done == max_evals, \ + 'Num of completed simulations should be {}. Is {}'.format(max_evals, sims_done) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py new file mode 100644 index 0000000000..dd9a4aff65 --- /dev/null +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_adv.py @@ -0,0 +1,65 @@ +# """ +# Tests the ability of libEnsemble to +# - give back all of the history to a persistent gen at shutdown + +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_persistent_uniform_sampling_adv.py +# python3 test_persistent_uniform_sampling_adv.py --nworkers 3 --comms local +# python3 test_persistent_uniform_sampling_adv.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local tcp +# TESTSUITE_NPROCS: 3 4 + +import sys +import numpy as np + +# Import libEnsemble items for this test +from libensemble.libE import libE +from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f +from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.tools import parse_args, save_libE_output, add_unique_random_streams + +nworkers, is_manager, libE_specs, _ = parse_args() + +libE_specs['use_persis_return'] = True + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float), ('grad', float, n)] + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 100, + 'replace_final_fields': True, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +persis_info = add_unique_random_streams({}, nworkers + 1) + +sim_max = 40 +exit_criteria = {'sim_max': 40} + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} + +libE_specs['final_fields'] = ['x', 'f', 'sim_id'] +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +if is_manager: + assert len(np.unique(H['gen_time'])) == 1, "Everything should have been generated in one batch" + assert np.all(H['x'][0:sim_max] == -1.23), "The persistent gen should have set these at shutdown" + + save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py index 4ed3a0231e..14bb74ef6a 100644 --- a/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/regression_tests/test_persistent_uniform_sampling_async.py @@ -40,12 +40,14 @@ 'in': [], 'out': [('x', float, (n,))], 'user': {'gen_batch_size': nworkers - 1, - 'async': True, 'lb': np.array([-3, -2]), 'ub': np.array([3, 2])} } -alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +alloc_specs = {'alloc_f': alloc_f, + 'out': [('given_back', bool)], + 'user': {'async_return': True} + } persis_info = add_unique_random_streams({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_sim_dirs_per_worker.py b/libensemble/tests/regression_tests/test_sim_dirs_per_worker.py index 0d42f1faa0..ef11048213 100644 --- a/libensemble/tests/regression_tests/test_sim_dirs_per_worker.py +++ b/libensemble/tests/regression_tests/test_sim_dirs_per_worker.py @@ -27,7 +27,7 @@ sim_input_dir = './sim_input_dir' dir_to_copy = sim_input_dir + '/copy_this' dir_to_symlink = sim_input_dir + '/symlink_this' -w_ensemble = '../ensemble_workdirs_w' + str(nworkers) + '_' + libE_specs.get('comms') +w_ensemble = './ensemble_workdirs_w' + str(nworkers) + '_' + libE_specs.get('comms') print('creating ensemble dir: ', w_ensemble, flush=True) for dir in [sim_input_dir, dir_to_copy, dir_to_symlink]: diff --git a/libensemble/tests/regression_tests/test_sim_dirs_with_exception.py b/libensemble/tests/regression_tests/test_sim_dirs_with_exception.py index d8516b6b34..5c9338c058 100644 --- a/libensemble/tests/regression_tests/test_sim_dirs_with_exception.py +++ b/libensemble/tests/regression_tests/test_sim_dirs_with_exception.py @@ -21,28 +21,17 @@ from libensemble.tests.regression_tests.support import write_sim_func as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.tools import parse_args, add_unique_random_streams -from libensemble.libE_manager import ManagerException +from libensemble.manager import LoggedException nworkers, is_manager, libE_specs, _ = parse_args() -sim_input_dir = './sim_input_dir' -dir_to_copy = sim_input_dir + '/copy_this' -dir_to_symlink = sim_input_dir + '/symlink_this' -e_ensemble = './ensemble_calcdirs_w' + str(nworkers) + '_' + libE_specs.get('comms') -print('attempting to use ensemble dir: ', e_ensemble, flush=True) -print('previous dir contains ', len(os.listdir(e_ensemble)), ' items.', flush=True) +e_ensemble = './ensemble_ex_w' + str(nworkers) + '_' + libE_specs.get('comms') -assert os.path.isdir(e_ensemble), \ - "Previous ensemble directory doesn't exist. Can't test exception." -assert len(os.listdir(e_ensemble)), \ - "Previous ensemble directory doesn't have any contents. Can't catch exception." +if not os.path.isdir(e_ensemble): + os.makedirs(os.path.join(e_ensemble, 'sim0_worker0'), exist_ok=True) libE_specs['sim_dirs_make'] = True libE_specs['ensemble_dir_path'] = e_ensemble -libE_specs['use_worker_dirs'] = False -libE_specs['sim_dir_copy_files'] = [dir_to_copy] -libE_specs['sim_dir_symlink_files'] = [dir_to_symlink] - libE_specs['abort_on_exception'] = False sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)]} @@ -63,7 +52,7 @@ try: H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) -except ManagerException as e: +except LoggedException as e: print("Caught deliberate exception: {}".format(e)) return_flag = 0 diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py new file mode 100644 index 0000000000..7fd65b80c7 --- /dev/null +++ b/libensemble/tests/regression_tests/test_uniform_sampling_cancel.py @@ -0,0 +1,141 @@ +# """ +# Runs libEnsemble on the 6-hump camel problem. Documented here: +# https://www.sfu.ca/~ssurjano/camel6.html +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_uniform_sampling_cancel.py +# python3 test_uniform_sampling_cancel.py --nworkers 3 --comms local +# python3 test_uniform_sampling_cancel.py --nworkers 3 --comms tcp +# +# The number of concurrent evaluations of the objective function will be 4-1=3. +# +# Tests sampling with cancellations. +# +# """ + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 2 4 + +import numpy as np +import gc + +# Import libEnsemble items for this test +from libensemble.libE import libE +from libensemble.sim_funcs.six_hump_camel import six_hump_camel +from libensemble.gen_funcs.sampling import uniform_random_sample_cancel +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima + +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first as gswf +from libensemble.alloc_funcs.fast_alloc import give_sim_work_first as fast_gswf +from libensemble.alloc_funcs.only_one_gen_alloc import ensure_one_active_gen +from libensemble.alloc_funcs.give_pregenerated_work import give_pregenerated_sim_work + + +def create_H0(persis_info, gen_specs, sim_max): + """Create an H0 for give_pregenerated_sim_work""" + + # Manually creating H0 + ub = gen_specs['user']['ub'] + lb = gen_specs['user']['lb'] + n = len(lb) + b = sim_max + + H0 = np.zeros(b, dtype=[('x', float, 2), ('sim_id', int), ('given', bool), ('cancel_requested', bool)]) + H0['x'] = persis_info[0]['rand_stream'].uniform(lb, ub, (b, n)) + H0['sim_id'] = range(b) + H0['given'] = False + for i in range(b): + if i % 10 == 0: + H0[i]['cancel_requested'] = True + + # Using uniform_random_sample_cancel call - need to adjust some gen_specs though + # gen_specs['out'].append(('sim_id', int)) + # gen_specs['out'].append(('given', bool)) + # gen_specs['user']['gen_batch_size'] = sim_max + # H0, persis_info[0] = uniform_random_sample_cancel({}, persis_info[0], gen_specs, {}) + # H0['sim_id'] = range(gen_specs['user']['gen_batch_size']) + # H0['given'] = False + return H0 + + +nworkers, is_manager, libE_specs, _ = parse_args() + +sim_specs = {'sim_f': six_hump_camel, # Function whose output is being minimized + 'in': ['x'], # Keys to be given to sim_f + 'out': [('f', float)], # Name of the outputs from sim_f + } +# end_sim_specs_rst_tag + + +# Note that it is unusual to specifiy cancel_requested as gen_specs['out']. It is here +# so that cancellations are combined with regular generator outputs for testing purposes. +# For a typical use case see test_persistent_surmise_calib.py. +gen_specs = {'gen_f': uniform_random_sample_cancel, # Function generating sim_f input + 'out': [('x', float, (2,)), ('cancel_requested', bool)], + 'user': {'gen_batch_size': 50, # Used by this specific gen_f + 'lb': np.array([-3, -2]), # Used by this specific gen_f + 'ub': np.array([3, 2]) # Used by this specific gen_f + } + } +# end_gen_specs_rst_tag + +persis_info = add_unique_random_streams({}, nworkers + 1) +sim_max = 500 +exit_criteria = {'sim_max': sim_max, 'elapsed_wallclock_time': 300} + +aspec1 = {'alloc_f': gswf, + 'out': [], + 'user': {'batch_mode': True, 'num_active_gens': 1}} + +aspec2 = {'alloc_f': gswf, + 'out': [], + 'user': {'batch_mode': True, 'num_active_gens': 2}} + +aspec3 = {'alloc_f': fast_gswf, + 'out': [], + 'user': {}} + +aspec4 = {'alloc_f': ensure_one_active_gen, + 'out': [], + 'user': {}} + +aspec5 = {'alloc_f': give_pregenerated_sim_work, + 'out': [], + 'user': {}} + +allocs = {1: aspec1, 2: aspec2, 3: aspec3, 4: aspec4, 5: aspec5} + +if is_manager: + print('Testing cancellations with non-persistent gen functions') + +for testnum in range(1, 6): + alloc_specs = allocs[testnum] + if is_manager: + print('\nRunning with alloc specs', alloc_specs, flush=True) + + if alloc_specs['alloc_f'] == give_pregenerated_sim_work: + H0 = create_H0(persis_info, gen_specs, sim_max) + else: + H0 = None + + # Reset for those that use them + persis_info['next_to_give'] = 0 + persis_info['total_gen_calls'] = 0 # 1 + + # Perform the run - do not overwrite persis_info + H, persis_out, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs=libE_specs, H0=H0) + + if is_manager: + assert flag == 0 + assert np.all(H['cancel_requested'][::10]), 'Some values should be cancelled but are not' + assert np.all(~H['given'][::10]), 'Some values are given that should not have been' + tol = 0.1 + for m in minima: + assert np.min(np.sum((H['x'] - m)**2, 1)) < tol + + print("libEnsemble found the 6 minima within a tolerance " + str(tol)) + del H + gc.collect() # Clean up memory space. diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py index 0cd1646e76..aa7350a2cb 100644 --- a/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/regression_tests/test_uniform_sampling_one_residual_at_a_time.py @@ -64,7 +64,7 @@ } alloc_specs = {'alloc_f': give_sim_work_first, # Allocation function - 'out': [('allocated', bool)], # Output fields (included in History) + 'out': [], # Output fields (included in History) 'user': {'stop_on_NaNs': True, # Should alloc preempt evals 'batch_mode': True, # Wait until all sim evals are done 'num_active_gens': 1, # Only allow one active generator diff --git a/libensemble/tests/regression_tests/test_uniform_sampling_with_different_resources.py b/libensemble/tests/regression_tests/test_uniform_sampling_with_different_resources.py index 96a46b2966..b970ebe7a5 100644 --- a/libensemble/tests/regression_tests/test_uniform_sampling_with_different_resources.py +++ b/libensemble/tests/regression_tests/test_uniform_sampling_with_different_resources.py @@ -79,7 +79,7 @@ } alloc_specs = {'alloc_f': give_sim_work_first, - 'out': [('allocated', bool)], + 'out': [], 'user': {'batch_mode': False, 'num_active_gens': 1}} diff --git a/libensemble/tests/regression_tests/test_worker_exceptions.py b/libensemble/tests/regression_tests/test_worker_exceptions.py index 1bbf47b3a6..c00b4f6a7f 100644 --- a/libensemble/tests/regression_tests/test_worker_exceptions.py +++ b/libensemble/tests/regression_tests/test_worker_exceptions.py @@ -18,7 +18,7 @@ from libensemble.libE import libE from libensemble.tests.regression_tests.support import nan_func as sim_f from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -from libensemble.libE_manager import ManagerException +from libensemble.manager import LoggedException from libensemble.tools import parse_args, add_unique_random_streams nworkers, is_manager, libE_specs, _ = parse_args() @@ -46,7 +46,7 @@ try: H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) -except ManagerException as e: +except LoggedException as e: print("Caught deliberate exception: {}".format(e)) return_flag = 0 diff --git a/libensemble/tests/regression_tests/test_zero_resource_workers.py b/libensemble/tests/regression_tests/test_zero_resource_workers.py index 14b264cd79..5cfafe358a 100644 --- a/libensemble/tests/regression_tests/test_zero_resource_workers.py +++ b/libensemble/tests/regression_tests/test_zero_resource_workers.py @@ -9,84 +9,25 @@ # The number of concurrent evaluations of the objective function will be 4-1=3. # """ -import os import sys import numpy as np -from libensemble.message_numbers import WORKER_DONE from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check as sim_f from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.tools import parse_args, add_unique_random_streams from libensemble.executors.mpi_executor import MPIExecutor -from libensemble import libE_logger +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger -# libE_logger.set_level('DEBUG') # For testing the test -libE_logger.set_level('INFO') +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 3 4 -nodes_per_worker = 2 - - -def exp_nodelist_for_worker(exp_list, workerID): - """Modify expected node-lists based on workerID""" - comps = exp_list.split() - new_line = [] - for comp in comps: - if comp.startswith('node-'): - new_node_list = [] - node_list = comp.split(',') - for node in node_list: - node_name, node_num = node.split('-') - new_num = int(node_num) + nodes_per_worker*(workerID - 2) # For 1 persistent gen - new_node = '-'.join([node_name, str(new_num)]) - new_node_list.append(new_node) - new_list = ','.join(new_node_list) - new_line.append(new_list) - else: - new_line.append(comp) - return ' '.join(new_line) - - -def runline_check(H, persis_info, sim_specs, libE_info): - """Check run-lines produced by executor provided by a list""" - calc_status = 0 - x = H['x'][0][0] - exctr = MPIExecutor.executor - test_list = sim_specs['user']['tests'] - exp_list = sim_specs['user']['expect'] - - for i, test in enumerate(test_list): - task = exctr.submit(calc_type='sim', - num_procs=test.get('nprocs', None), - num_nodes=test.get('nnodes', None), - ranks_per_node=test.get('ppn', None), - extra_args=test.get('e_args', None), - app_args='--testid ' + test.get('testid', None), - stdout='out.txt', - stderr='err.txt', - hyperthreads=test.get('ht', None), - dry_run=True) - - outline = task.runline - new_exp_list = exp_nodelist_for_worker(exp_list[i], libE_info['workerID']) - - if outline != new_exp_list: - print('outline is: {}\nexp is: {}'.format(outline, new_exp_list), flush=True) - - assert(outline == new_exp_list) - - calc_status = WORKER_DONE - output = np.zeros(1, dtype=sim_specs['out']) - output['f'][0] = np.linalg.norm(x) - return output, persis_info, calc_status - -# -------------------------------------------------------------------- - - nworkers, is_manager, libE_specs, _ = parse_args() rounds = 1 sim_app = '/path/to/fakeapp.x' @@ -96,22 +37,23 @@ def runline_check(H, persis_info, sim_specs, libE_info): # To allow visual checking - log file not used in test log_file = 'ensemble_zrw_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' -libE_logger.set_filename(log_file) +logger.set_filename(log_file) + +nodes_per_worker = 2 # For varying size test - relate node count to nworkers in_place = libE_specs['zero_resource_workers'] -nsim_workers = nworkers-len(in_place) +n_gens = len(in_place) +nsim_workers = nworkers-n_gens + comms = libE_specs['comms'] nodes_per_worker = 2 -node_file = 'nodelist_zero_resource_workers_' + str(comms) + '_wrks_' + str(nworkers) +node_file = 'nodelist_zero_resource_workers_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = nsim_workers*nodes_per_worker + if is_manager: - if os.path.exists(node_file): - os.remove(node_file) - with open(node_file, 'w') as f: - for i in range(1, (nsim_workers)*nodes_per_worker + 1): - f.write('node-' + str(i) + '\n') - f.flush() - os.fsync(f) + create_node_file(num_nodes=nnodes, name=node_file) + if comms == 'mpi': libE_specs['mpi_comm'].Barrier() @@ -123,7 +65,9 @@ def runline_check(H, persis_info, sim_specs, libE_info): 'node_file': node_file} # Name of file containing a node-list # Create executor and register sim to it. -exctr = MPIExecutor(zero_resource_workers=in_place, central_mode=True, auto_resources=True, custom_info=customizer) +exctr = MPIExecutor(zero_resource_workers=in_place, central_mode=True, + auto_resources=True, allow_oversubscribe=False, + custom_info=customizer) exctr.register_calc(full_path=sim_app, calc_type='sim') @@ -131,7 +75,7 @@ def runline_check(H, persis_info, sim_specs, libE_info): sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") n = 2 -sim_specs = {'sim_f': runline_check, +sim_specs = {'sim_f': sim_f, 'in': ['x'], 'out': [('f', float)], } @@ -160,7 +104,8 @@ def runline_check(H, persis_info, sim_specs, libE_info): test_list = test_list_base exp_list = exp_mpich -sim_specs['user'] = {'tests': test_list, 'expect': exp_list} +sim_specs['user'] = {'tests': test_list, 'expect': exp_list, + 'nodes_per_worker': nodes_per_worker, 'persis_gens': n_gens} # Perform the run diff --git a/libensemble/tests/regression_tests/test_zero_resource_workers_subnode.py b/libensemble/tests/regression_tests/test_zero_resource_workers_subnode.py new file mode 100644 index 0000000000..851a8b32b2 --- /dev/null +++ b/libensemble/tests/regression_tests/test_zero_resource_workers_subnode.py @@ -0,0 +1,121 @@ +# """ +# Runs libEnsemble testing the zero_resource_workers argument with 2 workers per node. +# +# This test must be run on an odd number of workers >= 3 (e.g. even no. of procs when using mpi4py). +# +# Execute via one of the following commands (e.g. 3 workers): +# mpiexec -np 4 python3 test_zero_resource_workers_subnode.py +# python3 test_zero_resource_workers_subnode.py --nworkers 3 --comms local +# python3 test_zero_resource_workers_subnode.py --nworkers 3 --comms tcp +# """ + +import sys +import numpy as np + +from libensemble.libE import libE +from libensemble.sim_funcs.run_line_check import runline_check as sim_f +from libensemble.gen_funcs.persistent_uniform_sampling import persistent_uniform as gen_f +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f +from libensemble.tools import parse_args, add_unique_random_streams +from libensemble.executors.mpi_executor import MPIExecutor +from libensemble.tests.regression_tests.common import create_node_file +from libensemble import logger + +# logger.set_level('DEBUG') # For testing the test +logger.set_level('INFO') + +# Do not change these lines - they are parsed by run-tests.sh +# TESTSUITE_COMMS: mpi local +# TESTSUITE_NPROCS: 4 + +nworkers, is_manager, libE_specs, _ = parse_args() +rounds = 1 +sim_app = '/path/to/fakeapp.x' +comms = libE_specs['comms'] +libE_specs['zero_resource_workers'] = [1] + + +# To allow visual checking - log file not used in test +log_file = 'ensemble_zrw_subnode_comms_' + str(comms) + '_wrks_' + str(nworkers) + '.log' +logger.set_filename(log_file) + +nodes_per_worker = 0.5 + +# For varying size test - relate node count to nworkers +in_place = libE_specs['zero_resource_workers'] +n_gens = len(in_place) +nsim_workers = nworkers-n_gens + +if not (nsim_workers*nodes_per_worker).is_integer(): + sys.exit("Sim workers ({}) must divide evenly into nodes".format(nsim_workers)) + +comms = libE_specs['comms'] +node_file = 'nodelist_zero_resource_workers_subnode_comms_' + str(comms) + '_wrks_' + str(nworkers) +nnodes = int(nsim_workers*nodes_per_worker) + +if is_manager: + create_node_file(num_nodes=nnodes, name=node_file) + +if comms == 'mpi': + libE_specs['mpi_comm'].Barrier() + + +# Mock up system +customizer = {'mpi_runner': 'srun', # Select runner: mpich, openmpi, aprun, srun, jsrun + 'runner_name': 'srun', # Runner name: Replaces run command if not None + 'cores_on_node': (16, 64), # Tuple (physical cores, logical cores) + 'node_file': node_file} # Name of file containing a node-list + +# Create executor and register sim to it. +exctr = MPIExecutor(zero_resource_workers=in_place, central_mode=True, + auto_resources=True, allow_oversubscribe=False, + custom_info=customizer) +exctr.register_calc(full_path=sim_app, calc_type='sim') + + +if nworkers < 2: + sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") + +n = 2 +sim_specs = {'sim_f': sim_f, + 'in': ['x'], + 'out': [('f', float)], + } + +gen_specs = {'gen_f': gen_f, + 'in': [], + 'out': [('x', float, (n,))], + 'user': {'gen_batch_size': 20, + 'lb': np.array([-3, -2]), + 'ub': np.array([3, 2])} + } + +alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)]} +persis_info = add_unique_random_streams({}, nworkers + 1) +exit_criteria = {'sim_max': (nsim_workers)*rounds} + +# Each worker has 2 nodes. Basic test list for portable options +test_list_base = [{'testid': 'base1'}, # Give no config and no extra_args + {'testid': 'base2', 'nprocs': 5}, + {'testid': 'base3', 'nnodes': 1}, + {'testid': 'base4', 'ppn': 6}, + ] + +exp_srun = \ + ['srun -w node-1 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base1', + 'srun -w node-1 --ntasks 5 --nodes 1 --ntasks-per-node 5 /path/to/fakeapp.x --testid base2', + 'srun -w node-1 --ntasks 8 --nodes 1 --ntasks-per-node 8 /path/to/fakeapp.x --testid base3', + 'srun -w node-1 --ntasks 6 --nodes 1 --ntasks-per-node 6 /path/to/fakeapp.x --testid base4', + ] + +test_list = test_list_base +exp_list = exp_srun +sim_specs['user'] = {'tests': test_list, 'expect': exp_list, + 'nodes_per_worker': nodes_per_worker, 'persis_gens': n_gens} + + +# Perform the run +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, + alloc_specs, libE_specs) + +# All asserts are in sim func diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index f63d94fa89..d73d57950d 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -136,7 +136,7 @@ cleanup() { filelist=(*.err); [ -e ${filelist[0]} ] && rm *.err filelist=(*.pickle); [ -e ${filelist[0]} ] && rm *.pickle filelist=(.cov_unit_out*); [ -e ${filelist[0]} ] && rm .cov_unit_out* - filelist=(my_simtask.x); [ -e ${filelist[0]} ] && rm my_simtask.x + filelist=(simdir/*.x); [ -e ${filelist[0]} ] && rm simdir/*.x filelist=(libe_task_*.out); [ -e ${filelist[0]} ] && rm libe_task_*.out filelist=(*libE_stats.txt*); [ -e ${filelist[0]} ] && rm *libE_stats.txt* filelist=(my_machinefile); [ -e ${filelist[0]} ] && rm my_machinefile @@ -288,7 +288,7 @@ done # If none selected default to running all tests if [ "$RUN_MPI" = false ] && [ "$RUN_LOCAL" = false ] && [ "$RUN_TCP" = false ];then - RUN_MPI=true && RUN_LOCAL=true && RUN_TCP=true + RUN_MPI=true && RUN_LOCAL=true && RUN_TCP=false fi #----------------------------------------------------------------------------------------- @@ -457,7 +457,7 @@ if [ "$root_found" = true ]; then #Need proc count here for now - still stop on failure etc. NPROCS_LIST=$(sed -n '/# TESTSUITE_NPROCS/s/# TESTSUITE_NPROCS: //p' $TEST_SCRIPT) OS_SKIP_LIST=$(sed -n '/# TESTSUITE_OS_SKIP/s/# TESTSUITE_OS_SKIP: //p' $TEST_SCRIPT) - for NPROCS in $NPROCS_LIST + for NPROCS in ${NPROCS_LIST:(-1)} do NWORKERS=$((NPROCS-1)) diff --git a/libensemble/tests/scaling_tests/forces/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/run_libe_forces.py index 6252c087af..3c130ea389 100644 --- a/libensemble/tests/scaling_tests/forces/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/run_libe_forces.py @@ -5,9 +5,9 @@ # Import libEnsemble modules from libensemble.libE import libE -from libensemble.libE_manager import ManagerException +from libensemble.manager import ManagerException from libensemble.tools import parse_args, save_libE_output, add_unique_random_streams -from libensemble import libE_logger +from libensemble import logger from forces_support import test_libe_stats, test_ensemble_dir, check_log_exception USE_BALSAM = False @@ -21,7 +21,7 @@ from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first as alloc_f -libE_logger.set_level('INFO') # INFO is now default +logger.set_level('INFO') # INFO is now default nworkers, is_manager, libE_specs, _ = parse_args() @@ -84,7 +84,7 @@ libE_specs['save_every_k_gens'] = 1000 # Save every K steps libE_specs['sim_dirs_make'] = True # Separate each sim into a separate directory -libE_specs['profile_worker'] = False # Whether to have libE profile on (default False) +libE_specs['profile'] = False # Whether to have libE profile on (default False) # Maximum number of simulations sim_max = 8 diff --git a/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py b/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py index 2fee2e1eff..36c3c7cd82 100644 --- a/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py +++ b/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py @@ -42,7 +42,7 @@ from libensemble.tools import parse_args, save_libE_output, \ add_unique_random_streams -from libensemble import libE_logger +from libensemble import logger from libensemble.executors.mpi_executor import MPIExecutor import all_machine_specs @@ -56,7 +56,7 @@ print("you shouldn' hit that") sys.exit() -libE_logger.set_level('INFO') +logger.set_level('INFO') nworkers, is_manager, libE_specs, _ = parse_args() diff --git a/libensemble/tests/unit_tests/simdir/borehole.c b/libensemble/tests/unit_tests/simdir/borehole.c new file mode 100644 index 0000000000..1e99136a6e --- /dev/null +++ b/libensemble/tests/unit_tests/simdir/borehole.c @@ -0,0 +1,67 @@ +/* + * Compute Single borehole evaluation with optional delay + * For testing subprocessing of evaluation and possible kill. + * Author: S Hudson. + */ + +#include +#include +#include +#include +#include + +double borehole_func(char *filename){ + FILE* fh; + int i,j; + double x[3]; + double theta[4]; + double Hu, Ld_Kw, Treff, powparam, rw, Hl, L; + double numer, denom1, denom2, f; + + //Maybe open outside function + fh = fopen(filename, "rb"); + fread( theta, sizeof( double ), 4, fh ); + fread( x, sizeof( double ), 3, fh ); + + Hu = theta[0]; + Ld_Kw = theta[1]; + Treff = theta[2]; + powparam = theta[3]; + rw = x[0]; + Hl = x[1]; + + numer = 2.0 * M_PI * (Hu - Hl); + denom1 = 2.0 * Ld_Kw / pow(rw,2); + denom2 = Treff; + f = numer / (denom1 + denom2) * exp(powparam * rw); + + fclose(fh); + return f; +} + +int main(int argc, char **argv){ + + char* filename; + double delay; + double f; + + if (argc >=2) { + filename = argv[1]; // input file + } + else { + fprintf(stderr,"No input file supplied"); + exit(EXIT_FAILURE); + } + + if (argc >=3) { + delay = atof(argv[2]); // delay in seconds + // fprintf(stderr, "delay is %f\n",delay); + } + + sleep(delay); // Simulate longer function + f = borehole_func(filename); + printf("%.*e\n",15, f); // Print result to standard out. + fflush(stdout); + + return 0; +} diff --git a/libensemble/tests/unit_tests/simdir/c_startup.c b/libensemble/tests/unit_tests/simdir/c_startup.c new file mode 100644 index 0000000000..8f6342b118 --- /dev/null +++ b/libensemble/tests/unit_tests/simdir/c_startup.c @@ -0,0 +1,11 @@ +#include +#include + +int main (void) +{ + struct timeval tv; + gettimeofday(&tv, NULL); + printf ("%f\n", + (double) (tv.tv_usec) / 1000000 + + (double) (tv.tv_sec)); +} diff --git a/libensemble/tests/unit_tests/simdir/my_serialtask.c b/libensemble/tests/unit_tests/simdir/my_serialtask.c new file mode 100644 index 0000000000..59ea483f3d --- /dev/null +++ b/libensemble/tests/unit_tests/simdir/my_serialtask.c @@ -0,0 +1,41 @@ +#include +#include +#include +#include + +int main(int argc, char **argv) +{ + int usec_delay, error; + double fdelay; + + fdelay=3.0; + error=0; + + if (argc >=3) { + if (strcmp( argv[1],"sleep") == 0 ) { + fdelay = atof(argv[2]); + } + } + if (argc >=4) { + if (strcmp( argv[3],"Error") == 0 ) { + error=1; + } + } + if (argc >=4) { + if (strcmp( argv[3],"Fail") == 0 ) { + return(1); + } + } + + printf("Hello world sleeping for %f seconds\n",fdelay); + usec_delay = (int)(fdelay*1e6); + usleep(usec_delay); + + if (error==1) { + printf("Oh Dear! An non-fatal Error seems to have occured\n"); + fflush(stdout); + usleep(usec_delay); + } + + return(0); +} diff --git a/libensemble/tests/unit_tests/simdir/py_startup.py b/libensemble/tests/unit_tests/simdir/py_startup.py new file mode 100644 index 0000000000..21714d52d1 --- /dev/null +++ b/libensemble/tests/unit_tests/simdir/py_startup.py @@ -0,0 +1,3 @@ +import time + +print(time.time()) diff --git a/libensemble/tests/unit_tests/test_allocation_funcs.py b/libensemble/tests/unit_tests/test_allocation_funcs.py index fa2df73a4a..89defc37f8 100644 --- a/libensemble/tests/unit_tests/test_allocation_funcs.py +++ b/libensemble/tests/unit_tests/test_allocation_funcs.py @@ -1,6 +1,6 @@ from mpi4py import MPI -import libensemble.libE_manager as man +import libensemble.manager as man import libensemble.tests.unit_tests.setup as setup from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.history import History diff --git a/libensemble/tests/unit_tests/test_comms.py b/libensemble/tests/unit_tests/test_comms.py index 87baf3b3f2..6c40afa279 100644 --- a/libensemble/tests/unit_tests/test_comms.py +++ b/libensemble/tests/unit_tests/test_comms.py @@ -7,12 +7,16 @@ import time import queue import logging +from libensemble.tools.tools import osx_set_mp_method import numpy as np import libensemble.comms.comms as comms import libensemble.comms.logs as commlogs +osx_set_mp_method() + + def test_qcomm(): "Test queue-based bidirectional communicator." diff --git a/libensemble/tests/unit_tests/test_executor.py b/libensemble/tests/unit_tests/test_executor.py index edea3d94c7..9ddc5b85d4 100644 --- a/libensemble/tests/unit_tests/test_executor.py +++ b/libensemble/tests/unit_tests/test_executor.py @@ -13,49 +13,56 @@ USE_BALSAM = False - NCORES = 1 -sim_app = './my_simtask.x' +build_sims = ['my_simtask.c', 'my_serialtask.c', 'c_startup.c'] + +sim_app = 'simdir/my_simtask.x' +serial_app = 'simdir/my_serialtask.x' +c_startup = 'simdir/c_startup.x' +py_startup = 'simdir/py_startup.py' def setup_module(module): - print("setup_module module:%s" % module.__name__) + try: + print("setup_module module:%s" % module.__name__) + except AttributeError: + print("setup_module (direct run) module:%s" % module) if Executor.executor is not None: del Executor.executor Executor.executor = None + build_simfuncs() def setup_function(function): - print("setup_function function:%s" % function.__name__) + print("setup_function function:%s" % function.__name__) if Executor.executor is not None: del Executor.executor Executor.executor = None def teardown_module(module): - print("teardown_module module:%s" % module.__name__) + try: + print("teardown_module module:%s" % module.__name__) + except AttributeError: + print("teardown_module (direct run) module:%s" % module) if Executor.executor is not None: del Executor.executor Executor.executor = None -def build_simfunc(): +def build_simfuncs(): import subprocess - - # Build simfunc - # buildstring='mpif90 -o my_simtask.x my_simtask.f90' # On cray need to use ftn - buildstring = 'mpicc -o my_simtask.x simdir/my_simtask.c' - # subprocess.run(buildstring.split(), check=True) # Python3.5+ - subprocess.check_call(buildstring.split()) + for sim in build_sims: + app_name = '.'.join([sim.split('.')[0], 'x']) + if not os.path.isfile(app_name): + buildstring = 'mpicc -o ' + os.path.join('simdir', app_name) + ' ' + os.path.join('simdir', sim) + subprocess.check_call(buildstring.split()) # This would typically be in the user calling script # Cannot test auto_resources here - as no workers set up. def setup_executor(): - # sim_app = './my_simtask.x' - if not os.path.isfile(sim_app): - build_simfunc() - + """Set up an MPI Executor with sim app""" if USE_BALSAM: from libensemble.executors.balsam_executor import BalsamMPIExecutor exctr = BalsamMPIExecutor(auto_resources=False) @@ -66,26 +73,23 @@ def setup_executor(): exctr.register_calc(full_path=sim_app, calc_type='sim') -def setup_executor_noreg(): - # sim_app = './my_simtask.x' - if not os.path.isfile(sim_app): - build_simfunc() +def setup_serial_executor(): + """Set up serial Executor""" + from libensemble.executors.executor import Executor + exctr = Executor() + exctr.register_calc(full_path=serial_app, calc_type='sim') - if USE_BALSAM: - from libensemble.executors.balsam_executor import BalsamMPIExecutor - exctr = BalsamMPIExecutor(auto_resources=False) - else: - from libensemble.executors.mpi_executor import MPIExecutor - exctr = MPIExecutor(auto_resources=False) - exctr.register_calc(full_path=sim_app, calc_type='sim') +def setup_executor_startups(): + """Set up serial Executor""" + from libensemble.executors.executor import Executor + exctr = Executor() + exctr.register_calc(full_path=c_startup, app_name='c_startup') + exctr.register_calc(full_path=py_startup, app_name='py_startup') def setup_executor_noapp(): - # sim_app = './my_simtask.x' - if not os.path.isfile(sim_app): - build_simfunc() - + """Set up an MPI Executor but do not register application""" if USE_BALSAM: from libensemble.executors.balsam_executor import BalsamMPIExecutor exctr = BalsamMPIExecutor(auto_resources=False) @@ -97,10 +101,7 @@ def setup_executor_noapp(): def setup_executor_fakerunner(): - # sim_app = './my_simtask.x' - if not os.path.isfile(sim_app): - build_simfunc() - + """Set up an MPI Executor with a non-existent MPI runner""" if USE_BALSAM: print('Balsom does not support this feature - running MPIExecutor') @@ -116,8 +117,8 @@ def setup_executor_fakerunner(): # ----------------------------------------------------------------------------- # The following would typically be in the user sim_func -def polling_loop(exctr, task, timeout_sec=0.5, delay=0.05): - # import time +def polling_loop(exctr, task, timeout_sec=1, delay=0.05): + """Iterate over a loop, polling for an exit condition""" start = time.time() while time.time() - start < timeout_sec: @@ -149,7 +150,7 @@ def polling_loop(exctr, task, timeout_sec=0.5, delay=0.05): def polling_loop_multitask(exctr, task_list, timeout_sec=4.0, delay=0.05): - # import time + """Iterate over a loop, polling for exit conditions on multiple tasks""" start = time.time() while time.time() - start < timeout_sec: @@ -270,6 +271,18 @@ def test_kill_on_timeout(): assert task.state == 'USER_KILLED', "task.state should be USER_KILLED. Returned " + str(task.state) +def test_kill_on_timeout_polling_loop_method(): + print("\nTest: {}\n".format(sys._getframe().f_code.co_name)) + setup_executor() + exctr = Executor.executor + cores = NCORES + args_for_sim = 'sleep 10' + task = exctr.submit(calc_type='sim', num_procs=cores, app_args=args_for_sim) + exctr.polling_loop(task, timeout=1) + assert task.finished, "task.finished should be True. Returned " + str(task.finished) + assert task.state == 'USER_KILLED', "task.state should be USER_KILLED. Returned " + str(task.state) + + def test_launch_and_poll_multitasks(): print("\nTest: {}\n".format(sys._getframe().f_code.co_name)) setup_executor() @@ -336,13 +349,13 @@ def test_procs_and_machinefile_logic(): f.write(socket.gethostname() + '\n') task = exctr.submit(calc_type='sim', machinefile=machinefilename, app_args=args_for_sim) - task = polling_loop(exctr, task, delay=0.02) + task = polling_loop(exctr, task, delay=0.05) assert task.finished, "task.finished should be True. Returned " + str(task.finished) assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) # Testing num_procs = num_nodes*ranks_per_node (shouldn't fail) task = exctr.submit(calc_type='sim', num_procs=6, num_nodes=2, ranks_per_node=3, app_args=args_for_sim) - task = polling_loop(exctr, task, delay=0.02) + task = polling_loop(exctr, task, delay=0.05) assert task.finished, "task.finished should be True. Returned " + str(task.finished) assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) @@ -357,7 +370,7 @@ def test_procs_and_machinefile_logic(): # Testing no num_procs (shouldn't fail) task = exctr.submit(calc_type='sim', num_nodes=2, ranks_per_node=3, app_args=args_for_sim) assert 1 - task = polling_loop(exctr, task, delay=0.02) + task = polling_loop(exctr, task, delay=0.05) assert task.finished, "task.finished should be True. Returned " + str(task.finished) assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) @@ -372,14 +385,14 @@ def test_procs_and_machinefile_logic(): # Testing no num_nodes (shouldn't fail) task = exctr.submit(calc_type='sim', num_procs=2, ranks_per_node=2, app_args=args_for_sim) assert 1 - task = polling_loop(exctr, task, delay=0.02) + task = polling_loop(exctr, task, delay=0.05) assert task.finished, "task.finished should be True. Returned " + str(task.finished) assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) # Testing no ranks_per_node (shouldn't fail) task = exctr.submit(calc_type='sim', num_nodes=1, num_procs=2, app_args=args_for_sim) assert 1 - task = polling_loop(exctr, task, delay=0.02) + task = polling_loop(exctr, task, delay=0.05) assert task.finished, "task.finished should be True. Returned " + str(task.finished) assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) @@ -483,18 +496,6 @@ def test_launch_as_gen(): assert 0 -def test_launch_default_reg(): - print("\nTest: {}\n".format(sys._getframe().f_code.co_name)) - setup_executor_noreg() - exctr = Executor.executor - cores = NCORES - args_for_sim = 'sleep 0.1' - task = exctr.submit(calc_type='sim', num_procs=cores, app_args=args_for_sim) - task = polling_loop(exctr, task) - assert task.finished, "task.finished should be True. Returned " + str(task.finished) - assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) - - def test_launch_no_app(): print("\nTest: {}\n".format(sys._getframe().f_code.co_name)) setup_executor_noapp() @@ -636,14 +637,50 @@ def test_register_apps(): # assert e.args[1] == "Registered applications: ['my_simtask.x', 'fake_app1', 'fake_app2']" +def test_serial_exes(): + setup_serial_executor() + exctr = Executor.executor + args_for_sim = 'sleep 0.1' + task = exctr.submit(calc_type='sim', app_args=args_for_sim, wait_on_run=True) + task.wait() + assert task.finished, "task.finished should be True. Returned " + str(task.finished) + assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) + + +def test_serial_startup_times(): + setup_executor_startups() + exctr = Executor.executor + + t1 = time.time() + task = exctr.submit(app_name='c_startup') + task.wait() + stime = float(task.read_stdout()) + startup_time = stime - t1 + print('start up time for c program', startup_time) + assert task.finished, "task.finished should be True. Returned " + str(task.finished) + assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) + assert 0 < startup_time < 1, "Start up time for C program took " + str(startup_time) + + t1 = time.time() + task = exctr.submit(app_name='py_startup') + task.wait() + stime = float(task.read_stdout()) + startup_time = stime - t1 + print('start up time for python program', startup_time) + assert task.finished, "task.finished should be True. Returned " + str(task.finished) + assert task.state == 'FINISHED', "task.state should be FINISHED. Returned " + str(task.state) + assert 0 < startup_time < 1, "Start up time for python program took " + str(startup_time) + + if __name__ == "__main__": - # setup_module(__file__) + setup_module(__file__) test_launch_and_poll() test_launch_and_wait() test_launch_and_wait_timeout() test_launch_wait_on_run() test_kill_on_file() test_kill_on_timeout() + test_kill_on_timeout_polling_loop_method() test_launch_and_poll_multitasks() test_get_task() test_procs_and_machinefile_logic() @@ -651,7 +688,6 @@ def test_register_apps(): test_finish_and_kill() test_launch_and_kill() test_launch_as_gen() - test_launch_default_reg() test_launch_no_app() test_kill_task_with_no_submit() test_poll_task_with_no_submit() @@ -659,4 +695,6 @@ def test_register_apps(): test_retries_launch_fail() test_retries_run_fail() test_register_apps() - # teardown_module(__file__) + test_serial_exes() + test_serial_startup_times() + teardown_module(__file__) diff --git a/libensemble/tests/unit_tests/test_history.py b/libensemble/tests/unit_tests/test_history.py index b760281100..8977b981a6 100644 --- a/libensemble/tests/unit_tests/test_history.py +++ b/libensemble/tests/unit_tests/test_history.py @@ -3,6 +3,9 @@ import numpy as np from numpy import inf +if tuple(np.__version__.split('.')) >= ('1', '15'): + from numpy.lib.recfunctions import repack_fields + # Consider fixtures for this - parameterization may save duplication if always use pytest. # Comparing hist produced: options (using mix of first two) @@ -10,64 +13,71 @@ # - compare selected values # - compare from npy file - stored -wrs_H0 = np.array([(False, 0., 0, 0., 1, True, 1, True, [0., 0., 0.], True, 0.1, 1.1), - (False, 0., 0, 0., 1, True, 2, True, [0., 0., 0.], True, 0.2, 1.2), - (False, 0., 0, 0., 1, True, 3, True, [0., 0., 0.], True, 0.3, 1.3)], +wrs_H0 = np.array([(False, 0., 0, 0., 1, True, 1, True, [0., 0., 0.], True, 0.1, 1.1, False, False), + (False, 0., 0, 0., 1, True, 2, True, [0., 0., 0.], True, 0.2, 1.2, False, False), + (False, 0., 0, 0., 1, True, 3, True, [0., 0., 0.], True, 0.3, 1.3, False, False)], dtype=[('local_pt', '?'), ('priority', '` @@ -11,8 +11,12 @@ def avail_worker_ids(W, persistent=None): if persistent is None: return W['worker_id'][W['active'] == 0] if persistent: - return W['worker_id'][np.logical_and(W['active'] == 0, - W['persis_state'] != 0)] + if active_recv: + return W['worker_id'][np.logical_and(W['persis_state'] != 0, + W['active_recv'] != 0)] + else: + return W['worker_id'][np.logical_and(W['active'] == 0, + W['persis_state'] != 0)] return W['worker_id'][np.logical_and(W['active'] == 0, W['persis_state'] == 0)] @@ -51,7 +55,7 @@ def sim_work(Work, i, H_fields, H_rows, persis_info, **libE_info): :returns: None """ - libE_info['H_rows'] = H_rows + libE_info['H_rows'] = np.atleast_1d(H_rows) Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, 'tag': EVAL_SIM_TAG, @@ -68,8 +72,29 @@ def gen_work(Work, i, H_fields, H_rows, persis_info, **libE_info): :returns: None """ - libE_info['H_rows'] = H_rows + + # Count total gens + try: + gen_work.gen_counter += 1 + except AttributeError: + gen_work.gen_counter = 1 + libE_info['gen_count'] = gen_work.gen_counter + + libE_info['H_rows'] = np.atleast_1d(H_rows) Work[i] = {'H_fields': H_fields, 'persis_info': persis_info, 'tag': EVAL_GEN_TAG, 'libE_info': libE_info} + + +def all_returned(H, pt_filter=True): + """Check if all expected points have returned from sim + + :param H: A :doc:`history array<../data_structures/history_array>` + :param pt_filter: Optional boolean array filtering expected returned points: Default: All True + + :returns: Boolean. True if all expected points have been returned + """ + # Exclude cancelled points that were not already given out + excluded_points = H['cancel_requested'] & ~H['given'] + return np.all(H['returned'][pt_filter & ~excluded_points]) diff --git a/libensemble/tools/check_inputs.py b/libensemble/tools/check_inputs.py index 39778e15b4..bbc895446e 100644 --- a/libensemble/tools/check_inputs.py +++ b/libensemble/tools/check_inputs.py @@ -187,6 +187,10 @@ def check_inputs(libE_specs=None, alloc_specs=None, sim_specs=None, # Detailed checking based on Required Keys in docs for each specs if libE_specs is not None: + for name in libE_specs.get('final_fields', []): + assert name in out_names, \ + name + " in libE_specs['fields_keys'] is not in sim_specs['out'], "\ + "gen_specs['out'], alloc_specs['out'], H0, or libE_fields." check_libE_specs(libE_specs, serial_check) if alloc_specs is not None: diff --git a/libensemble/tools/fields_keys.py b/libensemble/tools/fields_keys.py index 7bbd3018d4..987440bebe 100644 --- a/libensemble/tools/fields_keys.py +++ b/libensemble/tools/fields_keys.py @@ -2,21 +2,27 @@ Below are the fields used within libEnsemble """ -libE_fields = [('sim_id', int), # Unique id of entry in H that was generated - ('gen_worker', int), # Worker that generated the entry - ('gen_time', float), # Time (since epoch) entry was entered into H - ('given', bool), # True if entry has been given for sim eval - ('returned', bool), # True if entry has been returned from sim eval - ('given_time', float), # Time (since epoch) that the entry was given - ('sim_worker', int), # Worker that did (or is doing) the sim eval +libE_fields = [('sim_id', int), # Unique id of entry in H that was generated + ('gen_worker', int), # Worker that (first) generated the entry + ('gen_time', float), # Time (since epoch) entry (first) was entered into H from a gen + ('last_gen_time', float), # Time (since epoch) entry was last requested by a gen + ('given', bool), # True if entry has been given for sim eval + ('given_time', float), # Time (since epoch) that the entry was (first) given to be evaluated + ('last_given_time', float), # Time (since epoch) that the entry was last given to be evaluated + ('returned', bool), # True if entry has been returned from sim eval + ('returned_time', float), # Time entry was (last) returned from sim eval + ('sim_worker', int), # Worker that did (or is doing) the sim eval + ('cancel_requested', bool), # True if cancellation of this entry is requested + ('kill_sent', bool) # True if a kill signal has been sent to worker ] # end_libE_fields_rst_tag protected_libE_fields = ['gen_worker', 'gen_time', 'given', - 'returned', 'given_time', + 'returned', + 'returned_time', 'sim_worker'] allowed_sim_spec_keys = ['sim_f', # @@ -56,14 +62,17 @@ 'authkey', # 'comms', # 'disable_log_files', # + 'final_fields', # 'ip', # 'mpi_comm', # 'nworkers', # 'port', # - 'profile_worker', # + 'profile', # + 'safe_mode', # 'save_every_k_gens', # 'save_every_k_sims', # 'save_H_and_persis_on_abort', # + 'use_persis_return', # 'workerID', # 'worker_timeout', # 'zero_resource_workers', # diff --git a/libensemble/tools/gen_support.py b/libensemble/tools/gen_support.py index 2f129f133e..79e35aa9ba 100644 --- a/libensemble/tools/gen_support.py +++ b/libensemble/tools/gen_support.py @@ -37,5 +37,10 @@ def get_mgr_worker_msg(comm): if tag in [STOP_TAG, PERSIS_STOP]: comm.push_to_buffer(tag, Work) return tag, Work, None - _, calc_in = comm.recv() + + data_tag, calc_in = comm.recv() + # Check for unexpected STOP (e.g. error between sending Work info and rows) + if data_tag in [STOP_TAG, PERSIS_STOP]: + comm.push_to_buffer(data_tag, calc_in) + return data_tag, calc_in, None # calc_in is signal identifier return tag, Work, calc_in diff --git a/libensemble/tools/tools.py b/libensemble/tools/tools.py index 96cfa3bd9c..e67b484115 100644 --- a/libensemble/tools/tools.py +++ b/libensemble/tools/tools.py @@ -6,6 +6,7 @@ import os import sys import logging +import platform import numpy as np import pickle @@ -47,6 +48,16 @@ "Resolve this by ensuring libE_specs['ensemble_dir_path'] is unique for each run." + '\n' + 79*'*' + '\n\n') +# ==================== Warning that persistent return data is not uesd ========== + +_PERSIS_RETURN_WARNING = \ + ('\n' + 79*'*' + '\n' + + "A persistent worker has returned history data on shutdown. This data is\n" + + "not currently added to the manager's history to avoid possibly overwriting, but\n" + + "will be added to the manager's history in a future release. If you want to\n" + + "overwrite/append, you can set the libE_specs option ``use_persis_return``" + + '\n' + 79*'*' + '\n\n') + # =================== save libE output to pickle and np ======================== @@ -107,12 +118,13 @@ def save_libE_output(H, persis_info, calling_file, nworkers, mess='Run completed # ===================== per-process numpy random-streams ======================= -def add_unique_random_streams(persis_info, nstreams): +def add_unique_random_streams(persis_info, nstreams, seed=''): """ Creates nstreams random number streams for the libE manager and workers - when nstreams is num_workers + 1. Stream i is initialized with seed i. + when nstreams is num_workers + 1. Stream i is initialized with seed i by default. + Otherwise the streams can be initialized with a provided seed. - The entries are appended to the existing persis_info dictionary. + The entries are appended to the provided persis_info dictionary. .. code-block:: python @@ -130,16 +142,28 @@ def add_unique_random_streams(persis_info, nstreams): Number of independent random number streams to produce + seed: :obj:`int` + + (Optional) Seed for identical random number streams for each worker. If + explicitly set to ``None``, random number streams are unique and seed + via other pseudorandom mechanisms. + """ for i in range(nstreams): + + if isinstance(seed, int) or seed is None: + random_seed = seed + else: + random_seed = i + if i in persis_info: persis_info[i].update({ - 'rand_stream': np.random.RandomState(i), + 'rand_stream': np.random.RandomState(random_seed), 'worker_num': i}) else: persis_info[i] = { - 'rand_stream': np.random.RandomState(i), + 'rand_stream': np.random.RandomState(random_seed), 'worker_num': i} return persis_info @@ -148,3 +172,15 @@ def add_unique_random_streams(persis_info, nstreams): def eprint(*args, **kwargs): """Prints a user message to standard error""" print(*args, file=sys.stderr, **kwargs) + + +# ===================== OSX set multiprocessing start ======================= +# On Python 3.8 on macOS, the default start method for new processes was +# switched to 'spawn' by default due to 'fork' potentially causing crashes. +# These crashes haven't yet been observed with libE, but with 'spawn' runs, +# warnings about leaked semaphore objects are displayed instead. +# The next several statements enforce 'fork' on macOS (Python 3.8) +def osx_set_mp_method(): + if platform.system() == 'Darwin': + from multiprocessing import set_start_method + set_start_method('fork', force=True) diff --git a/libensemble/version.py b/libensemble/version.py new file mode 100644 index 0000000000..603c0f9eef --- /dev/null +++ b/libensemble/version.py @@ -0,0 +1 @@ +__version__ = '0.7.2+dev' diff --git a/libensemble/libE_worker.py b/libensemble/worker.py similarity index 50% rename from libensemble/libE_worker.py rename to libensemble/worker.py index f1d0aa01d3..121bc5d7cc 100644 --- a/libensemble/libE_worker.py +++ b/libensemble/worker.py @@ -5,24 +5,20 @@ import socket import logging -import os -import shutil -import re import logging.handlers -from itertools import count, groupby -from operator import itemgetter +from itertools import count from traceback import format_exc +from traceback import format_exception_only as format_exc_msg import numpy as np from libensemble.message_numbers import \ EVAL_SIM_TAG, EVAL_GEN_TAG, \ UNSET_TAG, STOP_TAG, PERSIS_STOP, CALC_EXCEPTION -from libensemble.message_numbers import MAN_SIGNAL_FINISH +from libensemble.message_numbers import MAN_SIGNAL_FINISH, MAN_SIGNAL_KILL from libensemble.message_numbers import calc_type_strings, calc_status_strings -from libensemble.tools.fields_keys import libE_spec_sim_dir_keys, libE_spec_gen_dir_keys, libE_spec_calc_dir_misc +from libensemble.output_directory import EnsembleDirectory -from libensemble.utils.loc_stack import LocationStack from libensemble.utils.timer import Timer from libensemble.executors.executor import Executor from libensemble.comms.logs import worker_logging_config @@ -64,7 +60,7 @@ def worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=None, log_comm= Whether to send logging over comm """ - if libE_specs.get('profile_worker'): + if libE_specs.get('profile'): pr = cProfile.Profile() pr.enable() @@ -80,7 +76,7 @@ def worker_main(comm, sim_specs, gen_specs, libE_specs, workerID=None, log_comm= worker = Worker(comm, dtypes, workerID, sim_specs, gen_specs, libE_specs) worker.run() - if libE_specs.get('profile_worker'): + if libE_specs.get('profile'): pr.disable() profile_state_fname = 'worker_%d.prof' % (workerID) @@ -123,9 +119,6 @@ class Worker: :ivar dict calc_iter: Dictionary containing counts for each type of calc (e.g. sim or gen) - - :ivar LocationStack loc_stack: - Stack holding directory structure of this Worker """ def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs, libE_specs): @@ -137,81 +130,11 @@ def __init__(self, comm, dtypes, workerID, sim_specs, gen_specs, libE_specs): self.workerID = workerID self.sim_specs = sim_specs self.libE_specs = libE_specs - self.startdir = os.getcwd() - self.prefix = libE_specs.get('ensemble_dir_path', './ensemble') self.calc_iter = {EVAL_SIM_TAG: 0, EVAL_GEN_TAG: 0} - self.loc_stack = None self._run_calc = Worker._make_runners(sim_specs, gen_specs) - self._calc_id_counter = count() + # self._calc_id_counter = count() Worker._set_executor(self.workerID, self.comm) - - @staticmethod - def _make_calc_dir(libE_specs, workerID, H_rows, calc_str, locs): - "Create calc dirs and intermediate dirs, copy inputs, based on libE_specs" - - if calc_str == 'sim': - calc_input_dir = libE_specs.get('sim_input_dir', '').rstrip('/') - do_calc_dirs = libE_specs.get('sim_dirs_make', False) - copy_files = libE_specs.get('sim_dir_copy_files', []) - symlink_files = libE_specs.get('sim_dir_symlink_files', []) - else: # calc_str is 'gen' - calc_input_dir = libE_specs.get('gen_input_dir', '').rstrip('/') - do_calc_dirs = libE_specs.get('gen_dirs_make', False) - copy_files = libE_specs.get('gen_dir_copy_files', []) - symlink_files = libE_specs.get('gen_dir_symlink_files', []) - - prefix = libE_specs.get('ensemble_dir_path', './ensemble') - do_work_dirs = libE_specs.get('use_worker_dirs', False) - - # If 'use_worker_dirs' only calc_dir option. Use worker dirs, but no calc dirs - if do_work_dirs and not libE_specs.get('sim_dirs_make') and not libE_specs.get('gen_dirs_make'): - do_calc_dirs = False - - # If using calc_input_dir, set of files to copy is contents of provided dir - if calc_input_dir: - copy_files = set(copy_files + [os.path.join(calc_input_dir, i) for i in os.listdir(calc_input_dir)]) - - # If identical paths to copy and symlink, remove those paths from symlink_files - if len(symlink_files): - symlink_files = [i for i in symlink_files if i not in copy_files] - - # Cases where individual sim_dirs not created. - if not do_calc_dirs: - if do_work_dirs: # Each worker does work in worker dirs - key = workerID - dir = "worker" + str(workerID) - else: # Each worker does work in prefix (ensemble_dir) - key = prefix - dir = prefix - prefix = None - - locs.register_loc(key, dir, prefix=prefix, copy_files=copy_files, - symlink_files=symlink_files, ignore_FileExists=True) - return key - - # All cases now should involve sim_dirs - # ensemble_dir/worker_dir registered here, set as parent dir for sim dirs - if do_work_dirs: - worker_dir = "worker" + str(workerID) - worker_path = os.path.abspath(os.path.join(prefix, worker_dir)) - calc_dir = calc_str + str(H_rows) - locs.register_loc(workerID, worker_dir, prefix=prefix) - calc_prefix = worker_path - - # Otherwise, ensemble_dir set as parent dir for sim dirs - else: - calc_dir = "{}{}_worker{}".format(calc_str, H_rows, workerID) - if not os.path.isdir(prefix): - os.makedirs(prefix, exist_ok=True) - calc_prefix = prefix - - # Register calc dir with adjusted parent dir and source-file location - locs.register_loc(calc_dir, calc_dir, # Dir name also label in loc stack dict - prefix=calc_prefix, - copy_files=copy_files, - symlink_files=symlink_files) - - return calc_dir + self.EnsembleDirectory = EnsembleDirectory(libE_specs=libE_specs) @staticmethod def _make_runners(sim_specs, gen_specs): @@ -245,97 +168,6 @@ def _set_executor(workerID, comm): logger.info("No executor set on worker {}".format(workerID)) return False - @staticmethod - def _extract_H_ranges(Work): - """ Convert received H_rows into ranges for logging, labeling """ - work_H_rows = Work['libE_info']['H_rows'] - if len(work_H_rows) == 1: - return str(work_H_rows[0]) - else: - # From https://stackoverflow.com/a/30336492 - # Create groups by difference between row values and sequential enumerations: - # e.g., [2, 3, 5, 6] -> [(0, 2), (1, 3), (2, 5), (3, 6)] - # -> diff=-2, group=[(0, 2), (1, 3)], diff=-3, group=[(2, 5), (3, 6)] - ranges = [] - for diff, group in groupby(enumerate(work_H_rows.tolist()), lambda x: x[0]-x[1]): - # Take second values (rows values) from each group element into lists: - # group=[(0, 2), (1, 3)], group=[(2, 5), (3, 6)] -> group=[2, 3], group=[5, 6] - group = list(map(itemgetter(1), group)) - if len(group) > 1: - ranges.append(str(group[0]) + '-' + str(group[-1])) - else: - ranges.append(str(group[0])) - return '_'.join(ranges) - - def _copy_back(self): - """Copy back all ensemble dir contents to launch location""" - if os.path.isdir(self.prefix) and self.libE_specs.get('ensemble_copy_back', False): - - no_calc_dirs = not self.libE_specs.get('sim_dirs_make', True) or \ - not self.libE_specs.get('gen_dirs_make', True) - - ensemble_dir_path = self.libE_specs.get('ensemble_dir_path', './ensemble') - copybackdir = os.path.basename(ensemble_dir_path) - - if os.path.relpath(ensemble_dir_path) == os.path.relpath(copybackdir): - copybackdir += '_back' - - for dir in self.loc_stack.dirs.values(): - dest_path = os.path.join(copybackdir, os.path.basename(dir)) - if dir == self.prefix: # occurs when no_calc_dirs is True - continue # otherwise, entire ensemble dir copied into copyback dir - - shutil.copytree(dir, dest_path, symlinks=True) - if os.path.basename(dir).startswith('worker'): - return # Worker dir (with all contents) has been copied. - - # If not using calc dirs, likely miscellaneous files to copy back - if no_calc_dirs: - p = re.compile(r"((^sim)|(^gen))\d+_worker\d+") - for file in [i for i in os.listdir(self.prefix) if not p.match(i)]: # each non-calc_dir file - source_path = os.path.join(self.prefix, file) - dest_path = os.path.join(copybackdir, file) - try: - if os.path.isdir(source_path): - shutil.copytree(source_path, dest_path, symlinks=True) - else: - shutil.copy(source_path, dest_path, follow_symlinks=False) - except FileExistsError: - continue - except shutil.SameFileError: # creating an identical symlink - continue - - def _determine_dir_then_calc(self, Work, calc_type, calc_in, calc): - "Determines choice for calc_dir structure, then performs calculation." - - if not self.loc_stack: - self.loc_stack = LocationStack() - - if calc_type == EVAL_SIM_TAG: - H_rows = Worker._extract_H_ranges(Work) - else: - H_rows = str(self.calc_iter[calc_type]) - - calc_str = calc_type_strings[calc_type] - - calc_dir = Worker._make_calc_dir(self.libE_specs, self.workerID, - H_rows, calc_str, self.loc_stack) - - with self.loc_stack.loc(calc_dir): # Switching to calc_dir - return calc(calc_in, Work['persis_info'], Work['libE_info']) - - def _use_calc_dirs(self, type): - "Determines calc_dirs enabling for each calc type" - - if type == EVAL_SIM_TAG: - dir_type_keys = libE_spec_sim_dir_keys - else: - dir_type_keys = libE_spec_gen_dir_keys - - dir_type_keys += libE_spec_calc_dir_misc - - return any([setting in self.libE_specs for setting in dir_type_keys]) - def _handle_calc(self, Work, calc_in): """Runs a calculation on this worker object. @@ -356,21 +188,37 @@ def _handle_calc(self, Work, calc_in): self.calc_iter[calc_type] += 1 # calc_stats stores timing and summary info for this Calc (sim or gen) - calc_id = next(self._calc_id_counter) + # calc_id = next(self._calc_id_counter) + + # SH from output_directory.py + if calc_type == EVAL_SIM_TAG: + enum_desc = 'sim_id' + calc_id = EnsembleDirectory.extract_H_ranges(Work) + else: + enum_desc = 'Gen no' + # Use global gen count if available + if Work['libE_info'].get('gen_count'): + calc_id = str(Work['libE_info']['gen_count']) + else: + calc_id = str(self.calc_iter[calc_type]) + # Add a right adjust (mininum width). + calc_id = calc_id.rjust(5, ' ') + timer = Timer() try: - logger.debug("Running {}".format(calc_type_strings[calc_type])) + logger.debug("Starting {}: {}".format(enum_desc, calc_id)) calc = self._run_calc[calc_type] with timer: - logger.debug("Calling calc {}".format(calc_type)) - - if self._use_calc_dirs(calc_type): - out = self._determine_dir_then_calc(Work, calc_type, calc_in, calc) + if self.EnsembleDirectory.use_calc_dirs(calc_type): + loc_stack, calc_dir = self.EnsembleDirectory.prep_calc_dir(Work, self.calc_iter, + self.workerID, calc_type) + with loc_stack.loc(calc_dir): # Changes to calculation directory + out = calc(calc_in, Work['persis_info'], Work['libE_info']) else: out = calc(calc_in, Work['persis_info'], Work['libE_info']) - logger.debug("Return from calc call") + logger.debug("Returned from user function for {} {}".format(enum_desc, calc_id)) assert isinstance(out, tuple), \ "Calculation output must be a tuple." @@ -379,6 +227,9 @@ def _handle_calc(self, Work, calc_in): calc_status = out[2] if len(out) >= 3 else UNSET_TAG + if calc_status is None: + calc_status = UNSET_TAG + # Check for buffered receive if self.comm.recv_buffer: tag, message = self.comm.recv() @@ -387,8 +238,8 @@ def _handle_calc(self, Work, calc_in): calc_status = MAN_SIGNAL_FINISH return out[0], out[1], calc_status - except Exception: - logger.debug("Re-raising exception from calc") + except Exception as e: + logger.debug("Re-raising exception from calc {}".format(e)) calc_status = CALC_EXCEPTION raise finally: @@ -396,15 +247,17 @@ def _handle_calc(self, Work, calc_in): if task_timing and Executor.executor.list_of_tasks: # Initially supporting one per calc. One line output. task = Executor.executor.list_of_tasks[-1] - calc_msg = "Calc {:5d}: {} {} {} Status: {}".\ - format(calc_id, + calc_msg = "{} {}: {} {} {} Status: {}".\ + format(enum_desc, + calc_id, calc_type_strings[calc_type], timer, task.timer, calc_status_strings.get(calc_status, "Not set")) else: - calc_msg = "Calc {:5d}: {} {} Status: {}".\ - format(calc_id, + calc_msg = "{} {}: {} {} Status: {}".\ + format(enum_desc, + calc_id, calc_type_strings[calc_type], timer, calc_status_strings.get(calc_status, "Not set")) @@ -466,7 +319,17 @@ def run(self): mtag, Work = self.comm.recv() if mtag == STOP_TAG: - break + if Work is MAN_SIGNAL_FINISH: + break + elif Work is MAN_SIGNAL_KILL: + continue + + # Active recv is for persistent worker only - throw away here + if Work.get('libE_info', False): + if Work['libE_info'].get('active_recv', False) and not Work['libE_info'].get('persistent', False): + if len(Work['libE_info']['H_rows']) > 0: + _, _, _ = self._recv_H_rows(Work) + continue response = self._handle(Work) if response is None: @@ -474,9 +337,8 @@ def run(self): self.comm.send(0, response) except Exception as e: - self.comm.send(0, WorkerErrMsg(str(e), format_exc())) - self._copy_back() # Copy back current results on Exception + self.comm.send(0, WorkerErrMsg(' '.join(format_exc_msg(type(e), e)).strip(), format_exc())) else: self.comm.kill_pending() finally: - self._copy_back() + self.EnsembleDirectory.copy_back() diff --git a/postproc_scripts/print_fields.py b/postproc_scripts/print_fields.py new file mode 100755 index 0000000000..5caf9d9a4f --- /dev/null +++ b/postproc_scripts/print_fields.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +import sys +import numpy as np +import argparse + +desc = "Script to print selected fields of a libEnsemble history array from a file" +exmple = '''examples: + ./print_fields.py out1.npy --fields sim_id x f returned + + If no fields are supplied the whole array is printed. + + You can filter by using conditions, but only boolean are supported currently e.g: + ./print_fields.py out1.npy -f sim_id x f -c given ~returned + + would show lines where given is True and returned is False. + ''' + +np.set_printoptions(linewidth=1) + +parser = argparse.ArgumentParser(description=desc, epilog=exmple, + formatter_class=argparse.RawDescriptionHelpFormatter) +parser.add_argument('-f', '--fields', nargs='+', default=[]) +parser.add_argument('-c', '--condition', nargs='+', default=[]) +parser.add_argument('-s', '--show-fields', dest='show_fields', action='store_true') +parser.add_argument('args', nargs='*', help='*.npy file') +args = parser.parse_args() + +fields = args.fields +npfile = args.args +cond = args.condition +show_fields = args.show_fields + +if len(npfile) >= 1: + np_array = np.load(npfile[0]) +else: + parser.print_help() + sys.exit() + +if not fields: + fields = list(np_array.dtype.names) + +if show_fields: + print('Showing fields:', fields) + +if cond: + fltr = None + for cn in cond: + if cn[0] == '~': + fltr = ~np_array[cn[1:]] if fltr is None else ~np_array[cn[1:]] & fltr + else: + fltr = np_array[cn] if fltr is None else np_array[cn] & fltr + print(np_array[fields][fltr]) +else: + print(np_array[fields]) diff --git a/postproc_scripts/readme.rst b/postproc_scripts/readme.rst index a6a447bf93..022c2495c4 100644 --- a/postproc_scripts/readme.rst +++ b/postproc_scripts/readme.rst @@ -29,6 +29,11 @@ Results analysis scripts ./print_npy.py run_libe_forces_results_History_length=1000_evals=8.npy done +* ``print_fields.py``: Prints to screen from a given ``*.npy`` file containing + a NumPy structured array. This is a more versatile version of ``print_npy.py`` + that allows the user to select fields to print and boolean conditions determining + which rows are printed (see ``./print_fields.py -h`` for usage). + * ``compare_npy.py``: Compares either two provided ``*.npy`` files or one provided ``*.npy`` file with an expected results file (by default located at ../expected.npy). A tolerance is given on floating-point results, and NANs are diff --git a/setup.py b/setup.py index 60a4baaae1..2e66ee61ca 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,8 @@ from setuptools import setup from setuptools.command.test import test as TestCommand +exec(open('libensemble/version.py').read()) + class Run_TestSuite(TestCommand): def run_tests(self): @@ -30,7 +32,7 @@ def run_tests(self): setup( name='libensemble', - version='0.7.1+dev', + version=__version__, description='Library to coordinate the concurrent evaluation of dynamic ensembles of calculations', url='https://github.com/Libensemble/libensemble', author='Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro', @@ -77,10 +79,10 @@ def run_tests(self): 'Operating System :: Unix', 'Operating System :: MacOS', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Scientific/Engineering', 'Topic :: Software Development :: Libraries :: Python Modules'