Merge pull request #1095 from Libensemble/develop

Release/v 1.0.0

.flake8

@@ -4,8 +4,6 @@ max-line-length = 120
 ignore =
         # E203 is not PEP 8 compliant. https://github.com/PyCQA/pycodestyle/issues/373
         E203
-        # Not worrying about spaces around arithmetic operations
-        E226
         # Linebreaks before/after unary operators are okay
         W503, W504
 exclude =
@@ -27,28 +25,23 @@ per-file-ignores =
         # systems/versions of MPI:
         libensemble/tests/standalone_tests/mpi_launch_test/create_mpi_jobs.py:E402
 
-        # Similar reasoning for configuring mpi4py
-        libensemble/tests/unit_tests/test_executor.py:E402
-
         # Matplotlib needs something set before import
         scripts/*:E402
 
         # Ignoring linelength in APOSMM
         libensemble/gen_funcs/*aposmm.py:E501
         examples/gen_funcs/*aposmm.py:E501
+        libensemble/tests/functionality_tests/test_mpi_runners.py:E501
+        libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py:E501
 
         # Need to set something before the APOSMM import
         libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py:E402
         examples/calling_scripts/run_libensemble_on_warpx.py:E402
-        libensemble/gen_funcs/persistent_aposmm.py:E402, E501
         libensemble/tests/regression_tests/test_persistent_aposmm*:E402
         libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py:E402
         libensemble/tests/regression_tests/test_ytopt_heffte.py:E402
         libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py:E402
         libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py:E402
-        libensemble/tests/functionality_tests/test_mpi_runners.py:E501
-        libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py:E501
-        libensemble/tests/deprecated_tests/test_old_aposmm*:E402
         libensemble/tests/unit_tests/test_persistent_aposmm.py:E402
 
         # Allow undefined name '__version__'

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# .github/dependabot.yml
+# Dependabot configuration file
+# See
+# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates
+
+version: 2
+
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: daily
+
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: daily
+
+  - package-ecosystem: gitsubmodule
+    directory: /
+    schedule:
+      interval: daily

.github/workflows/ci.yml

@@ -30,19 +30,10 @@ jobs:
                       python-version: "3.10"
                       mpi-version: "mpich"
                       comms-type: t
-                    # - os: ubuntu-latest
-                      # mpi-version: "openmpi"
-                      # python-version: "3.10"
-                      # comms-type: l
-                    # - os: windows-latest
-                    #   python-version: "3.10"
-                    #   comms-type: l
-                    #   mpi-version: "msmpi"
-                    # - os: windows-latest
-                    #   python-version: "3.10"
-                    #   comms-type: m
-                    #   mpi-version: "msmpi"
-
+                    - os: ubuntu-latest
+                      mpi-version: "openmpi"
+                      python-version: "3.10"
+                      comms-type: l
         env:
             HYDRA_LAUNCHER: "fork"
             TERM: xterm-256color
@@ -113,13 +104,12 @@ jobs:
         #   run: pip install mpi4py
 
         - name: Install mpi4py and MPI from conda
-          if: matrix.python-version != '3.10' && matrix.os != 'windows-latest'
+          if: (matrix.python-version != '3.10' && matrix.os == 'ubuntu-latest') || matrix.os == 'macos-latest'
           run: |
-            conda install ${{ matrix.mpi-version }}
-            conda install mpi4py
+            conda install mpi4py ${{ matrix.mpi-version }}
 
         - name: Install mpi4py from pip, MPI from conda
-          if: matrix.python-version == '3.10' && matrix.os != 'windows-latest'
+          if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest'
           run: |
             conda install ${{ matrix.mpi-version }}
             pip install mpi4py
@@ -135,7 +125,7 @@ jobs:
         - name: Install surmise
           if: matrix.os != 'macos-latest' && steps.cache.outputs.cache-hit != 'true'
           run: |
-            pip install --upgrade git+https://github.com/surmising/surmise.git@develop
+            pip install --upgrade git+https://github.com/bandframework/surmise.git@develop
 
         - name: Build ytopt and dependencies
           if: matrix.python-version != '3.11' && matrix.os != 'windows-latest' && matrix.os != 'macos-latest'
@@ -222,11 +212,6 @@ jobs:
             pip install -e .
             flake8 libensemble
 
-        - name: Activate API unit test if using mpich
-          if: matrix.mpi-version == 'mpich'
-          run: |
-            mv libensemble/tests/unit_tests/mpich-only_test_ensemble.py libensemble/tests/unit_tests/test_ensemble.py
-
         - name: Remove ytopt-heffte test on Python 3.11 (easy way)
           if: matrix.python-version == '3.11'
           run: |

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     - id: end-of-file-fixer
       exclude: ^(.*\.xml|.*\.svg)$

.readthedocs.yml

@@ -18,4 +18,3 @@ python:
         path: .
         extra_requirements:
          - docs
-    system_packages: false

.spell

@@ -4,3 +4,4 @@ apoints
 numer
 hist
 inout
+slac

.wci.yml

@@ -2,8 +2,7 @@ name: libEnsemble
 icon: https://raw.githubusercontent.com/Libensemble/libensemble/main/docs/images/libE_logo.png
 headline: Tool for running dynamic ensembles.
 description: |
-  libEnsemble is a Python toolkit for coordinating workflows of asynchronous
-  and dynamic ensembles of calculations.
+  libEnsemble is a Python toolkit for running dynamic ensembles of calculations.
 
   Users write generator and simulator functions to express their ensembles.
   A library of example functions is available which can be modified as needed.
@@ -17,8 +16,8 @@ description: |
 language: Python
 
 release:
-  version: 0.10.2
-  date: 2023-07-24
+  version: 1.0.0
+  date: 2023-09-25
 
 documentation:
   general: https://libensemble.readthedocs.io

CHANGELOG.rst

@@ -8,6 +8,75 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r
 
 .. _`github releases page`: https://github.com/Libensemble/libensemble/releases
 
+Release 1.0.0
+--------------
+
+:Date: September 25, 2023
+
+New capabilities:
+
+* *libE_specs* option `final_gen_send` returns last results to the generator (replaces `final_fields`). #1086
+* *libE_specs* option `reuse_output_dir` allows reuse of workflow and ensemble directories. #1028 #1041
+* *libE_specs* option `calc_dir_id_width` no. of digits for calc ID in output sim/gen directories. #1052 / #1066
+* Added `gen_num_procs` and `gen_num_gpus` *libE_specs* (and *persis_info*) options for resourcing a generator. #1068
+* Added `gpu_env_fallback` option to platform fields - specifies a GPU environment variable (for non-MPI usage). #1050
+* New MPIExecutor `submit()` argument `mpi_runner_type` specifies an MPI runner for current call only. #1054
+* Allow oversubscription when using the `num_procs` *gen_specs["out"]* option. #1058
+* sim/gen_specs can use `outputs` in place of `out` to be consistent with `inputs`. #1075
+* Executor can be obtained from `libE_info` (4th parameter) in user functions. #1078
+
+Breaking changes:
+
+* *libE_specs* option `final_fields` is removed in favor of `final_gen_send`. #1086
+* *libE_specs* option `kill_canceled_sims` now defaults to **False**. #1062
+* *parse_args* is not run automatically by `Ensemble` constructor.
+
+Updates to **Object Oriented** Ensemble interface:
+
+* Added `parse_args` as option to `Ensemble` constructor. #1065
+* The *executor* can be passed as an option to the `Ensemble` constructor. #1078
+* Better handling of `Ensemble.add_random_streams` and `ensemble.persis_info`. #1074
+
+Output changes:
+
+* The worker ID suffix is removed from sim/gen output directories. #1041
+* Separate *ensemble.log* and *libE_stats.txt* for different workflows directories. #1027 #1041
+* Defaults to four digits for sim/gen ID in output directories (adds digits on overflow). #1052 / #1066
+
+Bug fixes:
+
+* Resolved PETSc/OpenMPI issue (when using the Executor). #1064
+* Prevent `mpi4py` validation running during local comms (when using OO interface). #1065
+
+Performance changes:
+
+* Optimize `kill_cancelled_sims` function. #1043 / #1063
+* *safe_mode* defaults to **False** (for performance). #1053
+
+Updates to example functions:
+
+* Multiple regression tests and examples ported to use OO ensemble interface. #1014
+
+Update forces examples:
+
+* Make persistent generator the default for both simple and GPU examples (inc. updated tutorials).
+* Update to object oriented interface.
+* Added separate variable resources example for forces GPU.
+* Rename `multi_task` example to `multi_app`.
+
+Documentation:
+
+* General overhaul and simplification of documentation. #992
+
+:Note:
+
+* Tested platforms include Linux, MacOS, Windows, and major systems such as Frontier (OLCF), Polaris, and Perlmutter (NERSC). The major system tests ran heterogeneous workflows.
+* Tested Python versions: (Cpython) 3.7, 3.8, 3.9, 3.10, 3.11.
+
+:Known Issues:
+
+* See known issues section in the documentation.
+
 Release 0.10.2
 --------------
 
@@ -63,7 +132,7 @@ New capabilities:
   * Allow users to specify their own system configurations.
   * These changes remove a number of tweaks that were needed for particular platforms.
 
-*  Resource management supports GPU and non-GPU simulations in the same ensemble. #993
+* Resource management supports GPU and non-GPU simulations in the same ensemble. #993
   * User's can specify `num_procs` and `num_gpus` in the generator for each evaluation.
 
 * Pydantic models are used for validating major libE input (input can be provided as classes or dictionaries). #878