From 38335dcb3d0998e1b4d8fec01f0bce7aa22cf517 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 29 Aug 2024 15:51:04 -0500 Subject: [PATCH 1/8] initial commit, trying to move some installs into files --- .github/workflows/extra.yml | 2 +- install/gen_deps_pypi.txt | 2 ++ install/ubuntu_pypi.txt | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 install/gen_deps_pypi.txt create mode 100644 install/ubuntu_pypi.txt diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 1af0cb737a..9eeaea1dde 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -125,7 +125,7 @@ jobs: - name: Install surmise if: matrix.os != 'macos-latest' run: | - pip install --upgrade git+https://github.com/bandframework/surmise.git + pip install git+https://github.com/bandframework/surmise.git - name: Build ytopt and dependencies if: matrix.python-version <= '3.10' && matrix.os != 'macos-latest' diff --git a/install/gen_deps_pypi.txt b/install/gen_deps_pypi.txt new file mode 100644 index 0000000000..eba0fa7d33 --- /dev/null +++ b/install/gen_deps_pypi.txt @@ -0,0 +1,2 @@ +mpmath==1.3.0 +matplotlib==3.9.2 diff --git a/install/ubuntu_pypi.txt b/install/ubuntu_pypi.txt new file mode 100644 index 0000000000..7e6e70e566 --- /dev/null +++ b/install/ubuntu_pypi.txt @@ -0,0 +1,2 @@ +git+https://github.com/bandframework/surmise.git +git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject From 6333c1875bd8b2eb56cdc55295e9b4c1b2b0343d Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 30 Aug 2024 08:55:26 -0500 Subject: [PATCH 2/8] remove caching. we're getting into the realm of simpler ci, probably, especially if heffte/ytopt are removed --- .github/workflows/extra.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 9eeaea1dde..ce8eba5abc 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -67,20 +67,6 @@ jobs: channel-priority: flexible auto-update-conda: true - - uses: actions/cache/restore@v4 - name: Restore cached dependencies - id: cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/work/libensemble/libensemble/heffte/build/ - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }}-${{ matrix.pydantic-version }}-extra - - name: Force-update certifi run: | python --version @@ -195,19 +181,6 @@ jobs: cp ./benchmarks/speed3d_c2c ../../libensemble/tests/regression_tests/ # end heffte build and dependencies - - uses: actions/cache/save@v4 - name: Save dependencies to cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/work/libensemble/libensemble/heffte/build/ - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }} - - name: Install libEnsemble, flake8, lock environment run: | pip install pydantic==${{ matrix.pydantic-version }} From 480dc05cb19aec2cdcdf41014c70891ad34ef071 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 30 Aug 2024 09:01:01 -0500 Subject: [PATCH 3/8] remove some commented-out deps, fix pydantic version checks from probably weird merge --- .github/workflows/extra.yml | 6 ++---- install/ubuntu_no312.txt | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 install/ubuntu_no312.txt diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index ce8eba5abc..8768915aaa 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -138,8 +138,6 @@ jobs: run: | sudo apt-get install bc - # pip install dragonfly-opt - # pip install git+https://github.com/dragonfly/dragonfly.git pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject pip install scikit-build packaging Tasmanian --user @@ -201,13 +199,13 @@ jobs: rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py - name: Install redis/proxystore on Pydantic 2 - if: matrix.pydantic-version == '2.6.4' + if: matrix.pydantic-version == '2.8.2' run: | pip install redis pip install proxystore==0.7.0 - name: Remove proxystore test on Pydantic 1 - if: matrix.pydantic-version == '1.10.13' + if: matrix.pydantic-version == '1.10.17' run: | rm ./libensemble/tests/regression_tests/test_proxystore_integration.py diff --git a/install/ubuntu_no312.txt b/install/ubuntu_no312.txt new file mode 100644 index 0000000000..c9a654f051 --- /dev/null +++ b/install/ubuntu_no312.txt @@ -0,0 +1,4 @@ +gpcam +scikit-build +packaging +git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject From 371adcfee240e8266c3ce8e9fb68d2b552d290a6 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 30 Aug 2024 11:09:39 -0500 Subject: [PATCH 4/8] moving pip installs from extra.yml into other requirements.txts --- .github/workflows/extra.yml | 27 ++++++++------------------- install/gen_deps_pypi.txt | 2 -- install/testing_requirements.txt | 1 + 3 files changed, 9 insertions(+), 21 deletions(-) delete mode 100644 install/gen_deps_pypi.txt diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 8768915aaa..d0f718a5a8 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -96,25 +96,15 @@ jobs: - name: Install generator dependencies run: | python -m pip install --upgrade pip - pip install mpmath - pip install matplotlib conda env update --file install/gen_deps_environment.yml - #- name: Install ax-platform - # run: pip install "ax-platform<=0.4.0" - - - name: Install gpcam - if: matrix.python-version != '3.12' - run: | - pip install gpcam - - - name: Install surmise - if: matrix.os != 'macos-latest' + - name: Install some non-macOS dependencies + if: matrix.os == 'ubuntu-latest' run: | - pip install git+https://github.com/bandframework/surmise.git + pip install -r install/ubuntu_pypi.txt - name: Build ytopt and dependencies - if: matrix.python-version <= '3.10' && matrix.os != 'macos-latest' + if: matrix.python-version <= '3.10' && matrix.os == 'ubuntu-latest' run: | pip install scikit-learn==1.4.0 pip install pandas==2.2.1 @@ -133,13 +123,12 @@ jobs: pip install . cd .. - - name: Install generator dependencies for Ubuntu tests + - name: Install generator dependencies for Ubuntu tests, not Python 3.12 if: matrix.os == 'ubuntu-latest' && matrix.python-version != '3.12' run: | sudo apt-get install bc - - pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject - pip install scikit-build packaging Tasmanian --user + pip install -r install/ubuntu_no312.txt + pip install Tasmanian --user - name: Install Balsam on Pydantic 1 if: matrix.pydantic-version == '1.10.17' @@ -165,7 +154,7 @@ jobs: popd - name: Build heffte - if: matrix.os != 'macos-latest' + if: matrix.os == 'ubuntu-latest' run: | # begin heffte build and dependencies sudo apt install libfftw3-dev diff --git a/install/gen_deps_pypi.txt b/install/gen_deps_pypi.txt deleted file mode 100644 index eba0fa7d33..0000000000 --- a/install/gen_deps_pypi.txt +++ /dev/null @@ -1,2 +0,0 @@ -mpmath==1.3.0 -matplotlib==3.9.2 diff --git a/install/testing_requirements.txt b/install/testing_requirements.txt index bf07e69d11..e1d1fda382 100644 --- a/install/testing_requirements.txt +++ b/install/testing_requirements.txt @@ -7,3 +7,4 @@ mock==5.1.0 python-dateutil==2.9.0.post0 anyio==4.4.0 matplotlib==3.9.2 +mpmath==1.3.0 From 811002f3499018ddd483beea95978c64983fba22 Mon Sep 17 00:00:00 2001 From: jlnav Date: Fri, 30 Aug 2024 12:46:58 -0500 Subject: [PATCH 5/8] remove dragonfly from a .txt, should only be in non-3.12 file --- install/ubuntu_pypi.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/install/ubuntu_pypi.txt b/install/ubuntu_pypi.txt index 7e6e70e566..208268dca9 100644 --- a/install/ubuntu_pypi.txt +++ b/install/ubuntu_pypi.txt @@ -1,2 +1 @@ git+https://github.com/bandframework/surmise.git -git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject From 216ee8b700791dd05fab20c5229cbb2106eedce2 Mon Sep 17 00:00:00 2001 From: jlnav Date: Tue, 3 Sep 2024 10:41:34 -0500 Subject: [PATCH 6/8] surmise may only work on <= 3.11? --- .github/workflows/extra.yml | 7 ++----- install/ubuntu_no312.txt | 1 + install/ubuntu_pypi.txt | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 install/ubuntu_pypi.txt diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index d0f718a5a8..f9bb2e6a40 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -98,11 +98,6 @@ jobs: python -m pip install --upgrade pip conda env update --file install/gen_deps_environment.yml - - name: Install some non-macOS dependencies - if: matrix.os == 'ubuntu-latest' - run: | - pip install -r install/ubuntu_pypi.txt - - name: Build ytopt and dependencies if: matrix.python-version <= '3.10' && matrix.os == 'ubuntu-latest' run: | @@ -186,6 +181,8 @@ jobs: run: | rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py + rm ./libensemble/tests/regression_tests/test_persistent_surmise_calib.py + rm ./libensemble/tests/regression_tests/test_persistent_surmise_killsims.py - name: Install redis/proxystore on Pydantic 2 if: matrix.pydantic-version == '2.8.2' diff --git a/install/ubuntu_no312.txt b/install/ubuntu_no312.txt index c9a654f051..20d9136890 100644 --- a/install/ubuntu_no312.txt +++ b/install/ubuntu_no312.txt @@ -2,3 +2,4 @@ gpcam scikit-build packaging git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject +git+https://github.com/bandframework/surmise.git diff --git a/install/ubuntu_pypi.txt b/install/ubuntu_pypi.txt deleted file mode 100644 index 208268dca9..0000000000 --- a/install/ubuntu_pypi.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://github.com/bandframework/surmise.git From 8c17f4e4e1c436e8c5b84433a3efeff300204075 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2024 12:27:03 -0500 Subject: [PATCH 7/8] cleanup, removing caching sections again --- .github/workflows/basic.yml | 25 ------------------- .github/workflows/extra.yml | 48 +++---------------------------------- 2 files changed, 3 insertions(+), 70 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index dcdbd047d3..8373f298cd 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -63,19 +63,6 @@ jobs: channel-priority: flexible auto-update-conda: true - - uses: actions/cache/restore@v4 - name: Restore cached dependencies - id: cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }}-${{ matrix.pydantic-version }}-basic - - name: Force-update certifi run: | python --version @@ -115,18 +102,6 @@ jobs: pip install mpmath matplotlib conda install numpy nlopt scipy - - uses: actions/cache/save@v4 - name: Save dependencies to cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }} - - name: Install libEnsemble, flake8 run: | pip install pydantic==${{ matrix.pydantic-version }} diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index ff3516ce97..3b16f9e3f2 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -67,19 +67,6 @@ jobs: channel-priority: flexible auto-update-conda: true - - uses: actions/cache/restore@v4 - name: Restore cached dependencies - id: cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }}-${{ matrix.pydantic-version }}-extra - - name: Force-update certifi run: | python --version @@ -96,31 +83,20 @@ jobs: conda install clang_osx-64 - name: Install mpi4py and MPI from conda - if: (matrix.python-version != '3.10' && matrix.os == 'ubuntu-latest') || matrix.os == 'macos-latest' run: | conda install mpi4py ${{ matrix.mpi-version }} - - name: Install mpi4py from pip, MPI from conda - if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' - run: | - conda install ${{ matrix.mpi-version }} - pip install mpi4py - - name: Install generator dependencies run: | - python -m pip install --upgrade pip conda env update --file install/gen_deps_environment.yml - #- name: Install ax-platform - # run: pip install "ax-platform<=0.4.0" - - name: Install gpcam if: matrix.python-version != '3.12' run: | pip install gpcam - name: Install surmise - if: matrix.os != 'macos-latest' + if: matrix.os == 'ubuntu-latest' run: | pip install --upgrade git+https://github.com/bandframework/surmise.git @@ -154,18 +130,6 @@ jobs: pip install -e . popd - - uses: actions/cache/save@v4 - name: Save dependencies to cache - if: matrix.os == 'ubuntu-latest' - with: - path: | - /home/runner/.local - /usr/share/miniconda3/envs/condaenv - /usr/share/miniconda3/bin - /usr/share/miniconda3/lib - /usr/share/miniconda3/include - key: libe-${{ github.ref_name }}-${{ matrix.python-version }}-${{ matrix.comms-type }} - - name: Install libEnsemble, flake8, lock environment run: | pip install pydantic==${{ matrix.pydantic-version }} @@ -178,7 +142,7 @@ jobs: # rm ./libensemble/tests/regression_tests/test_gpCAM.py rm ./libensemble/tests/regression_tests/test_persistent_gp.py - - name: Remove test for persistent Tasmanian on Python 3.12 + - name: Remove test for persistent Tasmanian, Surmise on Python 3.12 if: matrix.python-version >= '3.12' run: | rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py @@ -209,16 +173,10 @@ jobs: with: redis-version: 7 - - name: Run extensive tests, Ubuntu - if: matrix.os == 'ubuntu-latest' + - name: Run extensive tests run: | ./libensemble/tests/run-tests.sh -e -z -${{ matrix.comms-type }} - - name: Run extensive tests, macOS - if: matrix.os == 'macos-latest' - run: | - ./libensemble/tests/run-tests.sh -e -z -${{ matrix.comms-type }} - - name: Merge coverage run: | mv libensemble/tests/.cov* . From ee4d4fc053d789c2fbfcc878693c69b17ed07895 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 4 Sep 2024 13:34:11 -0500 Subject: [PATCH 8/8] create install_ibcdfo.sh script, pin some deps --- .github/workflows/basic.yml | 10 +--------- .github/workflows/extra.yml | 10 +--------- install/install_ibcdfo.sh | 10 ++++++++++ install/ubuntu_no312.txt | 6 +++--- 4 files changed, 15 insertions(+), 21 deletions(-) create mode 100644 install/install_ibcdfo.sh diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 8373f298cd..000ed9e85c 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -82,15 +82,7 @@ jobs: run: | pip install -r install/testing_requirements.txt pip install -r install/misc_feature_requirements.txt - - git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git - pushd IBCDFO/minq/py/minq5/ - export PYTHONPATH="$PYTHONPATH:$(pwd)" - echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - popd - pushd IBCDFO/ibcdfo_pypkg/ - pip install -e . - popd + source install/install_ibcdfo.sh - name: Install mpi4py and MPI from conda run: | diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 3b16f9e3f2..11ebcdbb6e 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -120,15 +120,7 @@ jobs: conda install octave pip install -r install/testing_requirements.txt pip install -r install/misc_feature_requirements.txt - - git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git - pushd IBCDFO/minq/py/minq5/ - export PYTHONPATH="$PYTHONPATH:$(pwd)" - echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - popd - pushd IBCDFO/ibcdfo_pypkg/ - pip install -e . - popd + source install/install_ibcdfo.sh - name: Install libEnsemble, flake8, lock environment run: | diff --git a/install/install_ibcdfo.sh b/install/install_ibcdfo.sh new file mode 100644 index 0000000000..efd5f6dcb5 --- /dev/null +++ b/install/install_ibcdfo.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git +pushd IBCDFO/minq/py/minq5/ +export PYTHONPATH="$PYTHONPATH:$(pwd)" +echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV +popd +pushd IBCDFO/ibcdfo_pypkg/ +pip install -e . +popd diff --git a/install/ubuntu_no312.txt b/install/ubuntu_no312.txt index 20d9136890..c66fb45a01 100644 --- a/install/ubuntu_no312.txt +++ b/install/ubuntu_no312.txt @@ -1,5 +1,5 @@ -gpcam -scikit-build -packaging +gpcam==8.1.6 +scikit-build==0.18.1 +packaging==24.1 git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject git+https://github.com/bandframework/surmise.git