refactor: support python3.12, simplify tests and dependencies (#1999)

* use µmamba in CI tests to avoid pyzmq issue * remove pytest-virtualenv as test dependency * rewrite generate_classes tests with virtualenv * minor simplifications to autotest/conftest.py * add python 3.12 classifier to pyproject.toml * add code generation test marker to pytest.ini * add IPRN value where missing in some example input files * add pip and replace bmipy with xmipy in etc/environment.yml * deduplicate in CI workflows (consolidate windows/non-windows)
modflowpy · Nov 13, 2023 · af8954b · af8954b
1 parent 5ebc216
commit af8954b
Show file tree

Hide file tree

Showing 15 changed files with 111 additions and 247 deletions.
diff --git a/.docs/md/generate_classes.md b/.docs/md/generate_classes.md
@@ -100,4 +100,4 @@ By default, a backup is made of FloPy's package classes before rewriting them. T
 
 ## Testing class generation
 
-Tests for the `generate_classes()` utility are located in `test_generate_classes.py`. The tests depend on [`pytest-virtualenv`](https://pypi.org/project/pytest-virtualenv/) and will be skipped if run in parallel without the `--dist loadfile` option for `pytest-xdist`.
+Tests for the `generate_classes()` utility are located in `test_generate_classes.py`. The tests depend on [`virtualenv`](https://pypi.org/project/virtualenv/) and will be skipped if run in parallel without the `--dist loadfile` option for `pytest-xdist`.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -12,37 +12,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]
         exclude:
           # avoid shutil.copytree infinite recursion bug
           # https://github.com/python/cpython/pull/17098
           - python-version: '3.8.0'
     defaults:
       run:
-        shell: bash
+        shell: bash -l {0}
     timeout-minutes: 90
 
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
 
-      - name: Setup Python
-        if: runner.os != 'Windows'
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: pyproject.toml
-
-      - name: Install Python dependencies
-        if: runner.os != 'Windows'
-        run: |
-          pip install --upgrade pip
-          pip install .
-          pip install ".[test, optional]"
-
       - name: Setup Micromamba
-        if: runner.os == 'Windows'
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: etc/environment.yml
@@ -54,29 +38,14 @@ jobs:
             bash
             powershell
 
-      - name: Install extra Python dependencies
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
-        run: |
-          pip install xmipy
-          pip install .
+      - name: Install FloPy
+        run: pip install .
 
       - name: Install Modflow executables
         uses: modflowpy/install-modflow-action@v1
 
       - name: Run benchmarks
-        if: runner.os != 'Windows'
-        working-directory: ./autotest
-        run: |
-          mkdir -p .benchmarks
-          pytest -v --durations=0 --benchmark-only --benchmark-json .benchmarks/${{ matrix.os }}_python${{ matrix.python-version }}.json --keep-failed=.failed
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Run benchmarks
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
-        working-directory: ./autotest
+        working-directory: autotest
         run: |
           mkdir -p .benchmarks
           pytest -v --durations=0 --benchmark-only --benchmark-json .benchmarks/${{ matrix.os }}_python${{ matrix.python-version }}.json --keep-failed=.failed
@@ -115,7 +84,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
           cache: 'pip'
           cache-dependency-path: pyproject.toml
 
@@ -151,7 +120,6 @@ jobs:
           fi
           python ./scripts/process_benchmarks.py ./autotest/.benchmarks ./autotest/.benchmarks
         env:
-          ARTIFACTS: ${{steps.run_tests.outputs.artifact_ids}}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload benchmark results

diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml
@@ -32,11 +32,10 @@ jobs:
           python -c "import flopy; print(f'{flopy.__version__}')"
 
       - name: Build package
-        run: |
-          python -m build
+        run: python -m build
+
       - name: Check package
-        run: |
-          twine check --strict dist/*
+        run: twine check --strict dist/*
 
   lint:
     name: Lint
@@ -119,7 +118,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install .
-          pip install ".[test,optional]"
+          pip install ".[test, optional]"
       
       - name: Install Modflow executables
         uses: modflowpy/install-modflow-action@v1
@@ -135,13 +134,13 @@ jobs:
         if: failure()
         with:
           name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }}
-          path: ./autotest/.failed/**
+          path: autotest/.failed/**
 
       - name: Upload coverage
         if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request')
         uses: codecov/codecov-action@v3
         with:
-          files: ./autotest/coverage.xml
+          files: autotest/coverage.xml
 
   test:
     name: Test
@@ -151,37 +150,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]
         exclude:
           # avoid shutil.copytree infinite recursion bug
           # https://github.com/python/cpython/pull/17098
           - python-version: '3.8.0'
     defaults:
       run:
-        shell: bash
+        shell: bash -l {0}
     timeout-minutes: 45
     steps:
 
       - name: Checkout repo
         uses: actions/checkout@v4
 
-      - name: Setup Python
-        if: runner.os != 'Windows'
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: pyproject.toml
-
-      - name: Install Python dependencies
-        if: runner.os != 'Windows'
-        run: |
-          pip install --upgrade pip
-          pip install .
-          pip install ".[test, optional]"
-
       - name: Setup Micromamba
-        if: runner.os == 'Windows'
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: etc/environment.yml
@@ -193,12 +176,8 @@ jobs:
             bash
             powershell
 
-      - name: Install Python dependencies
-        shell: bash -l {0}
-        if: runner.os == 'Windows'
-        run: |
-          pip install xmipy
-          pip install .
+      - name: Install FloPy
+        run: pip install .
 
       - name: Install Modflow-related executables
         uses: modflowpy/install-modflow-action@v1
@@ -208,28 +187,11 @@ jobs:
         with:
           repo: modflow6-nightly-build
 
-      - name: Update FloPy packages
-        if: runner.os != 'Windows'
-        run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup
-
-      - name: Update FloPy packages
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
+      - name: Update package classes
         run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup
 
       - name: Run tests
-        if: runner.os != 'Windows'
-        working-directory: ./autotest
-        run: |
-          pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile
-          coverage report
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Run tests
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
-        working-directory: ./autotest
+        working-directory: autotest
         run: |
           pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile
           coverage report
@@ -241,11 +203,10 @@ jobs:
         if: failure()
         with:
           name: failed-${{ matrix.os }}-${{ matrix.python-version }}
-          path: |
-            ./autotest/.failed/**
+          path: autotest/.failed/**
 
       - name: Upload coverage
         if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request')
         uses: codecov/codecov-action@v3
         with:
-          files: ./autotest/coverage.xml
+          files: autotest/coverage.xml
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -12,36 +12,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]
         exclude:
           # avoid shutil.copytree infinite recursion bug
           # https://github.com/python/cpython/pull/17098
           - python-version: '3.8.0'
     defaults:
       run:
-        shell: bash
+        shell: bash -l {0}
     timeout-minutes: 90
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
 
-      - name: Setup Python
-        if: runner.os != 'Windows'
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: pyproject.toml
-
-      - name: Install Python dependencies
-        if: runner.os != 'Windows'
-        run: |
-          pip install --upgrade pip
-          pip install .
-          pip install ".[test, optional]"
-
       - name: Setup Micromamba
-        if: runner.os == 'Windows'
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: etc/environment.yml
@@ -53,14 +37,10 @@ jobs:
             bash
             powershell
 
-      - name: Install extra Python dependencies
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
-        run: |
-          pip install xmipy
-          pip install .
+      - name: Install FloPy
+        run: pip install .
 
-      - name: Workaround OpenGL issue on Linux
+      - name: OpenGL workaround on Linux
         if: runner.os == 'Linux'
         run: |
           # referenced from https://github.com/pyvista/pyvista/blob/main/.github/workflows/vtk-pre-test.yml#L53
@@ -85,28 +65,11 @@ jobs:
           repo: modflow6-nightly-build
 
       - name: Update FloPy packages
-        if: runner.os != 'Windows'
-        run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup
-
-      - name: Update FloPy packages
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
         run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup
 
       - name: Run example tests
-        if: runner.os != 'Windows'
-        working-directory: ./autotest
-        run: |
-          pytest -v -m="example" -n=auto -s --durations=0 --keep-failed=.failed
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Run example tests
-        if: runner.os == 'Windows'
-        shell: bash -l {0}
-        working-directory: ./autotest
-        run: |
-          pytest -v -m="example" -n=auto -s --durations=0 --keep-failed=.failed
+        working-directory: autotest
+        run: pytest -v -m="example" -n=auto -s --durations=0 --keep-failed=.failed
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -115,5 +78,4 @@ jobs:
         if: failure()
         with:
           name: failed-example-${{ matrix.os }}-${{ matrix.python-version }}
-          path: |
-            ./autotest/.failed/**
+          path: autotest/.failed/**