ci: convert addition autotests to use flopyTest testing framework

.github/workflows/ci.yml

@@ -130,7 +130,7 @@ jobs:
       uses: actions/cache@v2.1.0
       env:
         # Increase this value to reset cache if environment.yml has changed
-        CACHE_NUMBER: 1
+        CACHE_NUMBER: 2
       with:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.run-type }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('etc/environment.yml') }}
@@ -154,11 +154,7 @@ jobs:
 
     - name: Add packages to flopy environment using mamba or conda
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          conda env update --name flopy --file etc/environment.yml
-        else
-          mamba env update --name flopy --file etc/environment.yml
-        fi
+        mamba env update --name flopy --file etc/environment.yml
 
     - name: Install pymake, xmipy, and flopy
       run: |
@@ -175,14 +171,8 @@ jobs:
       run: |
         echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - name: Run pytest on autotest scripts on Linux and MacOS
-      if: matrix.run-type == 'std' && runner.os != 'Windows'
-      working-directory: ./autotest
-      run: |
-        pytest -v -n auto --durations=0 --cov=flopy --cov-report=xml
-
-    - name: Run pytest on autotest scripts on Windows
-      if: matrix.run-type == 'std' && runner.os == 'Windows'
+    - name: Run pytest on autotest scripts
+      if: matrix.run-type == 'std'
       working-directory: ./autotest
       run: |
         pytest -v -n auto --durations=0 --cov=flopy --cov-report=xml

CONTRIBUTING.md

@@ -47,31 +47,32 @@ You can file new issues by filling out our [new issue form](https://github.com/m
 Before you submit your Pull Request (PR) consider the following guidelines:
 
 1. Search [GitHub](https://github.com/modflowpy/flopy/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate effort.
-1. Fork the modflowpy/flopy repo.
-1. Make your changes in a new git branch:
+2. Fork the modflowpy/flopy repo.
+3. Make your changes in a new git branch:
 
      ```shell
      git checkout -b my-fix-branch develop
      ```
 
-1. Create your patch, **including appropriate test cases**.
-1. Run the [black formatter](https://github.com/psf/black) on Flopy source files from the git repository root directory using:
+4. Create your patch, **including appropriate test cases**. See [Autotesting,md](autotest/Autotesting.md) for guidelines for constructing autotests. 
+5. 
+6. Run the [black formatter](https://github.com/psf/black) on Flopy source files from the git repository root directory using:
 
    ```shell
    black ./flopy
    ```
    Note: Pull Requests must pass black format checks run on the [GitHub actions](https://github.com/modflowpy/flopy/actions) (*linting*) before they will be accepted. The black formatter can be installed using [`pip`](https://pypi.org/project/black/) and [`conda`](https://anaconda.org/conda-forge/black). 
 
-1. Run the full FloPy test suite and ensure that all tests pass:
+7. Run the full FloPy test suite and ensure that all tests pass:
 
     ```shell
     cd autotest
-    nosetests -v get_exes.py
-    nosetests -v
+    pytest -v ci_prepare.py
+    pytest -v
     ```
-   Note: the FloPy test suite requires the [nosetests](https://pypi.org/project/nose/) and [pymake](https://github.com/modflowpy/pymake) python packages. All the FloPy dependencies must also be installed for the tests to pass.
+   Note: the FloPy test suite requires the [pytest](https://pypi.org/project/pytest/) and [pymake](https://github.com/modflowpy/pymake) python packages. All the FloPy dependencies must also be installed for the tests to pass.
 
-1. Commit your changes using a descriptive commit message that follows our
+8. Commit your changes using a descriptive commit message that follows our
   [commit message conventions](#commit). Adherence to these conventions
   is necessary because release notes are automatically generated from these messages.
 
@@ -80,13 +81,13 @@ Before you submit your Pull Request (PR) consider the following guidelines:
      ```
    Note: the optional commit `-a` command line option will automatically "add" and "rm" edited files.
 
-1. Push your branch to GitHub:
+9. Push your branch to GitHub:
 
     ```shell
     git push origin my-fix-branch
     ```
 
-1. In GitHub, send a pull request to `flopy:develop`.
+10. In GitHub, send a pull request to `flopy:develop`.
 * If we suggest changes then:
   * Make the required updates.
   * Re-run the FloPy test suites, in the autotest directory, to ensure tests are still passing.

autotest/Autotesting.md

@@ -0,0 +1,90 @@
+## Running autotests
+
+### Requirements
+To run the autotests you will need to install `pytest`
+
+    pip install pytest
+
+If you want to run the autotests you should install `pytest-xd`
+
+    pip install pytest pytest-xd
+
+### Running tests
+If you want to run a single autotest run the following command from the 
+autotest directory
+
+    pytest -v t001_test.py
+
+If you want to run all of the standard autotests (tests that match 
+`tXXX_test_*.py`) run the following command from the autotest directory
+
+    pytest -v 
+
+If you want to run all of the autotests the match a pattern run the following
+command from the autotest directory
+
+    pytest -v -k "t01"
+
+This would run autotests 10 through 19.
+
+
+### Running tests in parallel
+
+If you want to run the autotests in parallel add `-n auto` after `-v` in your
+`pytest` command. For example,
+
+    pytest -v -n auto t001_test.py
+
+The `auto` keyword indicates that the `pytest-xd` extension will query your 
+computer to determine the number of processors available. You can specify a 
+specific number of processors to use (for example, `-n 2` to run on two 
+processors). 
+
+The space between `-n` and the number of processors can be replaced with a
+`=`. For example,
+
+    pytest -v -n=auto t001_test.py
+
+
+## Creating an autotest
+
+Limit your autotest to a few tests of the same type. See `t002_test.py` for 
+an example of how not to create an autotest. This test includes tests for 
+loading data from fixed and free format text, loading binary files, using 
+`util2D`, and using `util3D`. Preferably all of these tests should have 
+been grouped into like tests and put into a separate autotest script.  
+
+The autotests run on GitHub Actions in parallel so new autotests should be
+developed so that data written by a specific test is written to a 
+unique folder that includes the basename for the script with the test (`t013` 
+for `t013_test.py`) and the test name (`t013_mytest`). This can all be done
+programatically using functions and classes in `ci_framework` script. An
+example of how to construct an autotest is given below
+
+```python
+from ci_framework import baseTestDir, flopyTest
+
+# set the baseDir variable using the script name
+baseDir = baseTestDir(__file__, relPath="temp", verbose=True)
+
+def test_mytest():
+    ws = f"{baseDir}_test_mytest"
+    testFramework = flopyTest(verbose=True, testDirs=ws, create=True)
+
+    ...test something
+
+    assert something is True, "oops"
+
+    ws = f"{baseDir}_test_mytest_another_directory"
+    testFramework.addTestDir(ws, create=True)
+
+    ...test something_else
+
+    assert something_else is True, "oops"
+
+    return
+
+```
+
+Pull requests with new autotests will not be accepted if tests do not follow
+the example provided above.

autotest/ci_framework.py

@@ -1,9 +1,11 @@
 import os
 import sys
 import shutil
-import time
 import pymake
 
+# command line arguments to:
+#   1. keep (--keep) test files
+#
 for idx, arg in enumerate(sys.argv):
     if "--keep" in arg.lower():
         keep = True
@@ -56,6 +58,13 @@ def baseTestDir(
     baseDir : str
         base test directory to create for the autotest
 
+    Example
+    -------
+
+    >>> from ci_framework import baseTestDir
+    >>> baseDir = baseTestDir(__file__, relPath="temp", create=True)
+    >>> print(f"baseDir: {baseDir}")
+
     """
     fileName = os.path.basename(filePath)
     if not fileName.startswith("t"):
@@ -91,9 +100,12 @@ def createTestDir(testDir, clean=False, verbose=False):
         boolean indicating if diagnostic information should be written
         to the screen
 
-    Returns
+    Example
     -------
 
+    >>> from ci_framework import createTestDir
+    >>> createTestDir("temp/mydir", clean=True, verbose=True)
+
     """
     if clean:
         _cleanDir(testDir, verbose=verbose)
@@ -104,21 +116,78 @@ def createTestDir(testDir, clean=False, verbose=False):
 
 
 class flopyTest(object):
+    """
+    The flopyTest class is used to setup test directories for flopy
+    autotests.
+
+    Attributes
+    ----------
+    clean : bool
+        boolean indicating if an existing directory should be cleaned
+    create : bool
+        boolean indicating if the directory should be created
+    testDirs : str or list/tuple of strings
+        path to where the test directory should be located
+    verbose : bool
+        boolean indicating if diagnostic information should be written
+        to the screen
+    retain : bool
+        boolean indicating if the test files should be retained
+
+    Methods
+    -------
+    addTestDir(testDirs, clean=False, create=False)
+        Add a testDir or a list of testDirs to the object
+
+    Example
+    -------
+
+    >>> from ci_framework import flopyTest
+    >>> def test_function():
+    ...     testFramework = flopyTest(verbose=True, testDirs="temp/t091_01")
+    ...     testFramework.addTestDir("temp/t091_02", create=True)
+
+    """
+
     def __init__(
         self,
         clean=False,
         create=False,
         testDirs=None,
         verbose=False,
+        retain=None,
     ):
-        self.clean = clean
-        self.verbose = verbose
-        self.createDirs = create
-        self.testDirs = []
+        if retain is None:
+            retain = keep
+        self._clean = clean
+        self._verbose = verbose
+        self._createDirs = create
+        self._retain = retain
+        self._testDirs = []
         if testDirs is not None:
             self.addTestDir(testDirs, clean=clean, create=create)
 
+    def __del__(self):
+        if not self._retain:
+            for testDir in self._testDirs:
+                _cleanDir(testDir, verbose=self._verbose)
+        else:
+            print("Retaining test files")
+
     def addTestDir(self, testDirs, clean=False, create=False):
+        """
+        Add a test directory to the flopyTest object.
+
+        Parameters
+        ----------
+        testDirs : str or list/tuple of strings
+            path to where the test directory should be located
+        clean : bool
+            boolean indicating if an existing directory should be cleaned
+        create : bool
+            boolean indicating if the directory should be created
+
+        """
         if isinstance(testDirs, str):
             testDirs = [testDirs]
         elif isinstance(testDirs, (int, float, bool)):
@@ -127,22 +196,24 @@ def addTestDir(self, testDirs, clean=False, create=False):
                 "list of strings, or tuple of strings."
             )
         for testDir in testDirs:
-            if testDir not in self.testDirs:
-                self.testDirs.append(testDir)
-                if self.verbose:
+            if testDir not in self._testDirs:
+                self._testDirs.append(testDir)
+                if self._verbose:
                     print(f"adding test directory...{testDir}")
                 if create:
-                    createTestDir(testDir, clean=clean, verbose=self.verbose)
-
-    def teardown(self):
-        if not keep:
-            for testDir in self.testDirs:
-                _cleanDir(testDir, verbose=self.verbose)
-        else:
-            print("Retaining test files")
+                    createTestDir(testDir, clean=clean, verbose=self._verbose)
 
 
 def _get_mf6path():
+    """
+    Get the path for the MODFLOW 6 example problems
+
+    Returns
+    -------
+    mf6pth : str
+        path to the directory containing the MODFLOW 6 example problems.
+
+    """
     parentPath = get_parent_path()
     if parentPath is None:
         parentPath = "."
@@ -155,6 +226,17 @@ def download_mf6_examples(delete_existing=False):
     """
     Download mf6 examples and return location of folder
 
+    Parameters
+    ----------
+    delete_existing : bool
+        boolean flag indicating to delete the existing MODFLOW 6 example
+        directory (temp/mf6examples), if it exists.
+
+    Returns
+    -------
+    mf6pth : str
+        path to the directory containing the MODFLOW 6 example problems.
+
     """
     # save current directory
     cpth = os.getcwd()

autotest/ci_prepare.py

@@ -156,20 +156,6 @@ def test_list_download():
     list_exes()
 
 
-def test_build_dirs():
-    parent_path = get_parent_path()
-    if parent_path is None:
-        print("cannot build test directories")
-    else:
-        dirPath = os.path.join(parent_path, "temp")
-        createTestDir(dirPath, verbose=True)
-
-        tests = ["scripts"]
-        for test in tests:
-            testDir = os.path.join(dirPath, test)
-            createTestDir(testDir, verbose=True)
-
-
 def test_download_mf6_examples(delete_existing=True):
     if run_type == "std":
         downloadDir = download_mf6_examples(delete_existing=delete_existing)
@@ -184,5 +170,4 @@ def test_download_mf6_examples(delete_existing=True):
     test_update_flopy()
     cleanup()
     list_exes()
-    test_build_dirs()
     test_download_mf6_examples()