feat: get modflow utility (#1465)

.github/workflows/rtd.yml

@@ -35,28 +35,24 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2.2.2
         with:
-          python-version: 3.9
+          python-version: 3.8
 
       - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
 
       - name: Install prerequisites
         run: |
-          pip install https://github.com/modflowpy/pymake/zipball/master
           pip install jupyter jupytext
 
       - name: Install flopy and dependencies
         run: |
-          pip install .[test,doc]
+          pip install .[doc]
 
-      - name: Prepare for the autotests
-        working-directory: ./autotest
-        run: |
-          pytest -v ci_prepare.py
-
-      - name: Add executables directory to path
+      - name: Install executables and add to PATH
         run: |
+          mkdir -p $HOME/.local/bin
+          get-modflow $HOME/.local/bin
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Run jupytext on tutorials

autotest/ci_prepare.py

@@ -2,24 +2,15 @@
 Script to be used to download any required data prior to autotests
 """
 import os
-import shutil
 import subprocess
 import sys
 
-import pymake
-from ci_framework import download_mf6_examples, get_parent_path
+from ci_framework import download_mf6_examples
 
 import flopy
 
 # os.environ["CI"] = "1"
 
-# path where downloaded executables will be extracted
-parent_path = get_parent_path()
-exe_pth = os.path.join(parent_path, "temp", "exe_download")
-# make the directory if it does not exist
-if not os.path.isdir(exe_pth):
-    os.makedirs(exe_pth, exist_ok=True)
-
 # determine if running on Travis
 is_CI = "CI" in os.environ
 
@@ -29,13 +20,12 @@
     dotlocal = True
 
 if not dotlocal:
-    for idx, arg in enumerate(sys.argv):
+    for arg in sys.argv:
         if "--ci" in arg.lower():
             dotlocal = True
             break
 if dotlocal:
     bindir = os.path.join(os.path.expanduser("~"), ".local", "bin")
-    bindir = os.path.abspath(bindir)
     print(f"bindir: {bindir}")
     if not os.path.isdir(bindir):
         os.makedirs(bindir, exist_ok=True)
@@ -44,7 +34,7 @@
 print(f'modflow executables will be downloaded to:\n\n    "{bindir}"')
 
 run_type = "std"
-for idx, arg in enumerate(sys.argv):
+for arg in sys.argv:
     if "--other" in arg.lower():
         run_type = "other"
         break
@@ -86,39 +76,8 @@ def get_branch():
     return branch
 
 
-def cleanup():
-    if os.path.isdir(exe_pth):
-        shutil.rmtree(exe_pth)
-    return
-
-
-def move_exe():
-    files = os.listdir(exe_pth)
-    for file in files:
-        if file.startswith("__"):
-            continue
-        src = os.path.join(exe_pth, file)
-        dst = os.path.join(bindir, file)
-        print(f"moving {src} -> {dst}")
-        shutil.move(src, dst)
-    return
-
-
-def list_exes():
-    cmd = f"ls -l {bindir}"
-    os.system(cmd)
-    return
-
-
 def test_download_and_unzip():
-    error_msg = "pymake not installed - cannot download executables"
-    assert pymake is not None, error_msg
-
-    pymake.getmfexes(exe_pth, verbose=True)
-
-    move_exe()
-
-    return
+    flopy.utils.get_modflow_main(bindir)
 
 
 def test_download_nightly_build():
@@ -132,11 +91,7 @@ def test_download_nightly_build():
     # Replace MODFLOW 6 executables with the latest versions
     else:
         print("Updating MODFLOW 6 executables from the nightly-build repo")
-        pymake.getmfnightly(pth=exe_pth, exes=["mf6", "libmf6"], verbose=True)
-
-        move_exe()
-
-    return
+        flopy.utils.get_modflow_main(bindir, repo="modflow6-nightly-build")
 
 
 def test_update_flopy():
@@ -148,14 +103,6 @@ def test_update_flopy():
         flopy.mf6.utils.generate_classes(branch="develop", backup=False)
 
 
-def test_cleanup():
-    cleanup()
-
-
-def test_list_download():
-    list_exes()
-
-
 def test_download_mf6_examples(delete_existing=True):
     if run_type == "std":
         downloadDir = download_mf6_examples(delete_existing=delete_existing)
@@ -168,6 +115,4 @@ def test_download_mf6_examples(delete_existing=True):
     test_download_and_unzip()
     test_download_nightly_build()
     test_update_flopy()
-    cleanup()
-    list_exes()
     test_download_mf6_examples()

autotest/test_scripts.py

@@ -0,0 +1,136 @@
+"""Test scripts."""
+import socket
+import sys
+from pathlib import Path
+from subprocess import Popen, PIPE
+
+import pytest
+
+
+def is_connected(hostname):
+    """See https://stackoverflow.com/a/20913928/ to test hostname."""
+    try:
+        host = socket.gethostbyname(hostname)
+        s = socket.create_connection((host, 80), 2)
+        s.close()
+        return True
+    except Exception:
+        pass
+    return False
+
+
+requires_github = pytest.mark.skipif(
+    not is_connected("github.com"), reason="github.com is required."
+)
+
+flopy_dir = Path(__file__).parents[1] / "flopy"
+get_modflow_script = flopy_dir / "utils" / "get_modflow.py"
+
+
+@pytest.fixture(scope="session")
+def downloads_dir(tmp_path_factory):
+    downloads_dir = tmp_path_factory.mktemp("Downloads")
+    return downloads_dir
+
+
+def run_py_script(script, *args):
+    """Run a Python script, return tuple (stdout, stderr, returncode)."""
+    args = [sys.executable, str(script)] + [str(g) for g in args]
+    print("running: " + " ".join(args))
+    p = Popen(args, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode()
+    stderr = stderr.decode()
+    returncode = p.returncode
+    return stdout, stderr, returncode
+
+
+def run_get_modflow_script(*args):
+    return run_py_script(get_modflow_script, *args)
+
+
+def test_script_usage():
+    assert get_modflow_script.exists()
+    stdout, stderr, returncode = run_get_modflow_script("-h")
+    assert "usage" in stdout
+    assert len(stderr) == 0
+    assert returncode == 0
+
+
+@requires_github
+def test_get_modflow(tmp_path, downloads_dir):
+    # exit if extraction directory does not exist
+    bindir = tmp_path / "bin1"
+    assert not bindir.exists()
+    stdout, stderr, returncode = run_get_modflow_script(bindir)
+    assert "does not exist" in stderr
+    assert returncode == 1
+
+    # ensure extraction directory exists
+    bindir.mkdir()
+    assert bindir.exists()
+
+    # attempt to fetch a non-existing release-id
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir, "--release-id", "1.9", "--downloads-dir", downloads_dir
+    )
+    assert "Release '1.9' not found" in stderr
+    assert returncode == 1
+
+    # fetch latest
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir, "--downloads-dir", downloads_dir
+    )
+    assert len(stderr) == returncode == 0
+    files = [item.name for item in bindir.iterdir() if item.is_file()]
+    assert len(files) > 20
+
+    # take only a few files using --subset, starting with invalid
+    bindir = tmp_path / "bin2"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir, "--subset", "mfnwt,mpx", "--downloads-dir", downloads_dir
+    )
+    assert "subset item not found: mpx" in stderr
+    assert returncode == 1
+    # now valid subset
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir, "--subset", "mfnwt,mp6", "--downloads-dir", downloads_dir
+    )
+    assert len(stderr) == returncode == 0
+    files = [item.stem for item in bindir.iterdir() if item.is_file()]
+    assert sorted(files) == ["mfnwt", "mfnwtdbl", "mp6"]
+
+    # similar as before, but also specify a ostag
+    bindir = tmp_path / "bin3"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir,
+        "--subset",
+        "mfnwt",
+        "--release-id",
+        "2.0",
+        "--ostag",
+        "win64",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    assert len(stderr) == returncode == 0
+    files = [item.name for item in bindir.iterdir() if item.is_file()]
+    assert sorted(files) == ["mfnwt.exe", "mfnwtdbl.exe"]
+
+
+@requires_github
+def test_get_mf6_nightly(tmp_path, downloads_dir):
+    bindir = tmp_path / "bin1"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_modflow_script(
+        bindir,
+        "--repo",
+        "modflow6-nightly-build",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    assert len(stderr) == returncode == 0
+    files = [item.name for item in bindir.iterdir() if item.is_file()]
+    assert len(files) >= 4

docs/get_modflow.md

@@ -0,0 +1,42 @@
+# Install MODFLOW and related programs
+
+This method describes how to install USGS MODFLOW and related programs for Windows, Mac or Linux using a "get modflow" utility. If FloPy is installed, the utility is available in the Python environment as a `get-modflow` command. The same utility is also available as a Python script `get_modflow.py`, described later.
+
+The utility uses a [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives of programs that have been compiled with modern Intel Fortran compilers. The utility is able to match the binary archive to the operating system, and extract the console programs to a user-defined directory. A prompt can also be used to assist where to install programs.
+
+## Command-line interface
+
+### Using `get-modflow` from FloPy
+
+When FloPy is installed, a `get-modflow` (or `get-modflow.exe` for Windows) program is installed, which is usually installed to the PATH (depending on the Python setup). From a console:
+
+```console
+$ get-modflow --help
+usage: get-modflow [-h]
+...
+```
+
+### Using `get_modflow.py` as a script
+
+The script requires Python 3.6 or later and does not have any dependencies, not even FloPy. It can be downloaded separately and used the same as the console program, except with a different invocation. For example:
+
+```console
+$ wget https://raw.githubusercontent.com/modflowpy/flopy/develop/flopy/utils/get_modflow.py
+$ python3 get_modflow.py --help
+usage: get_modflow.py [-h]
+...
+```
+
+## FloPy module
+
+The same functionality of the command-line interface is available from the FloPy module, as demonstrated below:
+
+```python
+from pathlib import Path
+import flopy
+
+bindir = Path("/tmp/bin")
+bindir.mkdir()
+flopy.utils.get_modflow_main(bindir)
+list(bindir.iterdir())
+```

flopy/utils/__init__.py

@@ -31,6 +31,7 @@
 from .check import check
 from .flopy_io import read_fixed_var, write_fixed_var
 from .formattedfile import FormattedHeadFile
+from .get_modflow import run_main as get_modflow_main
 from .gridintersect import GridIntersect, ModflowGridIndices
 from .mflistfile import (
     Mf6ListBudget,