feat(get_modflow): support modflow6 repo releases

autotest/test_get_modflow.py

@@ -1,6 +1,7 @@
 """Test get-modflow utility."""
 import sys
 import urllib
+from os.path import splitext
 from urllib.error import HTTPError
 
 import pytest
@@ -13,11 +14,12 @@
 
 from flopy.utils import get_modflow
 
+rate_limit_msg = "rate limit exceeded"
 flopy_dir = get_project_root_path(__file__)
 get_modflow_script = flopy_dir / "flopy" / "utils" / "get_modflow.py"
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def downloads_dir(tmp_path_factory):
     downloads_dir = tmp_path_factory.mktemp("Downloads")
     return downloads_dir
@@ -35,15 +37,12 @@ def test_script_usage():
     assert returncode == 0
 
 
-rate_limit_msg = "rate limit exceeded"
-
-
 @flaky
 @requires_github
-def test_get_modflow_script(tmp_path, downloads_dir):
-    # exit if extraction directory does not exist
-    bindir = tmp_path / "bin1"
+def test_script_executables(tmpdir, downloads_dir):
+    bindir = tmpdir / "bin1"
     assert not bindir.exists()
+
     stdout, stderr, returncode = run_get_modflow_script(bindir)
     if rate_limit_msg in stderr:
         pytest.skip(f"GitHub {rate_limit_msg}")
@@ -74,7 +73,7 @@ def test_get_modflow_script(tmp_path, downloads_dir):
     assert len(files) > 20
 
     # take only a few files using --subset, starting with invalid
-    bindir = tmp_path / "bin2"
+    bindir = tmpdir / "bin2"
     bindir.mkdir()
     stdout, stderr, returncode = run_get_modflow_script(
         bindir, "--subset", "mfnwt,mpx", "--downloads-dir", downloads_dir
@@ -94,7 +93,7 @@ def test_get_modflow_script(tmp_path, downloads_dir):
     assert sorted(files) == ["mfnwt", "mfnwtdbl", "mp6"]
 
     # similar as before, but also specify a ostag
-    bindir = tmp_path / "bin3"
+    bindir = tmpdir / "bin3"
     bindir.mkdir()
 
     stdout, stderr, returncode = run_get_modflow_script(
@@ -117,8 +116,8 @@ def test_get_modflow_script(tmp_path, downloads_dir):
 
 @flaky
 @requires_github
-def test_get_nightly_script(tmp_path, downloads_dir):
-    bindir = tmp_path / "bin1"
+def test_script_modflow6_nightly_build(tmpdir, downloads_dir):
+    bindir = tmpdir / "bin1"
     bindir.mkdir()
 
     stdout, stderr, returncode = run_get_modflow_script(
@@ -137,7 +136,30 @@ def test_get_nightly_script(tmp_path, downloads_dir):
 
 @flaky
 @requires_github
-def test_get_modflow(tmpdir):
+def test_script_modflow6(tmpdir, downloads_dir):
+    stdout, stderr, returncode = run_get_modflow_script(
+        tmpdir,
+        "--repo",
+        "modflow6",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert len(stderr) == returncode == 0
+
+    actual_exes = [splitext(p.name)[0] for p in tmpdir.glob("*")]
+    expected_exes = ["mf6", "mf5to6", "zbud6", "libmf6"]
+    assert all(exe in actual_exes for exe in expected_exes)
+
+    downloads = [p.name for p in downloads_dir.glob("*")]
+    assert len(downloads) > 0
+    assert any(dl.endswith("zip") for dl in downloads)
+
+
+@flaky
+@requires_github
+def test_python_api_executables(tmpdir):
     try:
         get_modflow(tmpdir)
     except HTTPError as err:
@@ -180,7 +202,7 @@ def test_get_modflow(tmpdir):
 
 @flaky
 @requires_github
-def test_get_nightly(tmpdir):
+def test_python_api_modflow6_nightly_build(tmpdir, downloads_dir):
     try:
         get_modflow(tmpdir, repo="modflow6-nightly-build")
     except urllib.error.HTTPError as err:
@@ -194,3 +216,21 @@ def test_get_nightly(tmpdir):
     ]
 
     assert all(exe in actual for exe in expected)
+
+
+@flaky
+@requires_github
+def test_python_api_modflow6(tmpdir, downloads_dir):
+    try:
+        get_modflow(tmpdir, repo="modflow6", downloads_dir=downloads_dir)
+    except urllib.error.HTTPError as err:
+        if err.code == 403:
+            pytest.skip(f"GitHub {rate_limit_msg}")
+
+    actual_exes = [splitext(p.name)[0] for p in tmpdir.glob("*")]
+    expected_exes = ["mf6", "mf5to6", "zbud6", "libmf6"]
+    assert all(exe in actual_exes for exe in expected_exes)
+
+    downloads = [p.name for p in downloads_dir.glob("*")]
+    assert len(downloads) > 0
+    assert any(dl.endswith("zip") for dl in downloads)

docs/get_modflow.md

@@ -1,12 +1,25 @@
 # Install MODFLOW and related programs
 
-This method describes how to install USGS MODFLOW and related programs for Windows, Mac or Linux using a "get modflow" utility. If FloPy is installed, the utility is available in the Python environment as a `get-modflow` command. The same utility is also available as a Python script `get_modflow.py`, described later.
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
-The utility uses a [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives of programs that have been compiled with modern Intel Fortran compilers. The utility is able to match the binary archive to the operating system, and extract the console programs to a user-defined directory. A prompt can also be used to assist where to install programs.
+
+- [Command-line interface](#command-line-interface)
+  - [Using the `get-modflow` command](#using-the-get-modflow-command)
+  - [Using `get_modflow.py` as a script](#using-get_modflowpy-as-a-script)
+- [FloPy module](#flopy-module)
+- [Where to install?](#where-to-install)
+- [Selecting a distribution](#selecting-a-distribution)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+FloPy includes a `get-modflow` utility to install USGS MODFLOW and related programs for Windows, Mac or Linux. If FloPy is installed, the utility is available in the Python environment as a `get-modflow` command. The script `flopy/utils/get_modflow.py` has no dependencies and can be invoked independently.
+
+The utility uses the [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives containing executables compiled with [Intel Fortran](https://www.intel.com/content/www/us/en/developer/tools/oneapi/fortran-compiler.html). The utility is able to match the binary archive to the operating system and extract the console programs to a user-defined directory. A prompt can also be used to help the user choose where to install programs.
 
 ## Command-line interface
 
-### Using `get-modflow` from FloPy
+### Using the `get-modflow` command
 
 When FloPy is installed, a `get-modflow` (or `get-modflow.exe` for Windows) program is installed, which is usually installed to the PATH (depending on the Python setup). From a console:
 
@@ -57,3 +70,20 @@ Other auto-select options are only available if the current user can write files
  - `:local` - use `$HOME/.local/bin`
  - `:system` - use `/usr/local/bin`
  - `:windowsapps` - use `%LOCALAPPDATA%\Microsoft\WindowsApps`
+
+## Selecting a distribution
+
+By default the distribution from the [executables repository](https://github.com/MODFLOW-USGS/executables) is installed. This includes the MODFLOW 6 binary `mf6` and over 20 other related programs. The utility can also install from the main [MODFLOW 6 repo](https://github.com/MODFLOW-USGS/modflow6) or the [nightly build](https://github.com/MODFLOW-USGS/modflow6-nightly-build). To select a distribution, specify a repository name with the `--repo` command line option or the `repo` function argument. Valid names are:
+
+- `executables` (default)
+- `modflow6`
+- `modflow6-nightly-build`
+
+The `modflow6-nightly-build` distribution contains only:
+
+- `mf6`
+- `mf5to6`
+- `zbud6`
+- `libmf6.dylib`
+
+The `modflow6` release archive contains the entire repository with an internal `bin` directory, whose contents are as above. These are copied to the selected `bindir` after the archive is unzipped in the download location.

flopy/utils/get_modflow.py

@@ -15,6 +15,7 @@
 import urllib.request
 import zipfile
 from importlib.util import find_spec
+from os.path import splitext
 from pathlib import Path
 
 __all__ = ["run_main"]
@@ -23,6 +24,7 @@
 owner = "MODFLOW-USGS"
 # key is the repo name, value is the renamed file prefix for the download
 renamed_prefix = {
+    "modflow6": "modflow6",
     "executables": "modflow_executables",
     "modflow6-nightly-build": "modflow6_nightly",
 }
@@ -129,8 +131,8 @@ def run_main(
         colon character. See error message or other documentation for further
         information on auto-select options.
     repo : str, default "executables"
-        Name of GitHub repository. Choose one of "executables" (default) or
-        "modflow6-nightly-build".
+        Name of GitHub repository. Choose one of "executables" (default),
+        "modflow6", or "modflow6-nightly-build".
     release_id : str, default "latest"
         GitHub release ID.
     ostag : str, optional
@@ -190,8 +192,13 @@ def run_main(
 
     if ostag is None:
         ostag = get_ostag()
+
+    # Windows asset in modflow6 repo release has no OS tag
+    if repo == "modflow6":
+        ostag = ""
+
     exe_suffix = ""
-    if ostag in ["win32", "win64"]:
+    if ostag in ["win32", "win64"] or ostag == "":
         exe_suffix = ".exe"
         lib_suffix = ".dll"
     elif ostag == "linux":
@@ -466,7 +473,9 @@ def add_item(key, fname, do_chmod):
                         )
                     ):
                         add_item(key, fname, do_chmod=True)
-        else:  # release 1.0 did not have code.json
+        # executables release 1.0 did not have code.json
+        # neither do modflow6 or nightly build releases
+        else:
             for fname in sorted(files):
                 if nosub or (subset and fname in subset):
                     extract.add(fname)
@@ -476,9 +485,34 @@ def add_item(key, fname, do_chmod):
         if not quiet:
             print(
                 f"extracting {len(extract)} "
-                f"file{'s' if len(extract) != 1 else ''} to '{bindir}'"
+                f"file{'s' if len(extract) != 1 else ''} to '{downloads_dir if repo == 'modflow6' else bindir}'"
             )
-        zipf.extractall(bindir, members=extract)
+
+        # modflow6 release contains the whole repo with an internal bindir
+        # executables & nightly build releases contain binaries themselves
+        if repo == "modflow6":
+            zipf.extractall(downloads_dir, members=extract)
+
+            inner = downloads_dir / splitext(asset_name)[0]
+            inner_bin = inner / "bin"
+
+            if not quiet:
+                print(
+                    f"copying {len(list(inner_bin.glob('*')))} files to '{bindir}'"
+                )
+
+            if sys.version_info[:3] < (3, 8):
+                from distutils.dir_util import copy_tree
+
+                copy_tree(str(inner_bin), str(bindir))
+            else:
+                from shutil import copytree
+
+                copytree(inner / "bin", bindir, dirs_exist_ok=True)
+
+            chmod = set(list(bindir.glob("*")))
+        else:
+            zipf.extractall(bindir, members=extract)
 
     # If this is a TemporaryDirectory, then delete the directory and files
     del tmpdir
@@ -490,7 +524,7 @@ def add_item(key, fname, do_chmod):
             pth.chmod(pth.stat().st_mode | 0o111)
 
     # Show listing
-    if not quiet:
+    if code and not quiet:
         print(columns_str(items))
 
         if not subset: