feat(get_modflow): support modflow6 repo releases (#1573)

* feat(get_modflow): support modflow6 repo releases

* * evaluate files to extract (rather than extracting to dowloads_dir)
* avoid div/0 with columns_str (with items longer than 79 chars)
* change asset dst_fname for modflow6
* prevent "--repo modflow6 --ostag win32" from succeding

Co-authored-by: Mike Taves <mwtoews@gmail.com>

autotest/test_get_modflow.py

@@ -1,6 +1,9 @@
 """Test get-modflow utility."""
 import sys
 import urllib
+from pathlib import Path
+from platform import system
+from typing import List
 from urllib.error import HTTPError
 
 import pytest
@@ -13,11 +16,12 @@
 
 from flopy.utils import get_modflow
 
+rate_limit_msg = "rate limit exceeded"
 flopy_dir = get_project_root_path(__file__)
 get_modflow_script = flopy_dir / "flopy" / "utils" / "get_modflow.py"
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def downloads_dir(tmp_path_factory):
     downloads_dir = tmp_path_factory.mktemp("Downloads")
     return downloads_dir
@@ -27,6 +31,16 @@ def run_get_modflow_script(*args):
     return run_py_script(get_modflow_script, *args)
 
 
+def assert_exts(paths: List[Path]):
+    exts = set([p.suffix for p in paths])
+    if system() == "Windows":
+        assert exts == {".exe", ".dll"}
+    elif system() == "Darwin":
+        assert exts == {"", ".dylib"}
+    elif system() == "Linux":
+        assert exts == {"", ".so"}
+
+
 def test_script_usage():
     assert get_modflow_script.exists()
     stdout, stderr, returncode = run_get_modflow_script("-h")
@@ -35,15 +49,12 @@ def test_script_usage():
     assert returncode == 0
 
 
-rate_limit_msg = "rate limit exceeded"
-
-
 @flaky
 @requires_github
-def test_get_modflow_script(tmp_path, downloads_dir):
-    # exit if extraction directory does not exist
-    bindir = tmp_path / "bin1"
+def test_script_executables(tmpdir, downloads_dir):
+    bindir = tmpdir / "bin1"
     assert not bindir.exists()
+
     stdout, stderr, returncode = run_get_modflow_script(bindir)
     if rate_limit_msg in stderr:
         pytest.skip(f"GitHub {rate_limit_msg}")
@@ -74,7 +85,7 @@ def test_get_modflow_script(tmp_path, downloads_dir):
     assert len(files) > 20
 
     # take only a few files using --subset, starting with invalid
-    bindir = tmp_path / "bin2"
+    bindir = tmpdir / "bin2"
     bindir.mkdir()
     stdout, stderr, returncode = run_get_modflow_script(
         bindir, "--subset", "mfnwt,mpx", "--downloads-dir", downloads_dir
@@ -94,7 +105,7 @@ def test_get_modflow_script(tmp_path, downloads_dir):
     assert sorted(files) == ["mfnwt", "mfnwtdbl", "mp6"]
 
     # similar as before, but also specify a ostag
-    bindir = tmp_path / "bin3"
+    bindir = tmpdir / "bin3"
     bindir.mkdir()
 
     stdout, stderr, returncode = run_get_modflow_script(
@@ -117,8 +128,8 @@ def test_get_modflow_script(tmp_path, downloads_dir):
 
 @flaky
 @requires_github
-def test_get_nightly_script(tmp_path, downloads_dir):
-    bindir = tmp_path / "bin1"
+def test_script_modflow6_nightly_build(tmpdir, downloads_dir):
+    bindir = tmpdir / "bin1"
     bindir.mkdir()
 
     stdout, stderr, returncode = run_get_modflow_script(
@@ -137,15 +148,41 @@ def test_get_nightly_script(tmp_path, downloads_dir):
 
 @flaky
 @requires_github
-def test_get_modflow(tmpdir):
+def test_script_modflow6(tmpdir, downloads_dir):
+    stdout, stderr, returncode = run_get_modflow_script(
+        tmpdir,
+        "--repo",
+        "modflow6",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert len(stderr) == returncode == 0
+
+    downloads = [p.name for p in downloads_dir.glob("*")]
+    assert len(downloads) > 0
+    assert any(dl.endswith("zip") for dl in downloads)
+
+    actual_paths = list(tmpdir.glob("*"))
+    actual_stems = [p.stem for p in actual_paths]
+    expected_stems = ["mf6", "mf5to6", "zbud6", "libmf6"]
+    assert all(stem in expected_stems for stem in actual_stems)
+    assert_exts(actual_paths)
+
+
+@flaky
+@requires_github
+def test_python_api_executables(tmpdir):
     try:
         get_modflow(tmpdir)
     except HTTPError as err:
         if err.code == 403:
             pytest.skip(f"GitHub {rate_limit_msg}")
 
-    actual = [p.name for p in tmpdir.glob("*")]
-    expected = [
+    actual_paths = list(tmpdir.glob("*"))
+    actual_names = [p.name for p in actual_paths]
+    expected_names = [
         (exe + ".exe" if sys.platform.startswith("win") else exe)
         for exe in [
             "crt",
@@ -175,22 +212,45 @@ def test_get_modflow(tmpdir):
         ]
     ]
 
-    assert all(exe in actual for exe in expected)
+    assert all(name in actual_names for name in expected_names)
+    assert_exts(actual_paths)
 
 
 @flaky
 @requires_github
-def test_get_nightly(tmpdir):
+def test_python_api_modflow6_nightly_build(tmpdir, downloads_dir):
     try:
         get_modflow(tmpdir, repo="modflow6-nightly-build")
     except urllib.error.HTTPError as err:
         if err.code == 403:
             pytest.skip(f"GitHub {rate_limit_msg}")
 
-    actual = [p.name for p in tmpdir.glob("*")]
-    expected = [
+    actual_paths = list(tmpdir.glob("*"))
+    actual_names = [p.name for p in actual_paths]
+    expected_names = [
         (exe + ".exe" if sys.platform.startswith("win") else exe)
         for exe in ["mf6", "mf5to6", "zbud6"]
     ]
 
-    assert all(exe in actual for exe in expected)
+    assert all(name in actual_names for name in expected_names)
+    assert_exts(actual_paths)
+
+
+@flaky
+@requires_github
+def test_python_api_modflow6(tmpdir, downloads_dir):
+    try:
+        get_modflow(tmpdir, repo="modflow6", downloads_dir=downloads_dir)
+    except urllib.error.HTTPError as err:
+        if err.code == 403:
+            pytest.skip(f"GitHub {rate_limit_msg}")
+
+    downloads = [p.name for p in downloads_dir.glob("*")]
+    assert len(downloads) > 0
+    assert any(dl.endswith("zip") for dl in downloads)
+
+    actual_paths = list(tmpdir.glob("*"))
+    actual_stems = [p.stem for p in actual_paths]
+    expected_stems = ["mf6", "mf5to6", "zbud6", "libmf6"]
+    assert all(exe in actual_stems for exe in expected_stems)
+    assert_exts(actual_paths)

docs/get_modflow.md

@@ -1,12 +1,25 @@
 # Install MODFLOW and related programs
 
-This method describes how to install USGS MODFLOW and related programs for Windows, Mac or Linux using a "get modflow" utility. If FloPy is installed, the utility is available in the Python environment as a `get-modflow` command. The same utility is also available as a Python script `get_modflow.py`, described later.
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
-The utility uses a [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives of programs that have been compiled with modern Intel Fortran compilers. The utility is able to match the binary archive to the operating system, and extract the console programs to a user-defined directory. A prompt can also be used to assist where to install programs.
+
+- [Command-line interface](#command-line-interface)
+  - [Using the `get-modflow` command](#using-the-get-modflow-command)
+  - [Using `get_modflow.py` as a script](#using-get_modflowpy-as-a-script)
+- [FloPy module](#flopy-module)
+- [Where to install?](#where-to-install)
+- [Selecting a distribution](#selecting-a-distribution)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+FloPy includes a `get-modflow` utility to install USGS MODFLOW and related programs for Windows, Mac or Linux. If FloPy is installed, the utility is available in the Python environment as a `get-modflow` command. The script `flopy/utils/get_modflow.py` has no dependencies and can be invoked independently.
+
+The utility uses the [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives containing executables compiled with [Intel Fortran](https://www.intel.com/content/www/us/en/developer/tools/oneapi/fortran-compiler.html). The utility is able to match the binary archive to the operating system and extract the console programs to a user-defined directory. A prompt can also be used to help the user choose where to install programs.
 
 ## Command-line interface
 
-### Using `get-modflow` from FloPy
+### Using the `get-modflow` command
 
 When FloPy is installed, a `get-modflow` (or `get-modflow.exe` for Windows) program is installed, which is usually installed to the PATH (depending on the Python setup). From a console:
 
@@ -57,3 +70,20 @@ Other auto-select options are only available if the current user can write files
  - `:local` - use `$HOME/.local/bin`
  - `:system` - use `/usr/local/bin`
  - `:windowsapps` - use `%LOCALAPPDATA%\Microsoft\WindowsApps`
+
+## Selecting a distribution
+
+By default the distribution from the [executables repository](https://github.com/MODFLOW-USGS/executables) is installed. This includes the MODFLOW 6 binary `mf6` and over 20 other related programs. The utility can also install from the main [MODFLOW 6 repo](https://github.com/MODFLOW-USGS/modflow6) or the [nightly build](https://github.com/MODFLOW-USGS/modflow6-nightly-build). To select a distribution, specify a repository name with the `--repo` command line option or the `repo` function argument. Valid names are:
+
+- `executables` (default)
+- `modflow6`
+- `modflow6-nightly-build`
+
+The `modflow6-nightly-build` distribution contains only:
+
+- `mf6`
+- `mf5to6`
+- `zbud6`
+- `libmf6.dylib`
+
+The `modflow6` release archive contains the entire repository with an internal `bin` directory, whose contents are as above. These are copied to the selected `bindir` after the archive is unzipped in the download location.

flopy/utils/get_modflow.py

@@ -23,6 +23,7 @@
 owner = "MODFLOW-USGS"
 # key is the repo name, value is the renamed file prefix for the download
 renamed_prefix = {
+    "modflow6": "modflow6",
     "executables": "modflow_executables",
     "modflow6-nightly-build": "modflow6_nightly",
 }
@@ -97,6 +98,8 @@ def columns_str(items, line_chars=79):
     """Return str of columns of items, similar to 'ls' command."""
     item_chars = max(len(item) for item in items)
     num_cols = line_chars // item_chars
+    if num_cols == 0:
+        num_cols = 1
     num_rows = len(items) // num_cols
     if len(items) % num_cols != 0:
         num_rows += 1
@@ -129,8 +132,8 @@ def run_main(
         colon character. See error message or other documentation for further
         information on auto-select options.
     repo : str, default "executables"
-        Name of GitHub repository. Choose one of "executables" (default) or
-        "modflow6-nightly-build".
+        Name of GitHub repository. Choose one of "executables" (default),
+        "modflow6", or "modflow6-nightly-build".
     release_id : str, default "latest"
         GitHub release ID.
     ostag : str, optional
@@ -190,6 +193,7 @@ def run_main(
 
     if ostag is None:
         ostag = get_ostag()
+
     exe_suffix = ""
     if ostag in ["win32", "win64"]:
         exe_suffix = ".exe"
@@ -340,18 +344,29 @@ def run_main(
         print(f"fetched release {tag_name!r} from {owner}/{repo}")
 
     assets = release.get("assets", [])
-    for asset in assets:
-        if ostag in asset["name"]:
-            break
+
+    # Windows 64-bit asset in modflow6 repo release has no OS tag
+    if repo == "modflow6" and ostag == "win64":
+        asset = list(sorted(assets, key=lambda a: len(a["name"])))[0]
     else:
-        raise ValueError(
-            f"could not find ostag {ostag!r} from release {tag_name!r}; "
-            f"see available assets here:\n{release['html_url']}"
-        )
+        for asset in assets:
+            if ostag in asset["name"]:
+                break
+        else:
+            raise ValueError(
+                f"could not find ostag {ostag!r} from release {tag_name!r}; "
+                f"see available assets here:\n{release['html_url']}"
+            )
     asset_name = asset["name"]
     download_url = asset["browser_download_url"]
-    # change local download name so it is more unique
-    dst_fname = "-".join([renamed_prefix[repo], tag_name, asset_name])
+    if repo == "modflow6":
+        asset_pth = Path(asset_name)
+        asset_stem = asset_pth.stem
+        asset_suffix = asset_pth.suffix
+        dst_fname = "-".join([repo, tag_name, ostag]) + asset_suffix
+    else:
+        # change local download name so it is more unique
+        dst_fname = "-".join([renamed_prefix[repo], tag_name, asset_name])
     tmpdir = None
     if downloads_dir is None:
         downloads_dir = Path.home() / "Downloads"
@@ -390,6 +405,7 @@ def run_main(
     extract = set()
     chmod = set()
     items = []
+    full_path = {}
     if meta_path:
         from datetime import datetime
 
@@ -405,7 +421,17 @@ def run_main(
         if subset:
             meta["subset"] = sorted(subset)
     with zipfile.ZipFile(download_pth, "r") as zipf:
-        files = set(zipf.namelist())
+        if repo == "modflow6":
+            # modflow6 release contains the whole repo with an internal bindir
+            inner_bin = asset_stem + "/bin"
+            for pth in zipf.namelist():
+                if pth.startswith(inner_bin) and not pth.endswith("bin/"):
+                    full_path[Path(pth).name] = pth
+            files = set(full_path.keys())
+        else:
+            # assume all files to be extracted
+            files = set(zipf.namelist())
+
         code = False
         if "code.json" in files:
             # don't extract this file
@@ -466,10 +492,15 @@ def add_item(key, fname, do_chmod):
                         )
                     ):
                         add_item(key, fname, do_chmod=True)
-        else:  # release 1.0 did not have code.json
+
+        else:
+            # releases without code.json
             for fname in sorted(files):
                 if nosub or (subset and fname in subset):
-                    extract.add(fname)
+                    if full_path:
+                        extract.add(full_path[fname])
+                    else:
+                        extract.add(fname)
                     items.append(fname)
                     if not fname.endswith(lib_suffix):
                         chmod.add(fname)
@@ -478,11 +509,30 @@ def add_item(key, fname, do_chmod):
                 f"extracting {len(extract)} "
                 f"file{'s' if len(extract) != 1 else ''} to '{bindir}'"
             )
+
         zipf.extractall(bindir, members=extract)
 
     # If this is a TemporaryDirectory, then delete the directory and files
     del tmpdir
 
+    if full_path:
+        # move files that used a full path to bindir
+        rmdirs = set()
+        for fpath in extract:
+            fpath = Path(fpath)
+            bindir_path = bindir / fpath
+            bindir_path.rename(bindir / fpath.name)
+            rmdirs.add(fpath.parent)
+        # clean up directories, starting with the longest
+        for rmdir in reversed(sorted(rmdirs)):
+            bindir_path = bindir / rmdir
+            bindir_path.rmdir()
+            for subdir in rmdir.parents:
+                bindir_path = bindir / subdir
+                if bindir_path == bindir:
+                    break
+                bindir_path.rmdir()
+
     if ostag in ["linux", "mac"]:
         # similar to "chmod +x fname" for each executable
         for fname in chmod:
@@ -494,6 +544,8 @@ def add_item(key, fname, do_chmod):
         print(columns_str(items))
 
         if not subset:
+            if full_path:
+                extract = {Path(fpth).name for fpth in extract}
             unexpected = extract.difference(files)
             if unexpected:
                 print(f"unexpected remaining {len(unexpected)} files:")