diff --git a/.docs/introduction.rst b/.docs/introduction.rst index 7c644234c..52175603f 100644 --- a/.docs/introduction.rst +++ b/.docs/introduction.rst @@ -58,6 +58,14 @@ To install the bleeding edge version of FloPy from the git repository type: pip install git+https://github.com/modflowpy/flopy.git +After FloPy is installed, MODFLOW and related programs can be installed using the command: + +.. code-block:: bash + + get-modflow :flopy + +See documentation `get_modflow.md `_ +for more information. FloPy Resources diff --git a/README.md b/README.md index e2ec673e9..eaa87b74e 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,12 @@ or The release candidate version can also be installed from the git repository using the instructions provided [below](#relcand). +After FloPy is installed, MODFLOW and related programs can be installed using the command: + + get-modflow :flopy + +See documentation [get_modflow.md](https://github.com/modflowpy/flopy/blob/develop/docs/get_modflow.md) for more information. + Documentation ----------------------------------------------- diff --git a/autotest/test_scripts.py b/autotest/test_get_modflow.py similarity index 97% rename from autotest/test_scripts.py rename to autotest/test_get_modflow.py index 2997f17f9..dfc8d8ef5 100644 --- a/autotest/test_scripts.py +++ b/autotest/test_get_modflow.py @@ -1,4 +1,4 @@ -"""Test scripts.""" +"""Test get-modflow utility.""" import sys import urllib from urllib.error import HTTPError @@ -11,7 +11,7 @@ ) from flaky import flaky -from flopy.utils import get_modflow_main +from flopy.utils import get_modflow flopy_dir = get_project_root_path(__file__) get_modflow_script = flopy_dir / "flopy" / "utils" / "get_modflow.py" @@ -139,7 +139,7 @@ def test_get_nightly_script(tmp_path, downloads_dir): @requires_github def test_get_modflow(tmpdir): try: - get_modflow_main(tmpdir) + get_modflow(tmpdir) except HTTPError as err: if err.code == 403: pytest.skip(f"GitHub {rate_limit_msg}") @@ -182,7 +182,7 @@ def test_get_modflow(tmpdir): @requires_github def test_get_nightly(tmpdir): try: - get_modflow_main(tmpdir, repo="modflow6-nightly-build") + get_modflow(tmpdir, repo="modflow6-nightly-build") except urllib.error.HTTPError as err: if err.code == 403: pytest.skip(f"GitHub {rate_limit_msg}") diff --git a/docs/get_modflow.md b/docs/get_modflow.md index 343328587..4b5dbd8a5 100644 --- a/docs/get_modflow.md +++ b/docs/get_modflow.md @@ -36,7 +36,24 @@ from pathlib import Path import flopy bindir = Path("/tmp/bin") -bindir.mkdir() -flopy.utils.get_modflow_main(bindir) +bindir.mkdir(exist_ok=True) +flopy.utils.get_modflow(bindir) list(bindir.iterdir()) + +# Or use an auto-select option +flopy.utils.get_modflow(":flopy") ``` + +## Where to install? + +A required `bindir` parameter must be supplied to the utility, which specifies where to install the programs. This can be any existing directory, usually which is on the users' PATH environment variable. + +To assist the user, special values can be specified starting with the colon character. Use a single `:` to interactively select an option of paths. + +Other auto-select options are only available if the current user can write files (some may require `sudo` for Linux or macOS): + - `:prev` - if this utility was run by FloPy more than once, the first option will be the previously used `bindir` path selection + - `:flopy` - special option that will create and install programs for FloPy + - `:python` - use Python's bin (or Scripts) directory + - `:local` - use `$HOME/.local/bin` + - `:system` - use `/usr/local/bin` + - `:windowsapps` - use `%LOCALAPPDATA%\Microsoft\WindowsApps` diff --git a/flopy/mbase.py b/flopy/mbase.py index 04108d710..f4807d590 100644 --- a/flopy/mbase.py +++ b/flopy/mbase.py @@ -9,6 +9,7 @@ import os import queue as Queue import shutil +import sys import threading import warnings from datetime import datetime @@ -21,6 +22,13 @@ from .discretization.grid import Grid from .version import __version__ +# Prepend flopy appdir bin directory to PATH to work with "get-modflow :flopy" +if sys.platform.startswith("win"): + flopy_bin = os.path.expandvars(r"%LOCALAPPDATA%\flopy\bin") +else: + flopy_bin = os.path.join(os.path.expanduser("~"), ".local/share/flopy/bin") +os.environ["PATH"] = flopy_bin + os.path.pathsep + os.environ.get("PATH", "") + ## Global variables # Multiplier for individual array elements in integer and real arrays read by # MODFLOW's U2DREL, U1DREL and U2DINT. diff --git a/flopy/utils/__init__.py b/flopy/utils/__init__.py index ef38db4b1..ea5e9d933 100644 --- a/flopy/utils/__init__.py +++ b/flopy/utils/__init__.py @@ -21,6 +21,7 @@ """ from .utl_import import import_optional_dependency # isort:skip +from . import get_modflow as get_modflow_module from .binaryfile import ( BinaryHeader, CellBudgetFile, @@ -31,7 +32,8 @@ from .check import check from .flopy_io import read_fixed_var, write_fixed_var from .formattedfile import FormattedHeadFile -from .get_modflow import run_main as get_modflow_main + +get_modflow = get_modflow_module.run_main from .gridintersect import GridIntersect, ModflowGridIndices from .mflistfile import ( Mf6ListBudget, diff --git a/flopy/utils/get_modflow.py b/flopy/utils/get_modflow.py index db8ea3630..a66dcb137 100755 --- a/flopy/utils/get_modflow.py +++ b/flopy/utils/get_modflow.py @@ -14,6 +14,7 @@ import urllib import urllib.request import zipfile +from importlib.util import find_spec from pathlib import Path __all__ = ["run_main"] @@ -27,6 +28,16 @@ } available_repos = list(renamed_prefix.keys()) available_ostags = ["linux", "mac", "win32", "win64"] +max_http_tries = 3 + +# Check if this is running from flopy +within_flopy = False +spec = find_spec("flopy") +if spec is not None: + within_flopy = ( + Path(spec.origin).resolve().parent in Path(__file__).resolve().parents + ) +del spec def get_ostag(): @@ -68,19 +79,13 @@ def get_avail_releases(api_url): raise ValueError("GITHUB_TOKEN env is invalid") from err elif err.code == 403 and "rate limit exceeded" in err.reason: raise ValueError( - "use GITHUB_TOKEN env to bypass rate limit" + f"use GITHUB_TOKEN env to bypass rate limit ({err})" ) from err - elif err.code == 404: - if num_tries < 3: - # GitHub sometimes returns 404 for valid URLs, so retry - print(f"URL request {num_tries} did not work") - continue - else: - raise RuntimeError( - f"cannot retrieve data from {req_url}" - ) from err - else: - raise err + elif err.code in (404, 503) and num_tries < max_http_tries: + # GitHub sometimes returns this error for valid URLs, so retry + print(f"URL request {num_tries} did not work ({err})") + continue + raise RuntimeError(f"cannot retrieve data from {req_url}") from err releases = json.loads(result.decode()) avail_releases = ["latest"] @@ -120,7 +125,9 @@ def run_main( Parameters ---------- bindir : str or Path - Writable path to extract executables. + Writable path to extract executables. Auto-select options start with a + colon character. See error message or other documentation for further + information on auto-select options. repo : str, default "executables" Name of GitHub repository. Choose one of "executables" (default) or "modflow6-nightly-build". @@ -128,8 +135,9 @@ def run_main( GitHub release ID. ostag : str, optional Operating system tag; default is to automatically choose. - subset : str, optional - Optional subset of executables to extract, e.g. "mfnwt,mp6" + subset : list, set or str, optional + Optional subset of executables to extract, specified as a list (e.g.) + ``["mfnwt", "mp6"]`` or a comma-separated string "mfnwt,mp6". downloads_dir : str or Path, optional Manually specify directory to download archives. Default is to use home Downloads, if available, otherwise a temporary directory. @@ -142,6 +150,44 @@ def run_main( Control behavior of method if this is run as a command-line interface or as a Python function. """ + meta_path = False + prev_bindir = None + flopy_bin = False + if within_flopy: + meta_list = [] + # Store metadata and possibly 'bin' in a user-writable path + if sys.platform.startswith("win"): + flopy_appdata = Path(os.path.expandvars(r"%LOCALAPPDATA%\flopy")) + else: + flopy_appdata = Path.home() / ".local" / "share" / "flopy" + if not flopy_appdata.exists(): + flopy_appdata.mkdir(parents=True, exist_ok=True) + flopy_bin = flopy_appdata / "bin" + meta_path = flopy_appdata / "get_modflow.json" + meta_path_exists = meta_path.exists() + if meta_path_exists: + del_meta_path = False + try: + meta_list = json.loads(meta_path.read_text()) + except (OSError, json.JSONDecodeError) as err: + print(f"cannot read flopy metadata file '{meta_path}': {err}") + if isinstance(err, OSError): + meta_path = False + if isinstance(err, json.JSONDecodeError): + del_meta_path = True + try: + prev_bindir = Path(meta_list[-1]["bindir"]) + except (KeyError, IndexError): + del_meta_path = True + if del_meta_path: + try: + meta_path.unlink() + meta_path_exists = False + print(f"removed corrupt flopy metadata file '{meta_path}'") + except OSError as err: + print(f"cannot remove flopy metadata file: {err!r}") + meta_path = False + if ostag is None: ostag = get_ostag() exe_suffix = "" @@ -157,50 +203,81 @@ def run_main( f"unrecognized ostag {ostag!r}; choose one of {available_ostags}" ) - if _is_cli and bindir == "?": - options = [] - # check if conda - conda_bin = ( - Path(sys.prefix) - / "conda-meta" - / ".." - / ("Scripts" if ostag.startswith("win") else "bin") + if isinstance(bindir, Path): + pass + elif bindir.startswith(":"): + options = {} # key is an option name, value is (optpath, optinfo) + if prev_bindir is not None and os.access(prev_bindir, os.W_OK): + # Make previous bindir as the first option + options[":prev"] = (prev_bindir, "previously selected bindir") + if within_flopy: # don't check is_dir() or access yet + options[":flopy"] = (flopy_bin, "used by FloPy") + # Python bin (same for standard or conda varieties) + py_bin = Path(sys.prefix) / ( + "Scripts" if ostag.startswith("win") else "bin" ) - if conda_bin.exists() and os.access(conda_bin, os.W_OK): - options.append(conda_bin.resolve()) + if py_bin.is_dir() and os.access(py_bin, os.W_OK): + options[":python"] = (py_bin, "used by Python") home_local_bin = Path.home() / ".local" / "bin" if home_local_bin.is_dir() and os.access(home_local_bin, os.W_OK): - options.append(home_local_bin) + options[":home"] = (home_local_bin, "user-specific bindir") local_bin = Path("/usr") / "local" / "bin" if local_bin.is_dir() and os.access(local_bin, os.W_OK): - options.append(local_bin) + options[":system"] = (local_bin, "system local bindir") # Windows user windowsapps_dir = Path( os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\WindowsApps") ) if windowsapps_dir.is_dir() and os.access(windowsapps_dir, os.W_OK): + options[":windowsapps"] = (windowsapps_dir, "User App path") options.append(windowsapps_dir) - # any other possible locations? + # any other possible OS-specific hard-coded locations? if not options: raise RuntimeError("could not find any installable folders") - print("select a directory to extract executables:") - options_d = dict(enumerate(options, 1)) - for iopt, opt in options_d.items(): - print(f"{iopt:2d}: {opt}") - num_tries = 0 - while True: - num_tries += 1 - res = input("> ") - try: - bindir = options_d[int(res)] - break - except (KeyError, ValueError): - if num_tries < 3: - print("invalid option, try choosing option again") - else: - raise RuntimeError("invalid option, too many attempts") + opt_avail = ", ".join( + f"'{opt}' for '{optpath}'" for opt, (optpath, _) in options.items() + ) + if len(bindir) > 1: # auto-select mode + # match one option that starts with input, e.g. :Py -> :python + sel = list( + opt for opt in options if opt.startswith(bindir.lower()) + ) + if len(sel) != 1: + if bindir == ":flopy": + raise ValueError("option ':flopy' is only for flopy") + raise ValueError(f"invalid option, choose from: {opt_avail}") + bindir = options[sel[0]][0] + if not quiet: + print(f"auto-selecting option {sel[0]!r} for '{bindir}'") + elif not _is_cli: + raise ValueError(f"specify the option, choose from: {opt_avail}") + else: + ioptions = dict(enumerate(options.keys(), 1)) + print("select a number to extract executables to a directory:") + for iopt, opt in ioptions.items(): + optpath, optinfo = options[opt] + print(f" {iopt}: '{optpath}' -- {optinfo} ('{opt}')") + num_tries = 0 + while True: + num_tries += 1 + res = input("> ") + try: + opt = ioptions[int(res)] + print(f"selecting option {opt!r}") + bindir = options[opt][0] + break + except (KeyError, ValueError): + if num_tries < 2: + print("invalid option, try choosing option again") + else: + raise RuntimeError( + "invalid option, too many attempts" + ) from None bindir = Path(bindir).resolve() + if bindir == flopy_bin and not flopy_bin.exists(): + # special case option that can create non-existing directory + flopy_bin.mkdir(parents=True, exist_ok=True) if not bindir.is_dir(): raise OSError(f"extraction directory '{bindir}' does not exist") elif not os.access(bindir, os.W_OK): @@ -236,27 +313,26 @@ def run_main( raise ValueError("GITHUB_TOKEN env is invalid") from err elif err.code == 403 and "rate limit exceeded" in err.reason: raise ValueError( - "use GITHUB_TOKEN env to bypass rate limit" + f"use GITHUB_TOKEN env to bypass rate limit ({err})" ) from err elif err.code == 404: if avail_releases is None: avail_releases = get_avail_releases(api_url) if release_id in avail_releases: - if num_tries < 3: + if num_tries < max_http_tries: # GitHub sometimes returns 404 for valid URLs, so retry - print(f"URL request {num_tries} did not work") + print(f"URL request {num_tries} did not work ({err})") continue - else: - raise RuntimeError( - f"cannot retrieve data from {req_url}" - ) from err else: raise ValueError( f"Release {release_id!r} not found -- " f"choose from {avail_releases}" ) from err - else: - raise err + elif err.code == 503 and num_tries < max_http_tries: + # GitHub sometimes returns this error for valid URLs, so retry + print(f"URL request {num_tries} did not work ({err})") + continue + raise RuntimeError(f"cannot retrieve data from {req_url}") from err release = json.loads(result.decode()) tag_name = release["tag_name"] @@ -272,9 +348,10 @@ def run_main( f"could not find ostag {ostag!r} from release {tag_name!r}; " f"see available assets here:\n{release['html_url']}" ) + asset_name = asset["name"] download_url = asset["browser_download_url"] # change local download name so it is more unique - dst_fname = "-".join([renamed_prefix[repo], tag_name, asset["name"]]) + dst_fname = "-".join([renamed_prefix[repo], tag_name, asset_name]) tmpdir = None if downloads_dir is None: downloads_dir = Path.home() / "Downloads" @@ -303,18 +380,43 @@ def run_main( print(f"downloading to '{download_pth}'") urllib.request.urlretrieve(download_url, download_pth) + if subset: + if isinstance(subset, str): + subset = set(subset.replace(",", " ").split()) + elif not isinstance(subset, set): + subset = set(subset) + # Open archive and extract files extract = set() chmod = set() - items = list() + items = [] + if meta_path: + from datetime import datetime + + meta = { + "bindir": str(bindir), + "owner": owner, + "repo": repo, + "release_id": tag_name, + "name": asset_name, + "updated_at": asset["updated_at"], + "extracted_at": datetime.now().isoformat(), + } + if subset: + meta["subset"] = sorted(subset) with zipfile.ZipFile(download_pth, "r") as zipf: files = set(zipf.namelist()) - # print(f"{len(files)=}: {files}") code = False if "code.json" in files: # don't extract this file - code = json.loads(zipf.read("code.json").decode()) files.remove("code.json") + code_bytes = zipf.read("code.json") + code = json.loads(code_bytes.decode()) + if meta_path: + import hashlib + + code_md5 = hashlib.md5(code_bytes).hexdigest() + meta["code_json_md5"] = code_md5 if subset: nosub = False subset_keys = files @@ -382,6 +484,7 @@ def add_item(key, fname, do_chmod): del tmpdir if ostag in ["linux", "mac"]: + # similar to "chmod +x fname" for each executable for fname in chmod: pth = bindir / fname pth.chmod(pth.stat().st_mode | 0o111) @@ -396,53 +499,121 @@ def add_item(key, fname, do_chmod): print(f"unexpected remaining {len(unexpected)} files:") print(columns_str(sorted(unexpected))) + # Save metadata, only for flopy + if meta_path: + if "pytest" in str(bindir) or "pytest" in sys.modules: + # Don't write metadata if this is part of pytest + print("skipping writing flopy metadata for pytest") + return + meta_list.append(meta) + if not flopy_appdata.exists(): + flopy_appdata.mkdir(parents=True, exist_ok=True) + try: + meta_path.write_text(json.dumps(meta_list, indent=4) + "\n") + except OSError as err: + print(f"cannot write flopy metadata file: '{meta_path}': {err!r}") + if not quiet: + if meta_path_exists: + print(f"updated flopy metadata file: '{meta_path}'") + else: + print(f"wrote new flopy metadata file: '{meta_path}'") + def cli_main(): """Command-line interface.""" import argparse - parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - parser.add_argument( - "bindir", - help="directory to extract executables; use '?' to help choose", + # Show meaningful examples at bottom of help + prog = Path(sys.argv[0]).stem + if sys.platform.startswith("win"): + drv = Path("c:/") + else: + drv = Path("/") + example_bindir = drv / "path" / "to" / "bin" + examples = f"""\ +Examples: + + Install executables into an existing '{example_bindir}' directory: + $ {prog} {example_bindir} + + Install a development snapshot of MODFLOW 6 by choosing a repo: + $ {prog} --repo modflow6-nightly-build {example_bindir} + """ + if within_flopy: + examples += f"""\ + + FloPy users can install executables using a special option: + $ {prog} :flopy + """ + + parser = argparse.ArgumentParser( + description=__doc__.split("\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples, ) + + bindir_help = ( + "Directory to extract executables. Use ':' to interactively select an " + "option of paths. Other auto-select options are only available if the " + "current user can write files. " + ) + if within_flopy: + bindir_help += ( + "Option ':prev' is the previously used 'bindir' path selection. " + "Option ':flopy' will create and install programs for FloPy. " + ) + if sys.platform.startswith("win"): + bindir_help += ( + "Option ':python' is Python's Scripts directory. " + "Option ':windowsapps' is " + "'%%LOCALAPPDATA%%\\Microsoft\\WindowsApps'." + ) + else: + bindir_help += ( + "Option ':python' is Python's bin directory. " + "Option ':local' is '$HOME/.local/bin'. " + "Option ':system' is '/usr/local/bin'." + ) + parser.add_argument("bindir", help=bindir_help) parser.add_argument( "--repo", choices=available_repos, default="executables", - help="name of GitHub repository; default is 'executables'", + help="Name of GitHub repository; default is 'executables'.", ) parser.add_argument( "--release-id", default="latest", - help="GitHub release ID (default: latest)", + help="GitHub release ID; default is 'latest'.", ) parser.add_argument( "--ostag", choices=available_ostags, - help="operating system tag; default is to automatically choose", + help="Operating system tag; default is to automatically choose.", + ) + parser.add_argument( + "--subset", + help="Subset of executables to extract, specified as a " + "comma-separated string, e.g. 'mfnwt,mp6'.", ) - parser.add_argument("--subset", help="subset of executables") parser.add_argument( "--downloads-dir", - help="manually specify directory to download archives", + help="Manually specify directory to download archives.", ) parser.add_argument( - "--force", action="store_true", help="force re-download" + "--force", + action="store_true", + help="Force re-download archive. Default behavior will use archive if " + "previously downloaded in downloads-dir.", ) parser.add_argument( - "--quiet", action="store_true", help="show fewer messages" + "--quiet", action="store_true", help="Show fewer messages." ) args = vars(parser.parse_args()) - if args["subset"]: - args["subset"] = set(args["subset"].replace(",", " ").split()) - # print(args) try: run_main(**args, _is_cli=True) - except KeyboardInterrupt: - sys.exit(f" cancelling {sys.argv[0]}") - except (KeyError, OSError, RuntimeError, ValueError) as err: - sys.exit(err) + except (EOFError, KeyboardInterrupt): + sys.exit(f" cancelling '{sys.argv[0]}'") if __name__ == "__main__":