Use RavenPy v0.11.0 (#461)

## Overview

Changes:

* Updated several dependencies to be in sync with `ravenpy` v0.11.0
(including `raven-hydro` v3.6)
* Updated pre-commit hooks
* Modified (workaround) some processes to reflect breaking changes in
`fiona` v1.9
   * places upper limit below v2.0
* Updated default `pytest-xdist` scheduler to use `worksteal`
* Added `MAIN_TESTDATA_BRANCH` as a modifiable env var in main CI
workflow

## Related Issue / Discussion

The changes in fiona are breaking to `python-rasterstats`. I needed to
hack a workaround for fixing the properties out. Much of this logic
should be removed when a new version is released. Zonal statistics
processes need to be rewritten from the ground up anyway.

Other changes stemming from the new fiona popped up in a few other
places as well. GIS function in RavenPy need to be torn apart and
re-written. Good motivation to continue work on RavenGIS.

## Additional Information

perrygeo/python-rasterstats#281

* **The testing branch of raven-testdata must be modified back to
`master` before merging** (see:
Ouranosinc/raven-testdata#34)
   * Done.

.github/workflows/main.yml

@@ -6,6 +6,9 @@ on:
       - master
   pull_request:
 
+env:
+  MAIN_TESTDATA_BRANCH: "master"
+
 jobs:
   black:
     name: Code linting
@@ -45,7 +48,7 @@ jobs:
       - name: Conda and mamba versions
         shell: bash -l {0}
         run: |
-            mamba --version
+          mamba --version
       - name: Install and test RavenWPS
         shell: bash -l {0}
         run: |

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
     -   id: pretty-format-json
         args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ]
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: python-check-blanket-noqa
       - id: rst-inline-touching-normal
@@ -31,17 +31,17 @@ repos:
     -   id: flake8
         args: [ '--config=setup.cfg' ]
 -   repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.1.0
     hooks:
     -   id: black
         args: [ '--target-version=py38' ]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
     -   id: isort
         args: [ '--profile=black', '--filter-files', '--py=38' ]
 -   repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.6.0
+    rev: 1.6.3
     hooks:
       - id: nbqa-pyupgrade
         args: [ "--py38-plus" ]
@@ -50,7 +50,7 @@ repos:
       - id: nbqa-isort
         args: [ "--settings-file=setup.cfg" ]
 -   repo: https://github.com/pycqa/pydocstyle
-    rev: 6.2.0
+    rev: 6.3.0
     hooks:
     -   id: pydocstyle
         args: [ '--convention=numpy', '--match="(?!test_).*\.py"' ]
@@ -60,7 +60,7 @@ repos:
       - id: blackdoc
         additional_dependencies: [ 'black==22.10.0' ]
 -   repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.28.0
+    rev: v1.29.0
     hooks:
       - id: yamllint
         args: [ '--config-file', '.yamllint.yaml' ]
@@ -69,7 +69,7 @@ repos:
     hooks:
       - id: check-manifest
 -   repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.19.2
+    rev: 0.21.0
     hooks:
       - id: check-github-workflows
       - id: check-readthedocs

Makefile

@@ -11,6 +11,7 @@ WPS_URL := http://0.0.0.0:$(WPS_PORT)
 # If WPS_URL is overridden, this should also be overridden to match.
 WPS_OUTPUT_URL := http://localhost:$(WPS_PORT)/outputs
 
+# This will only work on Linux (not macOS/homebrew GDAL)
 GDAL_VERSION := $(shell gdal-config --version)
 
 # Used in target refresh-notebooks to make it looks like the notebooks have

docs/source/notebooks/03_Extracting_forcing_data.ipynb

@@ -89,7 +89,6 @@
     "\"\"\"\n",
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    ERA5_reference = subset.subset_shape(\n",
     "        ds.sel(time=slice(reference_start_day, reference_stop_day)), basin_contour\n",
     "    )\n",

docs/source/notebooks/08_Getting_and_bias_correcting_CMIP6_data.ipynb

@@ -3443,11 +3443,9 @@
    "source": [
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    # Load the files from the PanGEO catalogs, for reference and future variables of temperature and precipitation.\n",
     "    out = {}\n",
     "    for exp in [\"historical\", \"ssp585\"]:\n",
-    "\n",
     "        if exp == \"historical\":\n",
     "            period_start = reference_start_day\n",
     "            period_end = reference_end_day\n",
@@ -3524,7 +3522,6 @@
     "\"\"\"\n",
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    ERA5_reference = subset.subset_shape(\n",
     "        ds.sel(time=slice(reference_start_day, reference_end_day)), basin_contour\n",
     "    ).mean({\"latitude\", \"longitude\"})\n",

docs/source/notebooks/11_Climatological_ESP_forecasting.ipynb

@@ -578,6 +578,7 @@
     "# Once we have the correctly formatted datasets, Make the hindcast object for climpred\n",
     "hindcast_object = forecasting.make_climpred_hindcast_object(hindcasts, q_obs)\n",
     "\n",
+    "\n",
     "# This function is used to convert to binary to see if yes/no forecast is larger than observations\n",
     "def pos(x):\n",
     "    return x > 0  # Check for binary outcome\n",

docs/source/notebooks/paper/Get_ERA5_data_and_Calibrate_a_hydrological_model.ipynb

@@ -468,7 +468,6 @@
     "\"\"\"\n",
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    ERA5_reference = subset.subset_shape(\n",
     "        ds.sel(time=slice(reference_start_day, reference_stop_day)), basin_contour\n",
     "    )\n",

docs/source/notebooks/paper/Perform_a_climate_change_impact_assessment.ipynb

@@ -120,11 +120,9 @@
     "\n",
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    # Load the files from the PanGEO catalogs, for reference and future variables of temperature and precipitation.\n",
     "    out = {}\n",
     "    for exp in [\"historical\", \"ssp585\"]:\n",
-    "\n",
     "        if exp == \"historical\":\n",
     "            period_start = reference_start_day\n",
     "            period_end = reference_end_day\n",
@@ -201,7 +199,6 @@
     "\"\"\"\n",
     "# We will add a wrapper to ensure that the following operations will preserve the original data attributes, such as units and variable names.\n",
     "with xr.set_options(keep_attrs=True):\n",
-    "\n",
     "    ERA5_reference = subset.subset_shape(\n",
     "        ds.sel(time=slice(reference_start_day, reference_end_day)), basin_contour\n",
     "    ).mean({\"latitude\", \"longitude\"})\n",

environment-dev.yml

@@ -20,7 +20,7 @@ dependencies:
   - pytest-dependency
   - pytest-flake8
   - pytest-notebook
-  - pytest-xdist
+  - pytest-xdist >=3.2
   - sphinx >=1.7
   - sphinx-autoapi
   - twine

environment-docs.yml

@@ -5,7 +5,9 @@ channels:
   - defaults
 dependencies:
   - python >=3.8
-  - ravenpy >=0.9.0
+  - raven-hydro ==3.6
+  - ostrich ==21.03.16
+  - ravenpy >=0.11.0
   - pywps ==4.5.1
   - sphinx
   - ipython
@@ -14,5 +16,5 @@ dependencies:
   - numpy
   - pip
   - pypandoc
-  - sphinx_rtd_theme
+  - sphinx_rtd_theme >=1.0
   - xskillscore

environment.yml

@@ -3,16 +3,16 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - raven-hydro ==3.0.4.322
-  - ostrich ==21.03.16
   - python >=3.8
-  - ravenpy >=0.9.0
+  - raven-hydro ==3.6
+  - ostrich ==21.03.16
+  - ravenpy >=0.11.0
   - pywps ==4.5.1
   - affine
   - cartopy
   - click
   - dask
-  - fiona
+  - fiona >=1.9,<2.0
   - geopandas
   - geojson
   - gdal >=3.0
@@ -28,7 +28,7 @@ dependencies:
   - psutil
   - psycopg2
   - pymetalink
-  - pyproj
+  - pyproj >=3.4
   - pysheds
   - rasterio
   - rasterstats

raven/hpc_interface/HPC_demo.ipynb

@@ -478,10 +478,8 @@
     "job_finished = False\n",
     "abnormal_ending = False\n",
     "while not job_finished:\n",
-    "\n",
     "    time.sleep(60)\n",
     "    try:\n",
-    "\n",
     "        out, p = raven_proc.monitor()\n",
     "        # print(out)\n",
     "        wtext.value += out + \"<p>\"\n",

raven/hpc_interface/hpc_connection.py

@@ -26,7 +26,6 @@ def rand_fname(length=8):
 
 class HPCConnection:
     def __init__(self, external_init_dict=None):
-
         self.logger = logging.getLogger(constants.logging_name)
         init_dict = {}
         clsname = self.__class__.__name__
@@ -54,7 +53,6 @@ def __init__(self, external_init_dict=None):
         self.live_job_id = None
 
     def check_connection(self):
-
         status = True
         msg = None
         self.logger.debug("Testing connection...")
@@ -147,7 +145,6 @@ def copy_data_to_remote(self, dataset_, remote_temp_folder=None):
 
     # output files in base_dir/jobname/out
     def copy_data_from_remote(self, jobid, absolute_local_out_dir, cleanup_temp=True):
-
         self.logger.debug("Copying data from remote")
         absolute_tar_fname = os.path.join(
             self.remote_abs_working_folder, self.remote_working_folder + "_out.tar"
@@ -241,7 +238,6 @@ def copy_data_from_remote(self, jobid, absolute_local_out_dir, cleanup_temp=True
     def copy_singlefile_to_remote(
         self, local_filename, remote_path=".", is_executable=False
     ):
-
         r = os.path.join(
             self.remote_abs_working_folder,
             remote_path,
@@ -253,7 +249,6 @@ def copy_singlefile_to_remote(
             self.client.run_command("chmod ugo+x " + r)
 
     def create_remote_subdir(self, remote_subdir):
-
         self.client.run_command(
             "mkdir -p " + os.path.join(self.remote_abs_working_folder, remote_subdir)
         )
@@ -271,7 +266,6 @@ def copy_batchscript(
         batch_tmplt_fname,
         shub_hostname,
     ):
-
         template_file = open(os.path.join(self.template_path, batch_tmplt_fname))
         abs_remote_output_dir = os.path.join(self.remote_abs_working_folder, "out")
         tmplt = template_file.read()
@@ -306,7 +300,6 @@ def copy_batchscript(
         return os.path.join(self.remote_abs_working_folder, subst_fname)
 
     def submit_job(self, script_fname):
-
         self.logger.debug(f"Submitting job {script_fname}")
         # output = self.client.run_command("cd {}; ".format(self.home_dir) + constants.sbatch_cmd +
         #                                  " --parsable " + script_fname)
@@ -331,7 +324,6 @@ def submit_job(self, script_fname):
         return self.live_job_id
 
     def read_from_remote(self, remote_filename):
-
         filecontent = []
         self.logger.debug("read_from_remote")
         retry = True
@@ -357,7 +349,6 @@ def read_from_remote(self, remote_filename):
             #            print("SFTPIOError")
             #            return False
             except Exception as e:
-
                 if retry:
                     self.logger.debug(
                         f"exception {e}, retrying"
@@ -435,7 +426,6 @@ def cancel_job(self, jobid):
             raise Exception("Cancel error: " + stdout_str)
 
     def reconnect(self):
-
         self.client = ParallelSSHClient(
             [self.hostname], pkey=self.keypath, user=self.user, keepalive_seconds=300
         )
@@ -471,7 +461,6 @@ def check_slurmoutput_for(self, substr, jobid):
     """
 
     def cleanup(self, jobid):
-
         try:
             self.logger.debug("Deleting the remote folder")
             output1 = self.client.run_command(

raven/hpc_interface/main.py

@@ -23,7 +23,6 @@ def newmainfct(argv):
     try:
         opts, args = getopt.getopt(argv, "e:d:i:o:t:n")
     except getopt.GetoptError:
-
         print(
             "main.py -e executable (raven|ostrich) -d workingdir -i dataset -o outdir -t template_dir -n (nocleanup)"
         )
@@ -77,10 +76,8 @@ def newmainfct(argv):
     job_finished = False
     abnormal_ending = False
     while not job_finished:
-
         time.sleep(60)
         try:
-
             out, p = raven_proc.monitor()
             print(out)
             if out == "RUNNING":

raven/hpc_interface/raven_process.py

@@ -105,12 +105,10 @@ def job_ended_normally(self):
     """
 
     def cancel(self):
-
         self.hpc_connection.cancel_job(self.live_job_id)
         # self.cleanup()
 
     def monitor(self):
-
         # job_status, progressfilecontent
         progressfile = None
         if self.process_name == "raven":
@@ -121,17 +119,13 @@ def monitor(self):
         reconnect = False
         while True:
             try:
-
                 s = self.hpc_connection.get_status(self.live_job_id)
                 if s == "RUNNING":
-
                     if progressfile is not None:
-
                         progressfile_content = self.hpc_connection.read_from_remote(
                             progressfile
                         )
                         for line in progressfile_content:
-
                             match_obj = re.search(
                                 r"progress\": (\d*)", line, re.M | re.I
                             )

raven/processes/base_xclim.py

@@ -160,7 +160,6 @@ def write_log(self, message):
         LOGGER.info(message)
 
     def _handler(self, request, response):
-
         response.outputs["output_log"].file = self.log_file_path()
         self.write_log("Processing started")
 

raven/processes/wps_climpred_forecast_verification.py

@@ -7,7 +7,6 @@
 
 class ClimpredHindcastVerificationProcess(Process):
     def __init__(self):
-
         hindcasts = ComplexInput(
             "hindcasts",
             "3-dimensional xarray dataset / netcdf with hindcasts",
@@ -62,7 +61,6 @@ def __init__(self):
         )
 
     def _handler(self, request, response):
-
         hindcasts = xr.open_dataset(request.inputs["hindcasts"][0].file)
         qobs = xr.open_dataset(request.inputs["observations"][0].file)
         metric = request.inputs["metric"][0].data

raven/processes/wps_forecast_evaluation.py

@@ -111,7 +111,6 @@ def __init__(self):
         )
 
     def _handler(self, request, response):
-
         # Read inputs from request
         obs_fn = request.inputs["obs"][0].file
         obs_var = request.inputs["obs_var"][0].data
@@ -145,7 +144,6 @@ def _handler(self, request, response):
                 m = func(obs, hcst, dim="time", skipna=skipna)
 
             elif "member" in hcst.dims:
-
                 if metric == "threshold_brier_score":
                     m = func(obs, hcst, threshold=bss_threshold, dim="time")