[1pt]: Remove FIM3 and most references to GMS

config/deny_gms_branch_unittests.lst → config/deny_branch_unittests.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_branch_zero.lst → config/deny_branch_zero.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branch zero to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_branches.lst → config/deny_branches.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_unit.lst → config/deny_unit.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for units to delete
 # Use comment to allow list the file
 #branch_id.lst
 #branch_polygons.gpkg

config/params_template.env

@@ -20,7 +20,7 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
-#### gms parameters ####
+#### branch parameters ####
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.

data/nws/preprocess_ahps_nws.py

@@ -14,6 +14,15 @@
 from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
+
 def get_env_paths():
     load_dotenv()
     #import variables from .env file
@@ -354,6 +363,15 @@ def preprocess_nws(source_dir, destination, reference_raster):
 
 
 if __name__ == '__main__':
+
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
+
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
     parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)

data/usgs/acquire_and_preprocess_3dep_dems.py

@@ -174,7 +174,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
 
                 sys.exit(1)
 
-        # Send the executor to the progress bar and wait for all FR tasks to finish
+        # Send the executor to the progress bar and wait for all tasks to finish
         sf.progress_bar_handler(executor_dict, f"Downloading USGG 3Dep Dems")
 
     print(f"-- Downloading USGS DEMs Completed")

docs/CHANGELOG.md

@@ -2,6 +2,199 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.2.0.0 - 2023-02-16 - [PR#816](https://github.com/NOAA-OWP/inundation-mapping/pull/816)
+
+This update removes the remaining elements of FIM3 code.  It further removes the phrases "GMS" as basically the entire FIM4 model. FIM4 is GMS. With removing FIM3, it also means remove concepts of "MS" and "FR" which were no longer relevant in FIM4.  There are only a few remaining places that will continue with the phrase "GMS" which is in some inundation files which are being re-evaluated.  Some deprecated files have been removed and some subfolders removed.
+
+There are a lot of duplicate explanations for some of the changes, so here is a shortcut system.
+
+- desc 1:  Remove or rename values based on phrase "GMS, MS and/or FR"
+- desc 2:  Moved file from the /src/gms folder to /src  or /tools/gms_tools to /tools
+- desc 3:  No longer needed as we now use the `fim_pipeline.sh` processing model.
+
+### Removals
+
+- `data`
+    - `acquire_and_preprocess_inputs.py`:  No longer needed
+- `gms_pipeline.sh` : see desc 3
+- `gms_run_branch.sh` : see desc 3
+- `gms_run_post_processing.sh` : see desc 3
+- `gms_run_unit.sh` : see desc 3
+- `src`
+    - `gms`
+        - `init.py` : folder removed, no longer needed.
+        - `aggregate_branch_lists.py`: no longer needed.  Newer version already exists in src directory.
+        - `remove_error_branches.py` :  see desc 3
+        - `run_by_unit.sh` : see desc 3
+        - `test_new_crosswalk.sh` : no longer needed
+        - `time_and_tee_run_by_branch.sh` : see desc 3
+        - `time_and_tee_run_by_unit.sh` : see desc 3
+    - `output_cleanup.py` : see desc 3
+ - `tools/gms_tools`
+     - `init.py` : folder removed, no longer needed.
+
+### Changes
+
+- `config`
+   - `deny_branch_unittests.lst` :  renamed from `deny_gms_branch_unittests.lst`
+   - `deny_branch_zero.lst` : renamed from `deny_gms_branch_zero.lst`
+   - `deny_branches.lst` :  renamed from `deny_gms_branches.lst`
+   - `deny_unit.lst`  : renamed from `deny_gms_unit.lst`
+   - `params_template.env` : see desc 1
+
+- `data`
+    - `nws`
+        - `preprocess_ahps_nws.py`:   Added deprecation note: If reused, it needs review and/or upgrades.
+    - `acquire_and_preprocess_3dep_dems.py` : see desc 1
+ - `fim_post_processing.sh` : see desc 1, plus a small pathing change.
+ - `fim_pre_processing.sh` : see desc 1
+ - ` src`
+     - `add_crosswalk.py` : see desc 1. Also cleaned up some formatting and commented out a code block in favor of a better way to pass args from "__main__"
+     - `bash_variables.env` : see desc 1
+     - `buffer_stream_branches.py` : see desc 2
+     - `clip_rasters_to_branches.py` : see desc 2
+     - `crosswalk_nwm_demDerived.py` :  see desc 1 and desc 2
+     - `delineate_hydros_and_produce_HAND.sh` : see desc 1 and desc 2
+     - `derive_level_paths.py`  :  see desc 1 and desc 2
+     - `edit_points.py` : see desc  2
+     - `filter_inputs_by_huc.py`: see desc 1 and desc 2
+     - `finalize_srcs.py`:  see desc 2
+     - `generate_branch_list.py` : see desc 1
+     - `make_rem.py` : see desc 2
+     - `make_dem.py` : see desc  2
+     - `outputs_cleanup.py`:  see desc 1
+     - `process_branch.sh`:  see desc 1
+     - `query_vectors_by_branch_polygons.py`: see desc 2
+     - `reset_mannings.py` : see desc 2
+     - `run_by_branch.sh`:  see desc 1
+     - `run_unit_wb.sh`: see desc 1
+     - `stream_branches.py`:  see desc 2
+     - `subset_catch_list_by_branch_id.py`: see desc 2
+     - `toDo.md`: see desc 2
+     - `usgs_gage_aggregate.py`:  see desc 1
+     - `usgs_gage_unit_setup.py` : see desc 1
+     - `utils`
+         - `fim_enums.py` : see desc 1
+
+- `tools`
+    - `combine_crosswalk_tables.py` : see desc 2
+    - `compare_ms_and_non_ms_metrics.py` : see desc 2
+    - `compile_comp_stats.py`: see desc 2  and added note about possible deprecation.
+    - `compile_computation_stats.py` : see desc 2  and added note about possible deprecation.
+    - `composite_inundation.py` : see desc 1 : note.. references a file called inundate_gms which retains it's name for now.
+    - `consolidate_metrics.py`: added note about possible deprecation.
+    - `copy_test_case_folders.py`: see desc 1
+    - `eval_plots.py` : see desc 1
+    - `evaluate_continuity.py`: see desc 2
+    - `find_max_catchment_breadth.py` : see desc 2
+    - `generate_categorical_fim_mapping.py` : see desc 1
+    - `inundate_gms.py`: see desc 1 and desc 2. Note: This file has retained its name with the phrase "gms" in it as it might be upgraded later and there are some similar files with similar names.
+    - `inundate_nation.py` : see desc 1
+    - `inundation.py`:  text styling change
+    - `make_boxes_from_bounds.py`: text styling change
+    - `mosaic_inundation.py`:  see desc 1 and desc 2
+    - `overlapping_inundation.py`: see desc 2
+    - `plots.py` : see desc 2
+    - `run_test_case.py`:  see desc 1
+    - `synthesize_test_cases.py`: see desc 1
+
+- `unit_tests`
+    - `README.md`: see desc 1
+    - `__template_unittests.py`: see desc 1
+    - `check_unit_errors_params.json`  and `check_unit_errors_unittests.py` : see desc 1
+    - `derive_level_paths_params.json` and `derive_level_paths_unittests.py` : see desc 1 and desc 2
+    - `filter_catchments_and_add_attributes_unittests.py`: see desc 1
+    - `outputs_cleanup_params.json` and `outputs_cleanup_unittests.py`: see desc 1 and desc 2
+    - `split_flows_unittests.py` : see desc 1
+    - `tools`
+        - `inundate_gms_params.json` and `inundate_gms_unittests.py`: see desc 1 and desc 2
+
+<br/><br/>
+## v4.1.3.0 - 2023-02-13 - [PR#812](https://github.com/NOAA-OWP/inundation-mapping/pull/812)
+
+An update was required to adjust host name when in the AWS environment
+
+### Changes
+
+- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or 
+
+- `tools/calibration-db`
+    - `README.md`: Minor text correction.
+
+<br/><br/>
+
+## v4.1.2.0 - 2023-02-15 - [PR#808](https://github.com/NOAA-OWP/inundation-mapping/pull/808)
+
+Add `pytest` package and refactor existing unit tests. Update parameters to unit tests (`/unit_tests/*_params.json`) to valid paths. Add leading slash to paths in `/config/params_template.env`.
+
+### Additions
+
+- `/unit_tests`
+  - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+  - `pyproject.toml`  - used to specify which warnings are excluded/filtered.
+  - `/gms`  
+    - `__init__.py` - needed for `pytest` command line executable to pick up tests.
+  - `/tools`
+    - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+    - `inundate_gms_params.json` - file moved up into this directory
+    - `inundate_gms_test.py`     - file moved up into this directory
+    - `inundation_params.json`   - file moved up into this directory
+    - `inundation_test.py`       - file moved up into this directory
+
+### Removals
+
+- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`    
+
+### Changes
+
+- `Pipfile` - updated to include pytest as a dependency
+- `Pipfile.lock` - updated to include pytest as a dependency
+
+- `/config`
+  - `params_template.env` - leading slash added to paths
+
+- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.  
+  - `*_params.json` - valid paths on `fim-dev1` provided
+  - `README.md`  - updated to include documentation on pytest.  
+  - `unit_tests_utils.py`  
+  - `__template_unittests.py` -> `__template.py` - exclude the `_test` suffix to remove from test suite. Updated example on new format for pytest.
+  - `check_unit_errors_test.py`  
+  - `clip_vectors_to_wbd_test.py`  
+  - `filter_catchments_and_add_attributes_test.py`  
+  - `rating_curve_comparison_test.py`  
+  - `shared_functions_test.py`  
+  - `split_flow_test.py`  
+  - `usgs_gage_crosswalk_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `generate_branch_list_test.py`  
+    - `/gms`  
+      - `derive_level_paths_test.py`  
+      - `outputs_cleanup_test.py`
+    - `/tools`
+      - `inundate_unittests.py` -> `inundation_test.py`  
+      - `inundate_gms_test.py`
+
+
+<br/><br/>
+
+## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
+
+The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
+
+### Changes
+
+- `tools/generate_categorical_fim.py` (all changes made here)
+    - Added try-except blocks for code that didn't allow most sites to actually get processed because it was trying to check values of some USGS-related variables that most of the sites didn't have
+    - Overwrite abilities of the different outputs for the viz team were not consistent (i.e., one of the files had the ability to be overwritten but another didn't), so that has been made consistent to disallow any overwrites of the existing final outputs for a specified output folder.
+    - The code also has the ability to restart from an interrupted run and resume processing uncompleted HUCs by first checking for a simple "complete" file for each HUC. If a HUC has that file, then it is skipped (because it already completed processing during a run for a particular output folder / run name).
+    - When a HUC is successfully processed, an empty "complete" text file is created / touched.
+
+<br/><br/>
+
 ## v4.1.0.0 - 2023-01-30 - [PR#806](https://github.com/NOAA-OWP/inundation-mapping/pull/806)
 
 As we move to Amazon Web Service, AWS, we need to change our processing system. Currently, it is `gms_pipeline.sh` using bash "parallel" as an iterator which then first processes all HUCs, but not their branches. One of `gms_pipeline.sh`'s next steps is to do branch processing which is again iterated via "parallel". AKA. Units processed as one step, branches processed as second independent step.

fim_post_processing.sh

@@ -69,7 +69,7 @@ rm -f $outputRunDataDir/logs/subdiv_src_.log
 
 # load up enviromental information
 args_file=$outputRunDataDir/runtime_args.env
-gms_inputs=$outputRunDataDir/gms_inputs.csv
+fim_inputs=$outputRunDataDir/fim_inputs.csv
 
 source $args_file
 source $outputRunDataDir/params.env
@@ -87,7 +87,7 @@ post_proc_start_time=`date +%s`
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 echo -e $startDiv"Start branch aggregation"
-python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $gms_inputs
+python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $fim_inputs
 
 ## GET NON ZERO EXIT CODES FOR UNITS ##
 ## No longer applicable
@@ -96,17 +96,9 @@ python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.cs
 echo -e $startDiv"Start non-zero exit code checking"
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
-
-## REMOVE FAILED BRANCHES ##
-# Needed in case aggregation fails, we will need the logs
-#echo
-#echo -e $startDiv"Removing branches that failed with Exit status: 61"
-#Tstart
-#python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g #$gms_inputs
-
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -i $fim_inputs
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
@@ -191,7 +183,7 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
 Tcount
 date -u
 

fim_pre_processing.sh

@@ -26,22 +26,22 @@ usage ()
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
     echo '  -ud/--unitDenylist'
     echo '                    A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
+    echo '                    upon completion (see config/deny_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_unit.lst'
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -bd/--branchDenylist'
     echo '                    A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
+    echo '                    (see config/deny_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branches.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist'
     echo '                    A file with a line delimited list of files in BRANCH ZERO directories to' 
     echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    (see config/deny_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branch_zero.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'    
     echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
@@ -141,7 +141,7 @@ if [ -z "$isAWS" ]; then isAWS=0; fi
 # validate and set defaults for the deny lists
 if [ "$deny_unit_list" = "" ]
 then
-    deny_unit_list=$projectDir/config/deny_gms_unit.lst
+    deny_unit_list=$projectDir/config/deny_unit.lst
 elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
 then
     # NONE is not case sensitive
@@ -152,7 +152,7 @@ fi
 # validate and set defaults for the deny lists
 if [ "$deny_branches_list" = "" ]
 then
-    deny_branches_list=$projectDir/config/deny_gms_branches.lst
+    deny_branches_list=$projectDir/config/deny_branches.lst
 elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
 then
     # NONE is not case sensitive
@@ -167,7 +167,7 @@ fi
 has_deny_branch_zero_override=0
 if [ "$deny_branch_zero_list" = "" ]
 then
-    deny_branch_zero_list=$projectDir/config/deny_gms_branch_zero.lst
+    deny_branch_zero_list=$projectDir/config/deny_branch_zero.lst
 elif [ "${deny_branch_zero_list^^}" != "NONE" ]   # NONE is not case sensitive
 then
     if [ ! -f "$deny_branch_zero_list" ]
@@ -211,7 +211,7 @@ else
     rm -rdf $outputRunDataDir/unit_errors
     rm -rdf $outputRunDataDir/eval
     rm -f $outputRunDataDir/crosswalk_table.csv
-    rm -f $outputRunDataDir/gms_inputs*
+    rm -f $outputRunDataDir/fim_inputs*
     rm -f $outputRunDataDir/*.env
 fi