NOAA-OWP · CarsonPruitt-NOAA · Aug 18, 2022 · Jul 21, 2022 · Jul 25, 2022 · Jul 28, 2022
diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
@@ -37,7 +37,7 @@ gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt

diff --git a/config/deny_gms_branches_min.lst b/config/deny_gms_branches_min.lst
@@ -37,7 +37,7 @@ gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt

diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_default.lst
@@ -13,6 +13,7 @@ nwm_headwaters.gpkg
 #nwm_subset_streams_levelPaths.gpkg
 #nwm_subset_streams_levelPaths_dissolved.gpkg
 #nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#usgs_elev_table.csv
 #usgs_subset_gages.gpkg
 #wbd.gpkg
 #wbd8_clp.gpkg

diff --git a/config/params_template.env b/config/params_template.env
@@ -52,6 +52,16 @@ export vrough_suffix="_vmann" # text to append to output log and src_full_crossw
 export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_011.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 export bankfull_attribute="chann_volume_ratio" # src_full_crosswalked_bankfull.csv attribute (column id) containing the channel vs overbank ratio values (generated in the identify_src_bankfull.py)
 
+#### apply SRC adjustments using USGS rating curve database ####
+export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
+export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
+
+#### apply SRC adjustments using observed FIM/flow point database ####
+export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
+export fim_obs_pnt_data="data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
+#### path to env file with sensitive paths for accessing postgres database ####
+export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
@@ -60,4 +70,4 @@ export memfree=0G # min free memory required to start a new job or keep youngest
 
 #### logging parameters ####
 export startDiv="\n##########################################################################\n"
-export stopDiv="\n##########################################################################"
+export stopDiv="\n##########################################################################"
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -2,6 +2,43 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.7.0 - 2022-08-17 - [PR #657](https://github.com/NOAA-OWP/inundation-mapping/pull/657)
+
+Introduces synthetic rating curve calibration workflow. The calibration computes new Manning's coefficients for the HAND SRCs using input data: USGS gage locations, USGS rating curve csv, and a benchmark FIM extent point database stored in PostgreSQL database. This addresses [#535].
+
+## Additions
+
+- `src/src_adjust_spatial_obs.py`: new synthetic rating curve calibration routine that prepares all of the spatial (point data) benchmark data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
+- `src/src_adjust_usgs_rating.py`: new synthetic rating curve calibration routine that prepares all of the USGS gage location and observed rating curve data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
+- `src/src_roughness_optimization.py`: new SRC post-processing script that ingests observed data and HUC/branch FIM output data to compute optimized Manning's coefficient values and update the discharge values in the SRCs. Outputs a new hydroTable.csv.
+
+## Changes
+
+- `config/deny_gms_branch_zero.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
+- `config/deny_gms_branches_min.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
+- `config/deny_gms_unit_default.lst`: added `usgs_elev_table.csv` to list of files to keep (used in calibration workflow)
+- `config/params_template.env`: added new variables for user to control calibration
+  - `src_adjust_usgs`: Toggle to run src adjustment routine (True=on; False=off)
+  - `nwm_recur_file`: input file location with nwm feature_id and recurrence flow values
+  - `src_adjust_spatial`: Toggle to run src adjustment routine (True=on; False=off)
+  - `fim_obs_pnt_data`: input file location with benchmark point data used to populate the postgresql database
+  - `CALB_DB_KEYS_FILE`: path to env file with sensitive paths for accessing postgres database
+- `gms_run_branch.sh`: includes new steps in the workflow to connect to the calibration PostgreSQL database, run SRC calibration w/ USGS gage rating curves, run SRC calibration w/ benchmark point database
+- `src/add_crosswalk.py`: added step to create placeholder variables to be replaced in post-processing (as needed). Created here to ensure consistent column variables in the final hydrotable.csv
+- `src/gms/run_by_unit.sh`: added new steps to workflow to create the `usgs_subset_gages.gpkg` file for branch zero and then perform crosswalk and create `usgs_elev_table.csv` for branch zero
+- `src/make_stages_and_catchlist.py`: Reconcile flows and catchments hydroids
+- `src/usgs_gage_aggregate.py`: changed streamorder data type from integer to string to better handle missing values in `usgs_gage_unit_setup.py`
+- `src/usgs_gage_unit_setup.py`: added new inputs and function to populate `usgs_elev_table.csv` for branch zero using all available gages within the huc (not filtering to a specific branch)
+- `src/utils/shared_functions.py`: added two new functions for calibration workflow
+  - `check_file_age`: check the age of a file (use for flagging potentially outdated input)
+  - `concat_huc_csv`: concatenate huc csv files to a single dataframe/csv
+- `src/utils/shared_variables.py`: defined new SRC calibration threshold variables
+  - `DOWNSTREAM_THRESHOLD`: distance in km to propogate new roughness values downstream
+  - `ROUGHNESS_MAX_THRESH`: max allowable adjusted roughness value (void values larger than this)
+  - `ROUGHNESS_MIN_THRESH`: min allowable adjusted roughness value (void values smaller than this)
+
+<br/><br/>
+
 ## v4.0.6.2 - 2022-08-16 - [PR #639](https://github.com/NOAA-OWP/inundation-mapping/pull/639)
 
 This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile. This addresses [#629].

diff --git a/gms_run_branch.sh b/gms_run_branch.sh
@@ -113,6 +113,21 @@ fi
 source $envFile
 source $srcDir/bash_functions.env
 
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - src_adjust_spatial parameter is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
+        echo "Loading HUC Data"
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $input_WBD_gdb WBDHU8
+        echo "Loading Point Data"
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+    fi
+fi
+
 # default values
 if [ "$jobLimit" = "" ] ; then
     jobLimit=$default_max_jobs
@@ -187,6 +202,19 @@ echo
 echo "Processing usgs gage aggregation"
 python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
+if [ "$src_adjust_usgs" = "True" ]; then
+    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
+    time python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
+    time python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+fi
+
 # -------------------
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs

diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
@@ -240,8 +240,23 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+    ## Set placeholder variables to be replaced in post-processing (as needed). Create here to ensure consistent column vars
     output_hydro_table['barc_on'] = False # set barc_on attribute to Fasle (default) --> will be overwritten if BARC module runs
+    output_hydro_table['raw_discharge_cms'] = output_src['Discharge (m3s-1)']
+    output_hydro_table['raw_Volume (m3)'] = output_src['Volume (m3)']
+    output_hydro_table['raw_WetArea (m2)'] = output_src['WetArea (m2)']
+    output_hydro_table['raw_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
     output_hydro_table['vmann_on'] = False # set vmann_on attribute to Fasle (default) --> will be overwritten if variable roughness module runs
+    output_hydro_table['raw_ManningN'] = output_src['ManningN']
+    output_hydro_table['vmann_discharge_cms'] = pd.NA
+    output_hydro_table['vmann_ManningN'] = pd.NA
+    output_hydro_table['adjust_src_on'] = False
+    output_hydro_table['last_updated'] = pd.NA
+    output_hydro_table['submitter'] = pd.NA
+    output_hydro_table['adjust_ManningN'] = pd.NA
+    output_hydro_table['obs_source'] = pd.NA
+    output_hydro_table['default_discharge_cms'] = pd.NA
+    output_hydro_table['default_ManningN'] = pd.NA
 
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)

diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
@@ -214,18 +214,27 @@ else
     echo -e $startDiv"Skipping branch zero processing because there are no stream orders being dropped $hucNumber"$stopDiv
 fi
 
-## CLEANUP BRANCH ZERO OUTPUTS ##
-echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
+## CREATE USGS GAGES FILE
+echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
+python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id -bzero $dropLowStreamOrders
 Tcount
 
-## CREATE USGS GAGES FILE
-echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
+## USGS CROSSWALK ##
+if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
+    echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"$stopDiv
+    date -u
+    Tstart
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
+    Tcount
+fi
+
+## CLEANUP BRANCH ZERO OUTPUTS ##
+echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
 Tcount
 
 ## REMOVE FILES FROM DENY LIST ##

diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
@@ -12,13 +12,9 @@ def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filena
     flows = gpd.read_file(flows_filename)
     catchments = gpd.read_file(catchments_filename)
 
-
-    hydroIDs = flows['HydroID'].tolist()
-    len_of_hydroIDs = len(hydroIDs)
-    slopes = flows['S0'].tolist()
-    lengthkm = flows['LengthKm'].tolist()
-    areasqkm = catchments['areasqkm'].tolist()
-
+    # Reconcile flows and catchments hydroids
+    flows      = flows.merge(catchments[['HydroID']], on='HydroID', how='inner')
+    catchments = catchments.merge(flows[['HydroID']], on='HydroID', how='inner')
 
     stages_max = stages_max + stages_interval
     stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)