diff --git a/.gitignore b/.gitignore index ad1f6f600..173354d9d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,9 +9,7 @@ config/** !config/*default* !config/*calibrated* !config/symbology/ -config/*template*.swp -!config/*default*.swp -!config/*calibrated*.swp +config/*.swp .vscode/ **/.DS_Store **/*_pytest.py diff --git a/config/params_calibrated.env b/config/params_calibrated.env index 57ebbb8ea..88225947a 100644 --- a/config/params_calibrated.env +++ b/config/params_calibrated.env @@ -22,6 +22,7 @@ export min_stream_length=0.5 #### gms parameters #### export branch_id_attribute=levpa_id export branch_buffer_distance_meters=7000 +export branch_timeout=1000 # pass int or float. To make a percentage of median, pass a '%' at the end. #### bathy SRC estimation parameters #### export bathy_src_modification=True diff --git a/config/params_template.env b/config/params_template.env index b381d2a1f..63b415b9a 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -22,6 +22,7 @@ export min_stream_length=0.5 #### gms parameters #### export branch_id_attribute=levpa_id export branch_buffer_distance_meters=7000 +export branch_timeout=1000 # pass int or float. To make a percentage of median, pass a '%' at the end. #### bathy SRC estimation parameters #### export bathy_src_modification=True diff --git a/gms_run.sh b/gms_run.sh deleted file mode 100755 index 633118278..000000000 --- a/gms_run.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash -e -: -usage () -{ - echo 'Produce GMS hydrofabric' - echo 'Usage : gms_run.sh [REQ: -u -c -n ] [OPT: -h -j ]' - echo '' - echo 'REQUIRED:' - echo ' -u/--hucList : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file' - echo ' also accepted. HUCs must present in inputs directory.' - echo ' -e/--extent : full resolution or mainstem method; options are MS or FR' - echo ' -c/--config : configuration file with bash environment variables to export' - echo ' -n/--runName : a name to tag the output directories and log files as. could be a version tag.' - echo '' - echo 'OPTIONS:' - echo ' -h/--help : help file' - echo ' -j/--jobLimit : max number of concurrent jobs to run. Default 1 job at time. 1 outputs' - echo ' stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest' - echo ' -o/--overwrite : overwrite outputs if already exist' - echo ' -p/--production : only save final inundation outputs' - echo ' -w/--whitelist : list of files to save in a production run in addition to final inundation outputs' - echo ' ex: file1.tif,file2.json,file3.csv' - echo ' -v/--viz : compute post-processing on outputs to be used in viz' - exit -} - -if [ "$#" -lt 7 ] -then - usage -fi - -while [ "$1" != "" ]; do -case $1 -in - -u|--hucList) - shift - hucList="$1" - ;; - -c|--configFile ) - shift - envFile=$1 - ;; - -e|--extent) - shift - extent=$1 - ;; - -n|--runName) - shift - runName=$1 - ;; - -j|--jobLimit) - shift - jobLimit=$1 - ;; - -h|--help) - shift - usage - ;; - -o|--overwrite) - overwrite=1 - ;; - -og|--overwrite-gms) - overwrite_gms=1 - ;; - -p|--production) - production=1 - ;; - -w|--whitelist) - shift - whitelist="$1" - ;; - -v|--viz) - viz=1 - ;; - *) ;; - esac - shift -done - -# print usage if arguments empty -if [ "$hucList" = "" ] -then - usage -fi -if [ "$extent" = "" ] -then - usage -fi -if [ "$envFile" = "" ] -then - usage -fi -if [ "$runName" = "" ] -then - usage -fi -if [ "$overwrite" = "" ] -then - overwrite=0 -fi - -## SOURCE ENV FILE AND FUNCTIONS ## -source $envFile -source $srcDir/bash_functions.env - -# default values -if [ "$jobLimit" = "" ] ; then - jobLimit=$default_max_jobs -fi - -## Define Outputs Data Dir & Log File## -export outputRunDataDir=$outputDataDir/$runName -export extent=$extent -export production=$production -export whitelist=$whitelist -export viz=$viz -logFile_fr=$outputRunDataDir/logs/summary.log -logFile_gms_unit=$outputRunDataDir/logs/summary_gms_unit.log -logFile_gms_branch=$outputRunDataDir/logs/summary_gms_branch.log - -## Define inputs -export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg -export input_NWM_Lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg -export input_NWM_Catchments_fr=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg -export input_NWM_Catchments_ms=$inputDataDir/nwm_hydrofabric/nwm_catchments_ms.gpkg -export input_NWM_Flows_fr=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg -export input_NWM_Flows_ms=$inputDataDir/nwm_hydrofabric/nwm_flows_ms.gpkg -export input_NWM_Headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg -export input_nhd_flowlines_fr=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_fr_adjusted.gpkg -export input_nhd_flowlines_ms=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_ms_adjusted.gpkg -export input_nhd_headwaters_fr=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_fr.gpkg -export input_nhd_headwaters_ms=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_ms.gpkg - -## Input handling ## -$srcDir/check_huc_inputs.py -u "$hucList" - -## Make output and data directories ## -if [[ ( -d "$outputRunDataDir" ) && ( "$overwrite" -eq 1 ) && ( ( "$extent" -eq "FR" ) || ( "$extent" -eq "MS" ) ) ]]; then - rm -rf "$outputRunDataDir" -elif [[ -d "$outputRunDataDir" && "$overwrite_gms" -eq 1 && "$extent" -eq "GMS" ]]; then - find $outputRunDataDir -iname "gms" -type d -exec rm -rf {} + -elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then - echo "$runName data directories already exist. Use -o/--overwrite to continue" - exit 1 -fi -mkdir -p $outputRunDataDir/logs - - -jobLimit_gms_unit=1 -## UNIT GMS FIRST ## -if [ "$extent" = "GMS" ]; then - if [ -f "$hucList" ]; then - if [ "$jobLimit_gms_unit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit_gms_unit --joblog $logFile_gms_unit --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList - else - parallel --eta -j $jobLimit_gms_unit --joblog $logFile_gms_unit ---colsep ',' - $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList - fi - else - if [ "$jobLimit_gms_unit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit_gms_unit --joblog $logFile_gms_unit --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList - else - parallel --eta -j $jobLimit_gms_unit --joblog $logFile_gms_unit --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList - fi - fi - python3 $srcDir/gms/aggregate_branch_lists.py -fi - - -## RUN GMS ## -if [ "$extent" = "GMS" ]; then - if [ "$jobLimit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit --joblog $logFile_gms_branch --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv - else - parallel --eta -j $jobLimit --joblog $logFile_gms_branch --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv - fi -fi - diff --git a/gms_run_branch.sh b/gms_run_branch.sh index fe9d51bc7..837735769 100755 --- a/gms_run_branch.sh +++ b/gms_run_branch.sh @@ -103,21 +103,29 @@ export whitelist=$whitelist export viz=$viz logFile=$outputRunDataDir/logs/summary_gms_branch.log - ## Make output and data directories ## -if [[ -d "$outputRunDataDir" && "$overwrite" -eq 1 ]]; then - find $outputRunDataDir/**/gms/ -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} + -elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then - echo "$runName data directories already exist. Use -o/--overwrite to continue" +if [ -d "$outputRunDataDir" ]; then + branch_directories_count=$(find $outputRunDataDir/**/gms/ -maxdepth 1 -mindepth 1 -type d | wc -l) + + if [ $branch_directories_count -gt 0 ] && [ "$overwrite" -eq 1 ]; then + rm -rf $branch_directories_list + elif [ $branch_directories_count -gt 0 ] && [ "$overwrite" -eq 0 ] ; then + echo "GMS branch data directories for $runName already exist. Use -o/--overwrite to continue" + exit 1 + fi +else + echo "GMS depends on Full Resolution Data. Please produce data with fim_run.sh first." exit 1 fi + +# make log dir mkdir -p $outputRunDataDir/logs -## RUN GMS ## +## RUN GMS BY BRANCH ## if [ "$jobLimit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv + parallel --verbose --timeout $branch_timeout --lb -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv else - parallel --eta -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv + parallel --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv fi diff --git a/gms_run_unit.sh b/gms_run_unit.sh index a98f8e810..33d54cf0d 100755 --- a/gms_run_unit.sh +++ b/gms_run_unit.sh @@ -69,7 +69,7 @@ if [ "$runName" = "" ] then usage fi -if [ "$overwrite" = "" ] +if [ -z "$overwrite" ] then overwrite=0 fi @@ -91,27 +91,35 @@ logFile=$outputRunDataDir/logs/summary_gms_unit.log $srcDir/check_huc_inputs.py -u "$hucList" ## Make output and data directories ## -if [[ -d "$outputRunDataDir" && "$overwrite" -eq 1 ]]; then - find $outputRunDataDir -iname "gms" -type d -exec rm -rf {} + -elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then - echo "$runName data directories already exist. Use -o/--overwrite to continue" +if [ -d "$outputRunDataDir" ]; then + gms_directories_count=$(find $outputRunDataDir -iname "gms" -type d | wc -l) + if [ $gms_directories_count -gt 0 ] && [ "$overwrite" -eq 1 ]; then + rm -rf $gms_directories_list + elif [ $gms_directories_count -gt 0 ] && [ "$overwrite" -eq 0 ] ; then + echo "GMS data directory for $runName already exist. Use -o/--overwrite to continue" + exit 1 + fi +else + echo "GMS depends on Full Resolution Data. Please produce data with fim_run.sh first." exit 1 fi + +# make log dir mkdir -p $outputRunDataDir/logs ## GMS BY UNIT## if [ -f "$hucList" ]; then if [ "$jobLimit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList + parallel --verbose --lb -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList else - parallel --eta -j $jobLimit --joblog $logFile ---colsep ',' - $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList + parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList fi else if [ "$jobLimit" -eq 1 ]; then - parallel --verbose --lb -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList + parallel --verbose --lb -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList else - parallel --eta -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList + parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList fi fi diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh index 057ac8d88..57359cb10 100755 --- a/src/gms/run_by_branch.sh +++ b/src/gms/run_by_branch.sh @@ -21,9 +21,6 @@ input_demDerived_raster=$outputHucDataDir/demDerived_streamPixels.tif input_demDerived_reaches=$outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg input_demDerived_reaches_points=$outputHucDataDir/demDerived_reaches_split_points.gpkg input_demDerived_pixel_points=$outputHucDataDir/flows_points_pixels.gpkg -input_stage_list=$outputHucDataDir/stage.txt -input_hydroTable=$outputHucDataDir/hydroTable.csv -input_src_full=$outputHucDataDir/src_full_crosswalked.csv ## ECHO PARAMETERS @@ -73,7 +70,15 @@ Tcount echo -e $startDiv"Subsetting vectors to branches for $current_branch_id in HUC $hucNumber"$stopDiv date -u Tstart -$srcDir/gms/query_vectors_by_branch_polygons.py -a $outputGmsDataDir/polygons.gpkg -d $current_branch_id -i $branch_id_attribute -s $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg -o $outputBranchDataDir/demDerived_reaches_levelPaths.gpkg $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputBranchDataDir/demDerived_reaches_points.gpkg $outputBranchDataDir/demDerived_pixels_points.gpkg -v +echo -e "Querying demDerived_reaches_levelPaths.gpkg ..." +ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_levelPaths_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg +echo -e "Querying demDerived_reaches_levelPaths_dissolved.gpkg ..." +ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg +echo -e "Querying demDerived_reaches_points.gpkg ..." +ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_points_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg +echo -e "Querying demDerived_pixels_points.gpkg ..." +ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_pixels_points_$current_branch_id.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg +#$srcDir/gms/query_vectors_by_branch_polygons.py -a $outputGmsDataDir/polygons.gpkg -d $current_branch_id -i $branch_id_attribute -s $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg -o $outputBranchDataDir/demDerived_reaches_levelPaths.gpkg $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputBranchDataDir/demDerived_reaches_points.gpkg $outputBranchDataDir/demDerived_pixels_points.gpkg -v Tcount ## SPLIT DERIVED REACHES ## @@ -151,7 +156,7 @@ Tcount # mkdir -p $branchOutputDir #fi -if [ "$production" -eq 1 ]; then +if [ $production -eq 1 ]; then echo -e $startDiv"Remove files for branch_id: $current_branch_id in HUC: $hucNumber"$stopDiv cd $outputBranchDataDir diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh index 2d88200b8..8e6ad7ce3 100755 --- a/src/gms/run_by_unit.sh +++ b/src/gms/run_by_unit.sh @@ -37,7 +37,7 @@ fi echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv date -u Tstart -#$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -b $branch_id_attribute -o $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg -d $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -v +$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -b $branch_id_attribute -o $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg -d $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -v Tcount ## STREAM BRANCH POLYGONS @@ -46,7 +46,6 @@ date -u Tstart $srcDir/gms/buffer_stream_branches.py -s $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputGmsDataDir/polygons.gpkg -v Tcount -exit 0 ##### EDIT DEM DERIVED POINTS TO ADD BRANCH IDS ###### echo -e $startDiv"EDITING DEM DERIVED POINTS for $hucNumber"$stopDiv diff --git a/src/gms/test_stream_branches.py b/src/gms/test_stream_branches.py deleted file mode 100755 index bdf90bc9e..000000000 --- a/src/gms/test_stream_branches.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 - -from stream_branches import StreamNetwork -from stream_branches import StreamBranchPolygons -import argparse - - -if __name__ == '__main__': - - # parse arguments - parser = argparse.ArgumentParser(description='Identifies stream branches, generates branch polygons, and clips raster to polygons') - parser.add_argument('-u','--huc8', help='to-do', required=True) - #parser.add_argument('-s','--streams', help='to-do', required=False) - #parser.add_argument('-b','--branches', help='to-do', required=False) - #parser.add_argument('-d','--buffer-distance', help='to-do', required=False) - #parser.add_argument('-r','--raster', help='to-do', required=False) - #parser.add_argument('-v','--verbose', help='to-do', required=False,action='store_true') - - # extract to dictionary - args = vars(parser.parse_args()) - - from os.path import join - huc8 = args["huc8"] ; huc4 = huc8[0:4] - input_data_dir = "/data/outputs/slope_1eddd72_dem_all_mannings_6/{}/".format(huc8) #"/data/temp/stream_branches" - output_data_dir = "/data/temp/stream_branches/large_test/{}".format(huc8) - streams_file = join(output_data_dir,"NHDPlusBurnLineEvent_subset.gpkg")#'nhdplus_burnlines_subset.gpkg') - branches_file = join("/data/inputs/nhdplus_vectors/{}/".format(huc4),'NHDPlusFlowLineVAA{}.gpkg'.format(huc4)) - buffer_distance = 1000 - to_clip_raster_file = join(input_data_dir,'dem_meters.tif') - to_clip_vector_file = join(input_data_dir,'NHDPlusBurnLineEvent_subset.gpkg') - - stream_network_processed_file = join(output_data_dir,'stream_network_processed.gpkg') - stream_polygons_file = join(output_data_dir,'stream_branch_polygons.gpkg') - clipped_raster_files = join(output_data_dir,'test_{}.tif') - clipped_vector_files = join(output_data_dir,'test_{}.gpkg') - - # load file - print('Loading file') - stream_network = StreamNetwork.from_file(streams_file) - - # merge stream branch attributes - print('Merge attributes') - stream_network = stream_network.merge_stream_branches(stream_branch_dataset=branches_file, - on='NHDPlusID', - attributes=['StreamOrde','ToNode','FromNode'], - branch_id_attribute = "LevelPathI", - stream_branch_layer_name=None - ) - - # derive inlets and outlets - print('Derive inlets and outlets') - stream_network = stream_network.derive_inlets(toNode_attribute='ToNode', - fromNode_attribute='FromNode', - inlets_attribute='inlet_id' - ) - stream_network = stream_network.derive_outlets(toNode_attribute='ToNode', - fromNode_attribute='FromNode', - outlets_attribute='outlet_id' - ) - - # derive arbolate sum - - print("Calculate arbolate sum") - stream_network = stream_network.get_arbolate_sum( - arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id', - reach_id_attribute='NHDPlusID',toNode_attribute='ToNode', - fromNode_attribute='FromNode',length_conversion_factor_to_km=0.001 - ) - - # derive stream newtork branches - print("Derive stream branches") - stream_network = stream_network.derive_stream_branches( - toNode_attribute='ToNode',fromNode_attribute='FromNode', - outlet_attribute='outlet_id',branch_id_attribute='branchID', - reach_id_attribute='NHDPlusID',comparison_attributes='arbolate_sum', - comparison_function=max - ) - - # write stream network with derived stream branches - print("Write stream branches") - stream_network.write(stream_network_processed_file) - - # dissolving - print("Dissolve stream network by branch") - stream_network = stream_network.dissolve_by_branch(branch_id_attribute='branchID',attribute_excluded='StreamOrde',values_excluded=[1,2],out_vector_file_template=None) - - # make stream polygons - print("Buffer stream branches to polygons") - stream_polys = StreamBranchPolygons.buffer_stream_branches(stream_network,buffer_distance=buffer_distance) - - # write polygons - print("Write polygons") - stream_polys.write(stream_polygons_file) - - # clip rasters to polygons - print("Clip raster to polygons") - out_rasters = stream_polys.clip(to_clip_raster_file,clipped_raster_files) - - # clip rasters to polygons - print("Clip vector to polygons") - out_vectors = stream_polys.clip(to_clip_vector_file,clipped_vector_files) diff --git a/src/gms/time_and_tee_run_by_branch.sh b/src/gms/time_and_tee_run_by_branch.sh index d9119baf6..7681aa7ba 100755 --- a/src/gms/time_and_tee_run_by_branch.sh +++ b/src/gms/time_and_tee_run_by_branch.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/$1_gms.log +/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/$1_gms_branch_$2.log exit ${PIPESTATUS[0]} diff --git a/src/gms/toDo.md b/src/gms/toDo.md index 1283c7989..73a8d9b3d 100644 --- a/src/gms/toDo.md +++ b/src/gms/toDo.md @@ -1,13 +1,18 @@ # GMS To Do: ## Immediate -- overwrite feature in gms_run_branch and gms_run_unit. Use find command +- test GMS overwrite +- mannings n 0.06 test +- branch logs with commas +- GMS whitelist for FIM 3 +- convenience wrapper for fim_run, gms_run_unit.sh, gms_run_branch.sh. Move gms_run_branch.sh and gms_run_unit.sh to src/gms dir + ## integration - merge in latest cahaba/dev - compatability for new rem.py, add_crosswalk.py (checking small reaches), usgs_crosswalk.py ## optimize -- parameterize branch buffer distance by stream order +- ## Inundation - inundate_gms: "No forecast value found in passed .." diff --git a/tools/compare_gms_srcs_to_fr.py b/tools/compare_gms_srcs_to_fr.py index f935dbfc2..e15072506 100755 --- a/tools/compare_gms_srcs_to_fr.py +++ b/tools/compare_gms_srcs_to_fr.py @@ -7,10 +7,10 @@ import argparse import matplotlib.pyplot as plt -def join_srcs(gms_data_dir,fr_data_dir): +def join_srcs(fr_data_dir,outputs_dir=None): gms_srcs = None - gms_srcs_file_list_generator = iglob( join(gms_data_dir,"**","src_full_*.csv"), recursive=True) + gms_srcs_file_list_generator = iglob( join(fr_data_dir,"*","gms","*","src_full_*.csv"), recursive=True) for i in gms_srcs_file_list_generator: current_gms_src = pd.read_csv(i) @@ -21,7 +21,7 @@ def join_srcs(gms_data_dir,fr_data_dir): gms_srcs = pd.concat((gms_srcs,current_gms_src),ignore_index=True) # fr srcsi - fr_src_file_name = glob( join(fr_data_dir,"src_full_crosswalked.csv"))[0] + fr_src_file_name = glob( join(fr_data_dir,'*',"src_full_crosswalked.csv"))[0] fr_src = pd.read_csv( fr_src_file_name ) # average out values by stage @@ -35,7 +35,8 @@ def join_srcs(gms_data_dir,fr_data_dir): plt.xlabel('Discharge (m3s-1)') plt.ylabel('Stage (m)') plt.legend() - plt.savefig("/data/test.jpg") + if outputs_dir is not None: + plt.savefig( join(outputs_dir,"rating_curve.jpg") ) #combined = pd.concat((gms_srcs,fr_src), ignore_index=True, axis=1) # Volume plots @@ -45,7 +46,8 @@ def join_srcs(gms_data_dir,fr_data_dir): plt.xlabel('Stage (m)') plt.ylabel('Volume (m3)') plt.legend() - plt.savefig("/data/test_vol.jpg") + if outputs_dir is not None: + plt.savefig( join(outputs_dir,"volume.jpg") ) # Bed Area plots plt.figure() @@ -54,7 +56,8 @@ def join_srcs(gms_data_dir,fr_data_dir): plt.xlabel('Stage (m)') plt.ylabel('Bed Area (m2)') plt.legend() - plt.savefig("/data/test_bedarea.jpg") + if outputs_dir is not None: + plt.savefig( join(outputs_dir,"bedarea.jpg") ) # Volume and Bed Area plots plt.figure() @@ -67,7 +70,8 @@ def join_srcs(gms_data_dir,fr_data_dir): plt.xlabel('Stage (m)') plt.ylabel('Volume (m3s-1) ^ 5/3 divided by Bed Area (m2) ^ -2/3') plt.legend() - plt.savefig("/data/test_volume_and_bedarea.jpg") + if outputs_dir is not None: + plt.savefig( join(outputs_dir,"volume_and_bedarea.jpg") ) # Number of Cells plots @@ -77,7 +81,8 @@ def join_srcs(gms_data_dir,fr_data_dir): plt.xlabel('Stage (m)') plt.ylabel('Number of Cells') plt.legend() - plt.savefig("/data/test_number_of_cells.jpg") + if outputs_dir is not None: + plt.savefig( join(outputs_dir,"number_of_cells.jpg") ) @@ -85,8 +90,8 @@ def join_srcs(gms_data_dir,fr_data_dir): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Comparing SRCs') - parser.add_argument('-g','--gms-data-dir', help='Directory with GMS data', required=True) parser.add_argument('-f','--fr-data-dir', help='Directory with FR data', required=True) + parser.add_argument('-o','--outputs-dir', help='Directory for outputs data', required=False,default=None) args = vars(parser.parse_args()) diff --git a/tools/gms/compile_computational_stats.py b/tools/gms/compile_computational_stats.py new file mode 100755 index 000000000..f7ab6daed --- /dev/null +++ b/tools/gms/compile_computational_stats.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import argparse +import pandas as pd +import numpy as np +from glob import iglob +from os.path import join + +# desired output for branches +# dataframe columns: HUC, branch_id, exit status, ,time, ram, + +def compile_summary(gms_output_dir,ouput=None): + + unit_summary = join(gms_output_dir,logs, 'summary_gms_unit.log') + branch_summary = join(gms_output_dir,logs, 'summary_gms_branch.log') + + unit_summary = pd.read_csv(unit_summary,sep='\t') + branch_summary = pd.read_csv(branch_summary,sep='\t') + + + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Create stream network level paths') + parser.add_argument('-d','--gms-output-dir', help='Input stream network', required=True) + parser.add_argument('-o','--output', help='Input stream network', required=True) + + args = vars(parser.parse_args()) + + compile_summary(**args) +