Updates

* querying vectors with ogr2ogr now * fixed overwrite capabilities for gms_run_branch and _unit * fixed production mode deny list * removed gms_run.sh
NOAA-OWP · Jul 3, 2021 · 13e32cb · 13e32cb
1 parent e26816a
commit 13e32cb
Show file tree

Hide file tree

Showing 13 changed files with 101 additions and 317 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,9 +9,7 @@ config/**
 !config/*default*
 !config/*calibrated*
 !config/symbology/
-config/*template*.swp
-!config/*default*.swp
-!config/*calibrated*.swp
+config/*.swp
 .vscode/
 **/.DS_Store
 **/*_pytest.py

diff --git a/config/params_calibrated.env b/config/params_calibrated.env
@@ -22,6 +22,7 @@ export min_stream_length=0.5
 #### gms parameters ####
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
+export branch_timeout=1000 # pass int or float. To make a percentage of median, pass a '%' at the end.
 
 #### bathy SRC estimation parameters ####
 export bathy_src_modification=True

diff --git a/config/params_template.env b/config/params_template.env
@@ -22,6 +22,7 @@ export min_stream_length=0.5
 #### gms parameters ####
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
+export branch_timeout=1000 # pass int or float. To make a percentage of median, pass a '%' at the end.
 
 #### bathy SRC estimation parameters ####
 export bathy_src_modification=True

diff --git a/gms_run.sh b/gms_run.sh
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
@@ -103,21 +103,29 @@ export whitelist=$whitelist
 export viz=$viz
 logFile=$outputRunDataDir/logs/summary_gms_branch.log
 
-
 ## Make output and data directories ##
-if [[ -d "$outputRunDataDir" && "$overwrite" -eq 1 ]]; then
-    find $outputRunDataDir/**/gms/ -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} +
-elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then
-    echo "$runName data directories already exist. Use -o/--overwrite to continue"
+if [ -d "$outputRunDataDir" ]; then 
+    branch_directories_count=$(find $outputRunDataDir/**/gms/ -maxdepth 1 -mindepth 1 -type d | wc -l)
+
+    if [ $branch_directories_count -gt 0 ] && [ "$overwrite" -eq 1 ]; then
+        rm -rf $branch_directories_list
+    elif [ $branch_directories_count -gt 0 ] && [ "$overwrite" -eq 0 ] ; then
+        echo "GMS branch data directories for $runName already exist. Use -o/--overwrite to continue"
+        exit 1
+    fi
+else
+    echo "GMS depends on Full Resolution Data. Please produce data with fim_run.sh first."
     exit 1
 fi
+
+# make log dir
 mkdir -p $outputRunDataDir/logs
 
 
-## RUN GMS ##
+## RUN GMS BY BRANCH ##
 if [ "$jobLimit" -eq 1 ]; then
-    parallel --verbose --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv
+    parallel --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv
 else
-    parallel --eta -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv
+    parallel --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $outputRunDataDir/gms_inputs.csv
 fi
 
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
@@ -69,7 +69,7 @@ if [ "$runName" = "" ]
 then
     usage
 fi
-if [ "$overwrite" = "" ]
+if [ -z "$overwrite" ]
 then
     overwrite=0
 fi
@@ -91,27 +91,35 @@ logFile=$outputRunDataDir/logs/summary_gms_unit.log
 $srcDir/check_huc_inputs.py -u "$hucList"
 
 ## Make output and data directories ##
-if [[ -d "$outputRunDataDir" && "$overwrite" -eq 1 ]]; then
-     find $outputRunDataDir -iname "gms" -type d -exec rm -rf {} +
-elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then
-    echo "$runName data directories already exist. Use -o/--overwrite to continue"
+if [ -d "$outputRunDataDir" ]; then
+    gms_directories_count=$(find $outputRunDataDir -iname "gms" -type d | wc -l)
+    if [ $gms_directories_count -gt 0 ] && [ "$overwrite" -eq 1 ]; then
+         rm -rf $gms_directories_list
+    elif [ $gms_directories_count -gt 0 ] && [ "$overwrite" -eq 0 ] ; then
+        echo "GMS data directory for $runName already exist. Use -o/--overwrite to continue"
+        exit 1
+    fi
+else
+    echo "GMS depends on Full Resolution Data. Please produce data with fim_run.sh first."
     exit 1
 fi
+
+# make log dir
 mkdir -p $outputRunDataDir/logs
 
 
 ## GMS BY UNIT##
 if [ -f "$hucList" ]; then
     if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
     else
-        parallel --eta -j $jobLimit --joblog $logFile ---colsep ',' - $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
     fi
 else 
     if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
     else
-        parallel --eta -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
     fi
  fi
 

diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
@@ -21,9 +21,6 @@ input_demDerived_raster=$outputHucDataDir/demDerived_streamPixels.tif
 input_demDerived_reaches=$outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg
 input_demDerived_reaches_points=$outputHucDataDir/demDerived_reaches_split_points.gpkg
 input_demDerived_pixel_points=$outputHucDataDir/flows_points_pixels.gpkg
-input_stage_list=$outputHucDataDir/stage.txt
-input_hydroTable=$outputHucDataDir/hydroTable.csv
-input_src_full=$outputHucDataDir/src_full_crosswalked.csv
 
 
 ## ECHO PARAMETERS
@@ -73,7 +70,15 @@ Tcount
 echo -e $startDiv"Subsetting vectors to branches for $current_branch_id in HUC $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/query_vectors_by_branch_polygons.py -a $outputGmsDataDir/polygons.gpkg -d $current_branch_id -i $branch_id_attribute -s $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg -o $outputBranchDataDir/demDerived_reaches_levelPaths.gpkg $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputBranchDataDir/demDerived_reaches_points.gpkg $outputBranchDataDir/demDerived_pixels_points.gpkg -v
+echo -e "Querying demDerived_reaches_levelPaths.gpkg ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_levelPaths_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg
+echo -e "Querying demDerived_reaches_levelPaths_dissolved.gpkg ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg
+echo -e "Querying demDerived_reaches_points.gpkg ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_reaches_points_$current_branch_id.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg
+echo -e "Querying demDerived_pixels_points.gpkg ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputBranchDataDir/demDerived_pixels_points_$current_branch_id.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg
+#$srcDir/gms/query_vectors_by_branch_polygons.py -a $outputGmsDataDir/polygons.gpkg -d $current_branch_id -i $branch_id_attribute -s $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputGmsDataDir/demDerived_reaches_points.gpkg $outputGmsDataDir/demDerived_pixels_points.gpkg -o $outputBranchDataDir/demDerived_reaches_levelPaths.gpkg $outputBranchDataDir/demDerived_reaches_levelPaths_dissolved.gpkg $outputBranchDataDir/demDerived_reaches_points.gpkg $outputBranchDataDir/demDerived_pixels_points.gpkg -v
 Tcount
 
 ## SPLIT DERIVED REACHES ##
@@ -151,7 +156,7 @@ Tcount
 #    mkdir -p $branchOutputDir
 #fi
 
-if [ "$production" -eq 1 ]; then
+if [ $production -eq 1 ]; then
     echo -e $startDiv"Remove files for branch_id: $current_branch_id in HUC: $hucNumber"$stopDiv
 
     cd $outputBranchDataDir

diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
@@ -37,7 +37,7 @@ fi
 echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
 date -u
 Tstart
-#$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -b $branch_id_attribute -o $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg -d $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -v
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -b $branch_id_attribute -o $outputGmsDataDir/demDerived_reaches_levelPaths.gpkg -d $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -v
 Tcount
 
 ## STREAM BRANCH POLYGONS
@@ -46,7 +46,6 @@ date -u
 Tstart
 $srcDir/gms/buffer_stream_branches.py -s $outputGmsDataDir/demDerived_reaches_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputGmsDataDir/polygons.gpkg -v 
 Tcount
-exit 0
 
 ##### EDIT DEM DERIVED POINTS TO ADD BRANCH IDS ######
 echo -e $startDiv"EDITING DEM DERIVED POINTS for $hucNumber"$stopDiv