Run AutoRoute Multiprocessing
Alan D. Snow edited this page Jul 7, 2016
·
10 revisions
#How to import
from AutoRoutePy.run import run_autoroute_multiprocess
#Function
run_autoroute_multiprocess(autoroute_input_directory,
autoroute_output_directory,
log_directory,
autoroute_executable_location="",
autoroute_manager=None,
return_period="",
return_period_file="",
rapid_output_file="",
date_peak_search_start=None,
date_peak_search_end=None,
river_id="",
streamflow_id="",
stream_network_shapefile="",
mode="multiprocess",
generate_flood_map_raster=True,
generate_flood_depth_raster=False,
generate_flood_map_shapefile=False,
wait_for_all_processes_to_finish=True,
num_cpus=-17
)| Variable | Data Type | Description | Default |
|---|---|---|---|
| autoroute_input_directory | String | Main directory where the sub folders with simulation files are stored. | |
| autoroute_output_directory | String | Directory where the output GIS files will be placed. | |
| log_directory | String | Path to output logs for the AutoRoute runs (HTCondor or Multiprocessing). | |
| autoroute_executable_location | String | Path to AutoRoute executable. | |
| autoroute_manager | AutoRoute | AutoRoute manager class from AutoRoutePy. | |
| return_period | String | (Optional) This is the value of the field in the return period file generated by RAPIDpy. Valid values include: 'max_flow','return_period_20','return_period_10', and 'return_period_2'. | |
| return_period_file | String | (Optional) This is the path to the return period file generated by RAPIDpy. | |
| rapid_output_file | String | (Optional) This is the path to a CF compliant RAPID Qout output file. | |
| date_peak_search_start | datetime | (Optional) This constrains the start of the search for the peak in the RAPID Qout file. | |
| date_peak_search_end | datetime | (Optional) This constrains the end of the search for the peak in the RAPID Qout file. | |
| river_id | String | (Optional) This is the unique identifier field in the stream network shapefile. | "COMID" |
| streamflow_id | String | (Optional) This is the streamflow peak field in the stream network shapefile. | |
| stream_network_shapefile | String | Path to stream network shapefile with unique id and peak streamflow for each stream segment. | |
| mode | String | (Optional) This is the multiprocessing mode. Valid modes are "multiprocessing" and "htcondor" | "multiprocessing" |
| generate_flood_map_raster | Boolean | (Optional) If set to True, this will generate the flood inundation extents rasters in the simulation. | True |
| generate_flood_depth_raster | Boolean | (Optional) If set to True, this will generate the flood depth rasters in the simulation. | False |
| generate_flood_map_shapefile | Boolean | (Optional) If set to True, this will both generate and convert the flood inundation extents raster to a shapefile. If generate_flood_map_raster is not set to true, flood extents rasters will be deleted. | False |
| wait_for_all_processes_to_finish | Boolean | (Optional) If set to False, this will convert the flood inundation extents raster to a shapefile. | False |
| num_cpus | Int | (Optional) This is the number of CPUs to use in the process. | Use all CPUs |
There are several options to run AutoRoute using multiprocessing. In these examples, you will learn how to run this process with multiprocessing and HTCondor. Also, you will learn how to use the output of a RAPID (rapid-hub.org) simulation Qout file, the return period file from RAPIDpy, and from the streamflow field of a shapefile to generate peak flows and then run AutoRoute.
In multiprocessing mode, you have to have your folders organizes inside the input directory such that each elevation DEM tile has its own folder.
####Example 1: Using pre-generated inputs with multiprocessing This example demonstrates a basic AutoRoute run with all inputs pre-generated beforehand using multiprocessing.
from AutoRoutePy.run import run_autoroute_multiprocess
from multiprocessing import freeze_support #for WINDOWS
def main():
autoroute_watershed_input_directory = '/autoroute-io/input/watershed-directory'
autoroute_watershed_output_directory = '/autoroute-io/output/watershed-directory'
autoroute_executable_location = '/AutoRoute/src/autoroute'
log_dir = '/logs'
run_autoroute_multiprocess(autoroute_input_directory=autoroute_watershed_input_directory,
autoroute_output_directory=autoroute_watershed_output_directory,
log_directory=log_dir,
autoroute_executable_location=autoroute_executable_location,
#mode="multiprocess",
#generate_flood_map_raster=True,
#generate_flood_depth_raster=False,
#generate_flood_map_shapefile=False,
#wait_for_all_processes_to_finish=True,
#num_cpus=2,
)
if __name__=="__main__":
freeze_support() #for WINDOWS
main()####Example 2: Using RAPID Qout file with HTCondor This example also demonstrates how to run AutoRoute with the addition of adding RAPID Qout before the simulation run. The AutoRoute simulations are run using HTCondor.
from datetime import datetime
from AutoRoutePy.run import run_autoroute_multiprocess
from multiprocessing import freeze_support #for WINDOWS
def main():
autoroute_executable_location = '/AutoRoute/src/autoroute'
autoroute_watershed_input_directory = '/autoroute-io/input/watershed-directory'
autoroute_watershed_output_directory = '/autoroute-io/output/watershed-directory'
condor_log_dir = '/condor_logs'
rapid_output_file = '/rapid-io/output/watershed-directory/Qout_lis_1980to2014.nc'
run_autoroute_multiprocess(autoroute_input_directory=autoroute_watershed_input_directory,
autoroute_output_directory=autoroute_watershed_output_directory,
log_directory=condor_init_dir,
autoroute_executable_location=autoroute_executable_location,
rapid_output_file=rapid_output_file,
#date_peak_search_start=datetime(1990, 1, 1),
#date_peak_search_end=datetime(1990, 3, 5),
mode="htcondor",
#generate_flood_map_raster=True,
#generate_flood_depth_raster=False,
#generate_flood_map_shapefile=False,
#wait_for_all_processes_to_finish=True,
#num_cpus=2,
)
if __name__=="__main__":
freeze_support() #for WINDOWS
main()####Example 3: Using shapefile peakflow with multiprocessing This example demonstrates how to use peakflows in a shapefile in an AutoRoute simulation.
from AutoRoutePy.run import run_autoroute_multiprocess
from multiprocessing import freeze_support #for WINDOWS
def main():
autoroute_executable_location = '/AutoRoute/src/autoroute'
autoroute_watershed_input_directory = '/autoroute-io/input/watershed-directory'
autoroute_watershed_output_directory = '/autoroute-io/output/watershed-directory'
log_dir = '/logs'
stream_network = '/path/to/stream_network.shp'
run_autoroute_multiprocess(autoroute_input_directory=autoroute_watershed_input_directory,
autoroute_output_directory=autoroute_watershed_output_directory,
log_directory=log_dir,
autoroute_executable_location=autoroute_executable_location,
stream_network_shapefile=stream_network,
#river_id='COMID',
streamflow_id='Flow_50',
#mode="multiprocess",
#generate_flood_map_raster=True,
#generate_flood_depth_raster=False,
#generate_flood_map_shapefile=False,
#wait_for_all_processes_to_finish=True,
#num_cpus=2,
)
if __name__=="__main__":
freeze_support() #for WINDOWS
main()####Example 4: Using return period file with multiprocessing In addition to highlighting the usage of return period data, this example shows how you can use the defaults to generate the flood map shapefile and delete the flood map raster.
from AutoRoutePy.run import run_autoroute_multiprocess
from multiprocessing import freeze_support #for WINDOWS
def main():
autoroute_executable_location = '/AutoRoute/src/autoroute'
autoroute_watershed_input_directory = '/autoroute-io/input/watershed-directory'
autoroute_watershed_output_directory = '/autoroute-io/output/watershed-directory'
log_dir = '/logs'
#run based on return period data
return_period_file = '/rapid-io/output/watershed-directory/return_periods.nc'
run_autoroute_multiprocess(autoroute_input_directory=autoroute_watershed_input_directory,
autoroute_output_directory=autoroute_watershed_output_directory,
log_directory=log_dir,
autoroute_executable_location=autoroute_executable_location,
return_period='return_period_20',
return_period_file=return_period_file,
#mode="multiprocess",
generate_flood_map_raster=False,
#generate_flood_depth_raster=False,
generate_flood_map_shapefile=True,
#wait_for_all_processes_to_finish=True
#num_cpus=2,
)
if __name__=="__main__":
freeze_support() #for WINDOWS
main()####Example 5: Use AutoRoute manager to modify default parameters This example shows how you can use the AutoRoute class as a manager for your defaults. You simply name one of AutoRoute's parameters as an input.
For more parameters, see: https://github.com/erdc-cm/AutoRoute/tree/cgdal#autoroute-parameters
from AutoRoutePy import AutoRoute
from AutoRoutePy.run import run_autoroute_multiprocess
from multiprocessing import freeze_support #for WINDOWS
def main():
autoroute_watershed_input_directory = '/autoroute-io/input/watershed-directory'
autoroute_watershed_output_directory = '/autoroute-io/output/watershed-directory'
autoroute_executable_location = '/AutoRoute/src/autoroute'
log_dir = '/logs'
auto_mng = AutoRoute(autoroute_executable_location,
x_section_dist=5000.0,
low_spot_range=15,
degree_manipulation=6.1,
degree_interval=1.5
)
run_autoroute_multiprocess(autoroute_input_directory=autoroute_watershed_input_directory,
autoroute_output_directory=autoroute_watershed_output_directory,
log_directory=log_dir,
autoroute_manager=auto_mng,
#mode="multiprocess",
#generate_flood_map_raster=True,
#generate_flood_depth_raster=False,
#generate_flood_map_shapefile=False,
#wait_for_all_processes_to_finish=True
#num_cpus=2,
)
if __name__=="__main__":
freeze_support() #for WINDOWS
main()