diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 09d892805..a1314d95f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
All notable changes to this project will be documented in this file.
We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
+
+A couple of directly related issues were fixed in this PR.
+The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing.
+
+### Changes
+
+- `config`/`params_template.env`: Updated for a newer mannings global file. Changed and tested by Ryan Spies.
+- `tools`
+ - `inundate_mosiac_wrapper.py`: Took out a misleading and non-required print statement.
+ - `inundate_nation.py`: As mentioned above.
+
+
+
## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036)
Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change.
@@ -90,6 +104,7 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
+
## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index c75c4ea2e..5b274122a 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -93,8 +93,6 @@ def produce_mosaicked_inundation(
"Please lower the num_workers.".format(num_workers, total_cpus_available)
)
- fh.vprint("Running inundate for " + huc + "...", verbose)
-
# Call Inundate_gms
map_file = Inundate_gms(
hydrofabric_dir=hydrofabric_dir,
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
old mode 100644
new mode 100755
index fe8680418..050436670
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -6,17 +6,14 @@
import os
import re
import shutil
-import sys
from datetime import datetime
from multiprocessing import Pool
import rasterio
from inundate_mosaic_wrapper import produce_mosaicked_inundation
-from osgeo import gdal, ogr
-from rasterio.merge import merge
+from osgeo import gdal
from utils.shared_functions import FIM_Helpers as fh
-from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv
# INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
@@ -26,6 +23,9 @@
# DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
+# TODO: Nov 2023, Logging system appears to be not working correctly.
+
+
def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number):
assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
@@ -37,12 +37,14 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
+ " max jobs will be used instead."
)
+ print()
+ print("Inundation Nation script starting...")
+
fim_version = os.path.basename(os.path.normpath(fim_run_dir))
- logging.info(f"Using fim version: {fim_version}")
output_base_file_name = magnitude_key + "_" + fim_version
- # print(output_base_file_name)
__setup_logger(output_dir, output_base_file_name)
+ logging.info(f"Using fim version: {fim_version}")
start_dt = datetime.now()
@@ -52,48 +54,42 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
logging.info(f"flow_file: {flow_file}")
logging.info(f"inc_mosaic: {str(inc_mosaic)}")
- print("Preparing to generate inundation outputs for magnitude: " + magnitude_key)
- print("Input flow file: " + flow_file)
-
magnitude_output_dir = os.path.join(output_dir, output_base_file_name)
if not os.path.exists(magnitude_output_dir):
- print("Creating new output directory for raw mosaic files: " + magnitude_output_dir)
+ logging.info(
+ "Removing previous output dir and creating new output dir for inunation wrapper files: "
+ + magnitude_output_dir
+ )
os.mkdir(magnitude_output_dir)
else:
# we need to empty it. we will kill it and remake it (using rmtree to force it)
shutil.rmtree(magnitude_output_dir, ignore_errors=True)
os.mkdir(magnitude_output_dir)
- if huc_list is None:
+ if huc_list == 'all' or len(huc_list) == 0:
huc_list = []
for huc in os.listdir(fim_run_dir):
- # if (
- # huc != 'logs'
- # and huc != 'branch_errors'
- # and huc != 'unit_errors'
- # and os.path.isdir(os.path.join(fim_run_dir, huc))
- # ):
if re.match(r'\d{8}', huc):
huc_list.append(huc)
else:
for huc in huc_list:
- assert os.path.isdir(
- fim_run_dir + os.sep + huc
- ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
+ huc_path = os.path.join(fim_run_dir, huc)
+ assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}'
- print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
+ huc_list.sort()
+ logging.info(f"Inundation mosaic wrapper outputs will saved here: {magnitude_output_dir}")
run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number])
# Perform mosaic operation
if inc_mosaic:
fh.print_current_date_time()
logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))
- print("Performing bool mosaic process...")
logging.info("Performing bool mosaic process...")
-
output_bool_dir = os.path.join(output_dir, "bool_temp")
+ logging.info(f"output_bool_dir is {output_bool_dir}")
+
if not os.path.exists(output_bool_dir):
os.mkdir(output_bool_dir)
else:
@@ -105,7 +101,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
for rasfile in os.listdir(magnitude_output_dir):
if rasfile.endswith(".tif") and "extent" in rasfile:
# p = magnitude_output_dir + rasfile
- procs_list.append([magnitude_output_dir, rasfile, output_bool_dir])
+ procs_list.append([magnitude_output_dir, rasfile, output_bool_dir, fim_version])
# Multiprocess --> create boolean inundation rasters for all hucs
if len(procs_list) > 0:
@@ -116,10 +112,17 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
print(msg)
logging.info(msg)
- # now cleanup the raw mosiac directories
+ # Perform VRT creation and mosaic all of the huc rasters using boolean rasters
+ vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number)
+
+ # now cleanup the temp bool directory
shutil.rmtree(output_bool_dir, ignore_errors=True)
+ else:
+ print("Skipping mosiaking")
+
# now cleanup the raw mosiac directories
+ # comment this out if you want to see the individual huc rasters
shutil.rmtree(magnitude_output_dir, ignore_errors=True)
fh.print_current_date_time()
@@ -149,7 +152,16 @@ def run_inundation(args):
inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif")
- print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
+ logging.info(
+ "Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: "
+ + str(magnitude)
+ )
+ print(
+ "This will take a long time depending on the number of HUCs. Progress bar may not appear."
+ " Once it gets to boolean/mosiacing (if applicable), screen output will exist. To see if the script has frozen,"
+ " you should be able to watch the file system for some changes."
+ )
+ print()
produce_mosaicked_inundation(
fim_run_dir,
@@ -167,6 +179,7 @@ def create_bool_rasters(args):
in_raster_dir = args[0]
rasfile = args[1]
output_bool_dir = args[2]
+ fim_version = args[3]
print("Calculating boolean inundate raster: " + rasfile)
p = in_raster_dir + os.sep + rasfile
@@ -189,23 +202,70 @@ def create_bool_rasters(args):
dtype="int8",
compress="lzw",
)
- with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst:
+ with rasterio.open(
+ output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile
+ ) as dst:
dst.write(array.astype(rasterio.int8))
-def __setup_logger(output_folder_path, log_file_name_key):
+def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads):
+ rasters_to_mosaic = []
+ for rasfile in os.listdir(output_bool_dir):
+ if rasfile.endswith('.tif') and "extent" in rasfile:
+ p = output_bool_dir + os.sep + rasfile
+ rasters_to_mosaic.append(p)
+
+ output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt")
+ logging.info("Creating virtual raster: " + output_mosiac_vrt)
+ vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic)
+
+ output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif")
+ logging.info("Building raster mosaic: " + output_mosiac_raster)
+ logging.info("Using " + str(threads) + " threads for parallizing")
+ print("Note: This step can take a number of hours if processing 100s of hucs")
+ gdal.Translate(
+ output_mosiac_raster,
+ vrt,
+ xRes=10,
+ yRes=-10,
+ creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)],
+ )
+ vrt = None
+
+
+def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO):
start_time = datetime.now()
file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
log_file_name = f"{log_file_name_key}-{file_dt_string}.log"
log_file_path = os.path.join(output_folder_path, log_file_name)
+ print('Log file created here:' + str(log_file_path))
+
+ # Clear previous logging configuration
+ logging.getLogger().handlers = []
+
+ # Create a StreamHandler and set the level
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+
+ # Create a FileHandler and set the level
+ file_handler = logging.FileHandler(log_file_path)
+ file_handler.setLevel(log_level)
- logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s")
+ # Create a formatter and set the formatter for the handlers
+ formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+ console_handler.setFormatter(formatter)
+ file_handler.setFormatter(formatter)
- # yes.. this can do console logs as well, but it can be a bit unstable and ugly
+ # Add the handlers to the logger
+ logger = logging.getLogger()
+ logger.setLevel(log_level)
+ logger.addHandler(console_handler)
+ logger.addHandler(file_handler)
- logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
- logging.info("----------------")
+ # Log the start time
+ logger.info(f'Started: {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+ logger.info("----------------")
if __name__ == "__main__":