From 2c01ffed4b2093272773bc9651b3f3c1942f3cba Mon Sep 17 00:00:00 2001 From: sylwiamm Date: Tue, 2 Apr 2024 16:21:38 -0400 Subject: [PATCH 1/5] Logging --- GANDLF/cli/main_run.py | 35 ++-- GANDLF/cli/preprocess_and_save.py | 8 +- GANDLF/compute/forward_pass.py | 103 ++++++----- GANDLF/compute/generic.py | 16 +- GANDLF/compute/inference_loop.py | 23 ++- GANDLF/compute/loss_and_metric.py | 6 +- GANDLF/compute/step.py | 38 +++-- GANDLF/compute/training_loop.py | 232 ++++++++++++------------- GANDLF/config_manager.py | 55 +++--- GANDLF/{logger.py => csv_logger.py} | 4 +- GANDLF/data/ImagesFromDataFrame.py | 255 ++++++++++++++-------------- GANDLF/metrics/generic.py | 6 +- GANDLF/models/densenet.py | 5 +- GANDLF/models/efficientnet.py | 5 +- GANDLF/models/imagenet_unet.py | 6 +- GANDLF/models/resnet.py | 8 +- GANDLF/models/sdnet.py | 8 +- GANDLF/training_manager.py | 16 +- GANDLF/utils/__init__.py | 4 + GANDLF/utils/logger.py | 51 ++++++ GANDLF/utils/modelio.py | 18 +- GANDLF/utils/tensor.py | 6 +- GANDLF/utils/write_parse.py | 14 +- docs/usage.md | 7 + gandlf_run | 5 +- setup.py | 1 + testing/test_full.py | 32 +++- 27 files changed, 547 insertions(+), 420 deletions(-) rename GANDLF/{logger.py => csv_logger.py} (95%) create mode 100644 GANDLF/utils/logger.py diff --git a/GANDLF/cli/main_run.py b/GANDLF/cli/main_run.py index 45e303254..e0a5b8dce 100644 --- a/GANDLF/cli/main_run.py +++ b/GANDLF/cli/main_run.py @@ -1,3 +1,4 @@ +import yaml from typing import Optional from pathlib import Path @@ -8,6 +9,7 @@ populate_header_in_parameters, parseTrainingCSV, parseTestingCSV, + setup_logger, ) @@ -40,8 +42,6 @@ def main_run( file_data_full = data_csv model_parameters = config_file device = device - parameters = ConfigManager(model_parameters) - parameters["device_id"] = -1 if train_mode: if resume: @@ -49,15 +49,24 @@ def main_run( "Trying to resume training without changing any parameters from previous run.", flush=True, ) - parameters["output_dir"] = model_dir - Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) + output_dir = model_dir + else: + if output_dir is None: + output_dir = model_dir + + Path(output_dir).mkdir(parents=True, exist_ok=True) + + with open(config_file, 'r') as f: + config_params = yaml.safe_load(f) - # if the output directory is not specified, then use the model directory even for the testing data - # default behavior - parameters["output_dir"] = parameters.get("output_dir", output_dir) - if parameters["output_dir"] is None: - parameters["output_dir"] = model_dir - Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) + # setup logger + logger, logs_dir, logger_name = setup_logger(output_dir=output_dir, verbose=config_params.get("verbose", False)) + + parameters = ConfigManager(model_parameters) + parameters["device_id"] = -1 + parameters["output_dir"] = output_dir + parameters["logs_dir"] = logs_dir + parameters["logger_name"] = logger_name if "-1" in device: device = "cpu" @@ -73,7 +82,7 @@ def main_run( ) assert ( headers_train == headers_validation - ), "The training and validation CSVs do not have the same header information." + ), logger.error("The training and validation CSVs do not have the same header information.") # testing data is present data_testing = None @@ -84,10 +93,10 @@ def main_run( ) assert ( headers_train == headers_testing - ), "The training and testing CSVs do not have the same header information." + ), logger.error("The training and testing CSVs do not have the same header information.") parameters = populate_header_in_parameters(parameters, headers_train) - # if we are here, it is assumed that the user wants to do training + logger.debug("if we are here, it is assumed that the user wants to do training") if train_mode: TrainingManager_split( dataframe_train=data_train, diff --git a/GANDLF/cli/preprocess_and_save.py b/GANDLF/cli/preprocess_and_save.py index de192695d..0714eee6f 100644 --- a/GANDLF/cli/preprocess_and_save.py +++ b/GANDLF/cli/preprocess_and_save.py @@ -1,4 +1,4 @@ -import os, sys, pickle +import os, pickle, warnings from typing import Optional from pathlib import Path import SimpleITK as sitk @@ -88,10 +88,8 @@ def preprocess_and_save( (parameters["patch_sampler"] == "label") or (isinstance(parameters["patch_sampler"], dict)) ) and parameters["q_samples_per_volume"] > 1: - print( - "[WARNING] Label sampling has been enabled but q_samples_per_volume > 1; this has been known to cause issues, so q_samples_per_volume will be hard-coded to 1 during preprocessing. Please contact GaNDLF developers for more information", - file=sys.stderr, - flush=True, + warnings.warn( + "Label sampling has been enabled but q_samples_per_volume > 1; this has been known to cause issues, so q_samples_per_volume will be hard-coded to 1 during preprocessing. Please contact GaNDLF developers for more information" ) for _, (subject) in enumerate( diff --git a/GANDLF/compute/forward_pass.py b/GANDLF/compute/forward_pass.py index 9da87b8ff..c403fa22f 100644 --- a/GANDLF/compute/forward_pass.py +++ b/GANDLF/compute/forward_pass.py @@ -1,7 +1,7 @@ import os import pathlib from typing import Optional, Tuple - +import logging import numpy as np import pandas as pd import SimpleITK as sitk @@ -19,6 +19,7 @@ reverse_one_hot, get_ground_truths_and_predictions_tensor, print_and_format_metrics, + setup_logger, ) from GANDLF.metrics import overall_stats from tqdm import tqdm @@ -46,6 +47,11 @@ def validate_network( Returns: Tuple[float, dict]: The average validation loss and the average validation metrics. """ + if "logger_name" in params: + logger = logging.getLogger(params["logger_name"]) + else: + logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", False)) + print("*" * 20) print("Starting " + mode + " : ") print("*" * 20) @@ -70,9 +76,8 @@ def validate_network( is_inference = mode == "inference" # automatic mixed precision - https://pytorch.org/docs/stable/amp.html - if params["verbose"]: - if params["model"]["amp"]: - print("Using Automatic mixed precision", flush=True) + if params["model"]["amp"]: + logger.debug("Using Automatic mixed precision") if scheduler is None: current_output_dir = params["output_dir"] # this is in inference mode @@ -115,8 +120,7 @@ def validate_network( for batch_idx, (subject) in enumerate( tqdm(valid_dataloader, desc="Looping over " + mode + " data") ): - if params["verbose"]: - print("== Current subject:", subject["subject_id"], flush=True) + logger.debug("== Current subject:", subject["subject_id"]) # ensure spacing is always present in params and is always subject-specific params["subject_spacing"] = None @@ -247,16 +251,14 @@ def validate_network( output_prediction = 0 # this is used for regression/classification current_patch = 0 for patches_batch in patch_loader: - if params["verbose"]: - print( - "=== Current patch:", - current_patch, - ", time : ", - get_date_time(), - ", location :", - patches_batch[torchio.LOCATION], - flush=True, - ) + logger.debug( + "=== Current patch:", + current_patch, + ", time : ", + get_date_time(), + ", location :", + patches_batch[torchio.LOCATION] + ) current_patch += 1 image = ( torch.cat( @@ -279,14 +281,12 @@ def validate_network( if label is not None: label = label.to(params["device"]) - if params["verbose"]: - print( - "=== Validation shapes : label:", - label.shape, - ", image:", - image.shape, - flush=True, - ) + logger.debug( + "=== Validation shapes : label:", + label.shape, + ", image:", + image.shape + ) if is_inference: result = step(model, image, None, params, train=False) @@ -427,14 +427,12 @@ def validate_network( output_prediction.to(torch.float32), params, ) - if params["verbose"]: - print( - "Full image " + mode + ":: Loss: ", - final_loss, - "; Metric: ", - final_metric, - flush=True, - ) + logger.debug( + "Full image " + mode + ":: Loss: ", + final_loss, + "; Metric: ", + final_metric + ) # # Non network validing related # loss.cpu().data.item() @@ -453,28 +451,27 @@ def validate_network( total_epoch_valid_metric[metric] += final_metric[metric] if label_ground_truth is not None: - if params["verbose"]: - # For printing information at halftime during an epoch - if ((batch_idx + 1) % (len(valid_dataloader) / 2) == 0) and ( - (batch_idx + 1) < len(valid_dataloader) - ): - print( - "\nHalf-Epoch Average " + mode + " loss : ", - total_epoch_valid_loss / (batch_idx + 1), - ) - for metric in params["metrics"]: - if isinstance(total_epoch_valid_metric[metric], np.ndarray): - to_print = ( - total_epoch_valid_metric[metric] / (batch_idx + 1) - ).tolist() - else: - to_print = total_epoch_valid_metric[metric] / ( - batch_idx + 1 - ) - print( - "Half-Epoch Average " + mode + " " + metric + " : ", - to_print, + # For printing information at halftime during an epoch + if ((batch_idx + 1) % (len(valid_dataloader) / 2) == 0) and ( + (batch_idx + 1) < len(valid_dataloader) + ): + logger.debug( + "\nHalf-Epoch Average " + mode + " loss : ", + total_epoch_valid_loss / (batch_idx + 1), + ) + for metric in params["metrics"]: + if isinstance(total_epoch_valid_metric[metric], np.ndarray): + to_print = ( + total_epoch_valid_metric[metric] / (batch_idx + 1) + ).tolist() + else: + to_print = total_epoch_valid_metric[metric] / ( + batch_idx + 1 ) + logger.debug( + "Half-Epoch Average " + mode + " " + metric + " : ", + to_print, + ) if params["medcam_enabled"] and params["model"]["type"] == "torch": model.disable_medcam() diff --git a/GANDLF/compute/generic.py b/GANDLF/compute/generic.py index 8c253cc2c..8b1b2cd5e 100644 --- a/GANDLF/compute/generic.py +++ b/GANDLF/compute/generic.py @@ -1,6 +1,6 @@ from typing import Optional, Tuple from pandas.util import hash_pandas_object -import torch +import torch, logging from torch.utils.data import DataLoader from GANDLF.models import get_model @@ -12,6 +12,7 @@ parseTrainingCSV, send_model_to_device, get_class_imbalance_weights, + setup_logger ) @@ -39,7 +40,12 @@ def create_pytorch_objects( Returns: Tuple[ torch.nn.Module, torch.optim.Optimizer, DataLoader, DataLoader, torch.optim.lr_scheduler.LRScheduler, dict, ]: The model, optimizer, train loader, validation loader, scheduler, and parameters. - """ + """ + if "logger_name" in parameters: + logger = logging.getLogger(parameters["logger_name"]) + else: + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + # initialize train and val loaders train_loader, val_loader = None, None headers_to_populate_train, headers_to_populate_val = None, None @@ -60,9 +66,9 @@ def create_pytorch_objects( parameters["class_weights"], ) = get_class_imbalance_weights(parameters["training_data"], parameters) - print("Penalty weights : ", parameters["penalty_weights"]) - print("Sampling weights: ", parameters["sampling_weights"]) - print("Class weights : ", parameters["class_weights"]) + logger.debug(f"Penalty weights : {parameters['penalty_weights']}") + logger.debug(f"Sampling weights: {parameters['sampling_weights']}") + logger.debug(f"Class weights : {parameters['class_weights']}") # get the train loader train_loader = get_train_loader(parameters) diff --git a/GANDLF/compute/inference_loop.py b/GANDLF/compute/inference_loop.py index c09b44cf7..3f63f6ebe 100644 --- a/GANDLF/compute/inference_loop.py +++ b/GANDLF/compute/inference_loop.py @@ -10,6 +10,7 @@ import torch import cv2 +import logging import numpy as np from torch.utils.data import DataLoader from skimage.io import imsave @@ -23,6 +24,7 @@ load_ov_model, print_model_summary, applyCustomColorMap, + setup_logger ) from GANDLF.data.inference_dataloader_histopath import InferTumorSegDataset @@ -46,12 +48,17 @@ def inference_loop( modelDir (str): The path to the directory containing the model to be used for inference. outputDir (str): The path to the directory where the output of the inference session will be stored. """ + if "logger_name" in parameters: + logger = logging.getLogger(parameters["logger_name"]) + else: + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + # Defining our model here according to parameters mentioned in the configuration file - print("Current model type : ", parameters["model"]["type"]) - print("Number of dims : ", parameters["model"]["dimension"]) + logger.debug("Current model type : ", parameters["model"]["type"]) + logger.debug("Number of dims : ", parameters["model"]["dimension"]) if "num_channels" in parameters["model"]: - print("Number of channels : ", parameters["model"]["num_channels"]) - print("Number of classes : ", len(parameters["model"]["class_list"])) + logger.debug("Number of channels : ", parameters["model"]["num_channels"]) + logger.debug("Number of classes : ", len(parameters["model"]["class_list"])) parameters["testing_data"] = inferenceDataFromPickle # ensure outputs are saved properly @@ -115,7 +122,7 @@ def inference_loop( parameters["model"]["IO"] = [input_blob, output_blob] if not (os.environ.get("HOSTNAME") is None): - print("\nHostname :" + str(os.environ.get("HOSTNAME")), flush=True) + logger.debug("\nHostname :" + str(os.environ.get("HOSTNAME"))) # radiology inference if parameters["modality"] == "rad": @@ -131,7 +138,7 @@ def inference_loop( # Setting up the inference loader inference_loader = get_testing_loader(parameters) - print("Data Samples: ", len(inference_loader.dataset), flush=True) + logger.debug("Data Samples: ", len(inference_loader.dataset)) average_epoch_valid_loss, average_epoch_valid_metric = validate_network( model, inference_loader, None, parameters, mode="inference" @@ -287,8 +294,8 @@ def inference_loop( # Check if out_probs_map is greater than 1, print a warning if np.max(probs_map) > 1: # Print a warning - print( - "Warning: Probability map is greater than 1, report the images to GaNDLF developers" + logger.warning( + "Probability map is greater than 1, report the images to GaNDLF developers" ) if count_map is not None: diff --git a/GANDLF/compute/loss_and_metric.py b/GANDLF/compute/loss_and_metric.py index e149c08db..f2247ba3b 100644 --- a/GANDLF/compute/loss_and_metric.py +++ b/GANDLF/compute/loss_and_metric.py @@ -1,4 +1,4 @@ -import sys +import warnings from typing import Dict, Tuple from GANDLF.losses import global_losses_dict from GANDLF.metrics import global_metrics_dict @@ -63,8 +63,8 @@ def get_loss_and_metrics( if loss_str_lower in global_losses_dict: loss_function = global_losses_dict[loss_str_lower] else: - sys.exit( - "WARNING: Could not find the requested loss function '" + warnings.warn( + "Could not find the requested loss function '" + params["loss_function"] ) diff --git a/GANDLF/compute/step.py b/GANDLF/compute/step.py index c36258c47..cb9c41a15 100644 --- a/GANDLF/compute/step.py +++ b/GANDLF/compute/step.py @@ -1,7 +1,9 @@ from typing import Optional, Tuple import torch import psutil +import logging from .loss_and_metric import get_loss_and_metrics +from GANDLF.utils import setup_logger def step( @@ -24,20 +26,23 @@ def step( Returns: Tuple[float, dict, torch.Tensor, torch.Tensor]: The loss, metrics, output, and attention map. """ - if params["verbose"]: - if torch.cuda.is_available(): - print(torch.cuda.memory_summary()) - print( - "|===========================================================================|" - ) - print( - "| CPU Utilization |" - ) - print("Load_Percent :", psutil.cpu_percent(interval=None)) - print("MemUtil_Percent:", psutil.virtual_memory()[2]) - print( - "|===========================================================================|" - ) + + if "logger_name" in params: + logger = logging.getLogger(params["logger_name"]) + else: + logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params["verbose"]) + + + if torch.cuda.is_available(): + logger.debug(torch.cuda.memory_summary()) + logger.debug( + f"""\n + |===========================================================================| \n + | CPU Utilization | \n + | Load_Percent : {psutil.cpu_percent(interval=None)} | \n + | MemUtil_Percent: {psutil.virtual_memory()[2]} | \n + |===========================================================================|""" + ) # for the weird cases where mask is read as an RGB image, ensure only the first channel is used if label is not None: @@ -46,9 +51,8 @@ def step( label = label[:, 0, ...].unsqueeze(1) # this warning should only come up once if params["print_rgb_label_warning"]: - print( - "WARNING: The label image is an RGB image, only the first channel will be used.", - flush=True, + logger.warning( + "The label image is an RGB image, only the first channel will be used." ) params["print_rgb_label_warning"] = False diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 61e0e6b0f..7bd7184e1 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -1,7 +1,7 @@ import os, time, psutil from typing import Tuple import pandas as pd -import torch +import torch, logging from torch.utils.data import DataLoader from tqdm import tqdm import numpy as np @@ -25,9 +25,10 @@ get_ground_truths_and_predictions_tensor, get_model_dict, print_and_format_metrics, + setup_logger, ) from GANDLF.metrics import overall_stats -from GANDLF.logger import Logger +from GANDLF.csv_logger import CSVLogger from .step import step from .forward_pass import validate_network from .generic import create_pytorch_objects @@ -54,6 +55,11 @@ def train_network( Returns: Tuple[float, dict]: The average epoch training loss and metrics. """ + if "logger_name" in params: + logger = logging.getLogger(params["logger_name"]) + else: + logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", False)) + print("*" * 20) print("Starting Training : ") print("*" * 20) @@ -74,8 +80,7 @@ def train_network( # automatic mixed precision - https://pytorch.org/docs/stable/amp.html if params["model"]["amp"]: scaler = GradScaler() - if params["verbose"]: - print("Using Automatic mixed precision", flush=True) + logger.debug("Using Automatic mixed precision") # get ground truths if calculate_overall_metrics: @@ -85,99 +90,98 @@ def train_network( ) = get_ground_truths_and_predictions_tensor(params, "training_data") # Set the model to train model.train() - for batch_idx, (subject) in enumerate( - tqdm(train_dataloader, desc="Looping over training data") - ): - optimizer.zero_grad() - image = ( - torch.cat( - [subject[key][torchio.DATA] for key in params["channel_keys"]], dim=1 - ) - .float() - .to(params["device"]) - ) - if "value_keys" in params: - label = torch.cat([subject[key] for key in params["value_keys"]], dim=0) - # min is needed because for certain cases, batch size becomes smaller than the total remaining labels - label = label.reshape( - min(params["batch_size"], len(label)), len(params["value_keys"]) + log_file = os.path.join(parameters["logs_dir"], "data_loop.log") + with open(log_file, 'a') as fl: + for batch_idx, (subject) in enumerate( + tqdm(train_dataloader, file = fl, desc="Looping over training data") + ): + optimizer.zero_grad() + image = ( + torch.cat( + [subject[key][torchio.DATA] for key in params["channel_keys"]], dim=1 + ) + .float() + .to(params["device"]) ) - else: - label = subject["label"][torchio.DATA] - label = label.to(params["device"]) + if "value_keys" in params: + label = torch.cat([subject[key] for key in params["value_keys"]], dim=0) + # min is needed because for certain cases, batch size becomes smaller than the total remaining labels + label = label.reshape( + min(params["batch_size"], len(label)), len(params["value_keys"]), + ) + else: + label = subject["label"][torchio.DATA] + label = label.to(params["device"]) if params["save_training"]: write_training_patches(subject, params) - # ensure spacing is always present in params and is always subject-specific - if "spacing" in subject: - params["subject_spacing"] = subject["spacing"] - else: - params["subject_spacing"] = None - loss, calculated_metrics, output, _ = step(model, image, label, params) - # store predictions for classification - if calculate_overall_metrics: - predictions_array[ - batch_idx - * params["batch_size"] : (batch_idx + 1) - * params["batch_size"] - ] = (torch.argmax(output[0], 0).cpu().item()) - - nan_loss = torch.isnan(loss) - second_order = ( - hasattr(optimizer, "is_second_order") and optimizer.is_second_order - ) - if params["model"]["amp"]: - with torch.cuda.amp.autocast(): - # if loss is nan, don't backprop and don't step optimizer + # ensure spacing is always present in params and is always subject-specific + if "spacing" in subject: + params["subject_spacing"] = subject["spacing"] + else: + params["subject_spacing"] = None + loss, calculated_metrics, output, _ = step(model, image, label, params) + # store predictions for classification + if calculate_overall_metrics: + predictions_array[ + batch_idx + * params["batch_size"] : (batch_idx + 1) + * params["batch_size"] + ] = (torch.argmax(output[0], 0).cpu().item()) + + nan_loss = torch.isnan(loss) + second_order = ( + hasattr(optimizer, "is_second_order") and optimizer.is_second_order + ) + if params["model"]["amp"]: + with torch.cuda.amp.autocast(): + # if loss is nan, don't backprop and don't step optimizer + if not nan_loss: + scaler( + loss=loss, + optimizer=optimizer, + clip_grad=params["clip_grad"], + clip_mode=params["clip_mode"], + parameters=model_parameters_exclude_head( + model, clip_mode=params["clip_mode"] + ), + create_graph=second_order, + ) + else: if not nan_loss: - scaler( - loss=loss, - optimizer=optimizer, - clip_grad=params["clip_grad"], - clip_mode=params["clip_mode"], - parameters=model_parameters_exclude_head( - model, clip_mode=params["clip_mode"] - ), - create_graph=second_order, - ) - else: + loss.backward(create_graph=second_order) + if params["clip_grad"] is not None: + dispatch_clip_grad_( + parameters=model_parameters_exclude_head( + model, clip_mode=params["clip_mode"] + ), + value=params["clip_grad"], + mode=params["clip_mode"], + ) + optimizer.step() + + # Non network training related if not nan_loss: - loss.backward(create_graph=second_order) - if params["clip_grad"] is not None: - dispatch_clip_grad_( - parameters=model_parameters_exclude_head( - model, clip_mode=params["clip_mode"] - ), - value=params["clip_grad"], - mode=params["clip_mode"], - ) - optimizer.step() - - # Non network training related - if not nan_loss: - total_epoch_train_loss += loss.detach().cpu().item() - for metric in calculated_metrics.keys(): - if isinstance(total_epoch_train_metric[metric], list): - if len(total_epoch_train_metric[metric]) == 0: - total_epoch_train_metric[metric] = np.array( - calculated_metrics[metric] - ) + total_epoch_train_loss += loss.detach().cpu().item() + for metric in calculated_metrics.keys(): + if isinstance(total_epoch_train_metric[metric], list): + if len(total_epoch_train_metric[metric]) == 0: + total_epoch_train_metric[metric] = np.array( + calculated_metrics[metric] + ) + else: + total_epoch_train_metric[metric] += np.array( + calculated_metrics[metric] + ) else: - total_epoch_train_metric[metric] += np.array( - calculated_metrics[metric] - ) - else: - total_epoch_train_metric[metric] += calculated_metrics[metric] + total_epoch_train_metric[metric] += calculated_metrics[metric] - if params["verbose"]: - # For printing information at halftime during an epoch if ((batch_idx + 1) % (len(train_dataloader) / 2) == 0) and ( (batch_idx + 1) < len(train_dataloader) ): - print( - "\nHalf-Epoch Average train loss : ", - total_epoch_train_loss / (batch_idx + 1), + logger.debug( + f"\nHalf-Epoch Average train loss : {total_epoch_train_loss / (batch_idx + 1)}" ) for metric in params["metrics"]: if isinstance(total_epoch_train_metric[metric], np.ndarray): @@ -186,10 +190,12 @@ def train_network( ).tolist() else: to_print = total_epoch_train_metric[metric] / (batch_idx + 1) - print("Half-Epoch Average train " + metric + " : ", to_print) + logger.debug( + f"Half-Epoch Average train {metric}: {to_print}" + ) average_epoch_train_loss = total_epoch_train_loss / len(train_dataloader) - print(" Epoch Final train loss : ", average_epoch_train_loss) + logger.info(f"Epoch Final train loss : {average_epoch_train_loss}") # get overall stats for classification if calculate_overall_metrics: @@ -228,6 +234,11 @@ def training_loop( testing_data (pd.DataFrame): The data to use for testing. epochs (int): The number of epochs to train; if None, take from params. """ + if "logger_name" in params: + logger = logging.getLogger(params["logger_name"]) + else: + logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params["verbose"]) + # Some autodetermined factors if epochs is None: epochs = params["num_epochs"] @@ -265,7 +276,7 @@ def training_loop( params["previous_parameters"] = main_dict.get("parameters", None) # Defining our model here according to parameters mentioned in the configuration file - print("Number of channels : ", params["model"]["num_channels"]) + logger.debug(f"Number of channels : {params['model']['num_channels']}") ( model, @@ -290,7 +301,7 @@ def training_loop( model_paths["initial"], onnx_export=False, ) - print("Initial model saved.") + logger.debug("Initial model saved.") # if previous model file is present, load it up if main_dict is not None: @@ -300,7 +311,7 @@ def training_loop( optimizer.load_state_dict(main_dict["optimizer_state_dict"]) best_loss = main_dict["loss"] params["previous_parameters"] = main_dict.get("parameters", None) - print("Previous model successfully loaded.") + logger.debug("Previous model successfully loaded.") except RuntimeWarning: RuntimeWarning("Previous model could not be loaded, initializing model") @@ -320,10 +331,10 @@ def training_loop( start_time = time.time() if not (os.environ.get("HOSTNAME") is None): - print("Hostname :", os.environ.get("HOSTNAME")) + logger.debug(f"Hostname : {os.environ.get('HOSTNAME')}") # datetime object containing current date and time - print("Initializing training at :", get_date_time(), flush=True) + print(f"Initializing training at : {get_date_time()}") calculate_overall_metrics = (params["problem_type"] == "classification") or ( params["problem_type"] == "regression" @@ -345,24 +356,24 @@ def training_loop( if metric not in metrics_log: metrics_log[metric] = 0 - # Setup a few loggers for tracking - train_logger = Logger( + # Setup logging to csv files + train_csv_logger = CSVLogger( logger_csv_filename=os.path.join(output_dir, "logs_training.csv"), metrics=metrics_log, ) - valid_logger = Logger( + valid_csv_logger = CSVLogger( logger_csv_filename=os.path.join(output_dir, "logs_validation.csv"), metrics=metrics_log, ) if testingDataDefined: - test_logger = Logger( + test_csv_logger = CSVLogger( logger_csv_filename=os.path.join(output_dir, "logs_testing.csv"), metrics=metrics_log, ) - train_logger.write_header(mode="train") - valid_logger.write_header(mode="valid") + train_csv_logger.write_header(mode="train") + valid_csv_logger.write_header(mode="valid") if testingDataDefined: - test_logger.write_header(mode="test") + test_csv_logger.write_header(mode="test") if "medcam" in params: model = medcam.inject( @@ -378,7 +389,7 @@ def training_loop( ) params["medcam_enabled"] = False - print("Using device:", device, flush=True) + logger.debug(f"Using device: {device}") # Iterate for number of epochs for epoch in range(start_epoch, epochs): @@ -427,8 +438,7 @@ def training_loop( print("*" * 20) print("*" * 20) print("Starting Epoch : ", epoch) - if params["verbose"]: - print("Epoch start time : ", get_date_time()) + logger.debug(f"Epoch start time : {get_date_time()}") params["current_epoch"] = epoch @@ -441,25 +451,19 @@ def training_loop( patience += 1 - # Write the losses to a logger - train_logger.write(epoch, epoch_train_loss, epoch_train_metric) - valid_logger.write(epoch, epoch_valid_loss, epoch_valid_metric) + # Write the losses to a csv logger + train_csv_logger.write(epoch, epoch_train_loss, epoch_train_metric) + valid_csv_logger.write(epoch, epoch_valid_loss, epoch_valid_metric) if testingDataDefined: epoch_test_loss, epoch_test_metric = validate_network( model, test_dataloader, scheduler, params, epoch, mode="testing" ) - test_logger.write(epoch, epoch_test_loss, epoch_test_metric) + test_csv_logger.write(epoch, epoch_test_loss, epoch_test_metric) - if params["verbose"]: - print("Epoch end time : ", get_date_time()) + logger.debug(f"Epoch end time : {get_date_time()}") epoch_end_time = time.time() - print( - "Time taken for epoch : ", - (epoch_end_time - epoch_start_time) / 60, - " mins", - flush=True, - ) + logger.info(f"Time taken for epoch : {(epoch_end_time - epoch_start_time) / 60} mins") model_dict = get_model_dict(model, params["device_id"]) diff --git a/GANDLF/config_manager.py b/GANDLF/config_manager.py index 49fda1b58..06d641454 100644 --- a/GANDLF/config_manager.py +++ b/GANDLF/config_manager.py @@ -1,5 +1,5 @@ from typing import Optional, Union -import sys, yaml, ast, pkg_resources +import sys, yaml, ast, pkg_resources, warnings import numpy as np from copy import deepcopy @@ -70,8 +70,8 @@ def initialize_parameter( params[parameter_to_initialize] ) else: - print( - "WARNING: Initializing '" + parameter_to_initialize + "' as " + str(value) + warnings.warn( + "Initializing '" + parameter_to_initialize + "' as " + str(value) ) params[parameter_to_initialize] = value @@ -157,9 +157,8 @@ def _parseConfig( assert "patch_size" in params, "Patch size needs to be defined in the config file" if "resize" in params: - print( - "WARNING: 'resize' should be defined under 'data_processing', this will be skipped", - file=sys.stderr, + warnings.warn( + "'resize' should be defined under 'data_processing', this will be skipped" ) assert "modality" in params, "'modality' needs to be defined in the config file" @@ -413,9 +412,8 @@ def _parseConfig( if isinstance(default_downsampling, list): if len(default_downsampling) != 2: initialize_downsampling = True - print( - "WARNING: 'anisotropic' augmentation needs to be either a single number of a list of 2 numbers: https://torchio.readthedocs.io/transforms/augmentation.html?highlight=randomswap#torchio.transforms.RandomAnisotropy.", - file=sys.stderr, + warnings.warn( + "'anisotropic' augmentation needs to be either a single number of a list of 2 numbers: https://torchio.readthedocs.io/transforms/augmentation.html?highlight=randomswap#torchio.transforms.RandomAnisotropy." ) default_downsampling = default_downsampling[0] # only else: @@ -423,9 +421,8 @@ def _parseConfig( if initialize_downsampling: if default_downsampling < 1: - print( - "WARNING: 'anisotropic' augmentation needs the 'downsampling' parameter to be greater than 1, defaulting to 1.5.", - file=sys.stderr, + warnings.warn( + "'anisotropic' augmentation needs the 'downsampling' parameter to be greater than 1, defaulting to 1.5." ) # default params["data_augmentation"]["anisotropic"]["downsampling"] = 1.5 @@ -473,9 +470,8 @@ def _parseConfig( if key in params["data_preprocessing"]: params["data_preprocessing"].pop(key) - print( - "WARNING: Different 'resize' operations are ignored as 'resample' is defined under 'data_processing'", - file=sys.stderr, + warnings.warn( + "Different 'resize' operations are ignored as 'resample' is defined under 'data_processing'" ) # iterate through all keys @@ -551,7 +547,7 @@ def _parseConfig( if "amp" in params["model"]: pass else: - print("NOT using Mixed Precision Training") + warnings.warn("NOT using Mixed Precision Training") params["model"]["amp"] = False if "norm_type" in params["model"]: @@ -564,21 +560,20 @@ def _parseConfig( "Normalization type cannot be 'None' for non-VGG architectures" ) else: - print("WARNING: Initializing 'norm_type' as 'batch'", flush=True) + warnings.warn("Initializing 'norm_type' as 'batch'", flush=True) params["model"]["norm_type"] = "batch" if not ("base_filters" in params["model"]): base_filters = 32 params["model"]["base_filters"] = base_filters - print("Using default 'base_filters' in 'model': ", base_filters) + warnings.warn(f"Using default 'base_filters' in 'model': {base_filters}") if not ("class_list" in params["model"]): params["model"]["class_list"] = [] # ensure that this is initialized if not ("ignore_label_validation" in params["model"]): params["model"]["ignore_label_validation"] = None if "batch_norm" in params["model"]: - print( - "WARNING: 'batch_norm' is no longer supported, please use 'norm_type' in 'model' instead", - flush=True, + warnings.warn( + "'batch_norm' is no longer supported, please use 'norm_type' in 'model' instead" ) params["model"]["print_summary"] = params["model"].get("print_summary", True) @@ -601,8 +596,8 @@ def _parseConfig( params["model"]["save_at_every_epoch"] = False if params["model"]["save_at_every_epoch"]: - print( - "WARNING: 'save_at_every_epoch' will result in TREMENDOUS storage usage; use at your own risk." + warnings.warn( + "'save_at_every_epoch' will result in TREMENDOUS storage usage; use at your own risk." ) if isinstance(params["model"]["class_list"], str): @@ -610,8 +605,8 @@ def _parseConfig( "&&" in params["model"]["class_list"] ): # special case for multi-class computation - this needs to be handled during one-hot encoding mask construction - print( - "WARNING: This is a special case for multi-class computation, where different labels are processed together, `reverse_one_hot` will need mapping information to work correctly" + warnings.warn( + "This is a special case for multi-class computation, where different labels are processed together, `reverse_one_hot` will need mapping information to work correctly" ) temp_classList = params["model"]["class_list"] # we don't need the brackets @@ -649,7 +644,7 @@ def _parseConfig( params["parallel_compute_command"] = parallel_compute_command if "opt" in params: - print("DeprecationWarning: 'opt' has been superseded by 'optimizer'") + warnings.warn("DeprecationWarning: 'opt' has been superseded by 'optimizer'") params["optimizer"] = params["opt"] # initialize defaults for patch sampler @@ -663,8 +658,8 @@ def _parseConfig( if "patch_sampler" in params: # if "patch_sampler" is a string, then it is the type of sampler if isinstance(params["patch_sampler"], str): - print( - "WARNING: Defining 'patch_sampler' as a string will be deprecated in a future release, please use a dictionary instead" + warnings.warn( + "Defining 'patch_sampler' as a string will be deprecated in a future release, please use a dictionary instead" ) temp_patch_sampler_dict["type"] = params["patch_sampler"].lower() elif isinstance(params["patch_sampler"], dict): @@ -698,8 +693,8 @@ def _parseConfig( if not ("step_size" in params["scheduler"]): params["scheduler"]["step_size"] = params["learning_rate"] / 5.0 - print( - "WARNING: Setting default step_size to:", params["scheduler"]["step_size"] + warnings.warn( + f"Setting default step_size to: {params['scheduler']['step_size']}" ) # initialize default optimizer diff --git a/GANDLF/logger.py b/GANDLF/csv_logger.py similarity index 95% rename from GANDLF/logger.py rename to GANDLF/csv_logger.py index bb3168583..5786fa732 100755 --- a/GANDLF/logger.py +++ b/GANDLF/csv_logger.py @@ -11,10 +11,10 @@ import torch -class Logger: +class CSVLogger: def __init__(self, logger_csv_filename: str, metrics: Dict[str, float]) -> None: """ - Logger class to log the training and validation metrics to a csv file. + CSVLogger class to log the training and validation metrics to a csv file. Args: logger_csv_filename (str): Path to a filename where the csv has to be stored. diff --git a/GANDLF/data/ImagesFromDataFrame.py b/GANDLF/data/ImagesFromDataFrame.py index 39bc9f8cb..c85ec1b4f 100644 --- a/GANDLF/data/ImagesFromDataFrame.py +++ b/GANDLF/data/ImagesFromDataFrame.py @@ -2,6 +2,7 @@ import os from pathlib import Path import numpy as np +import logging import pandas import torch @@ -15,6 +16,7 @@ resize_image, get_filename_extension_sanitized, get_correct_padding_size, + setup_logger ) from .preprocessing import get_transforms_for_preprocessing from .augmentation import global_augs_dict @@ -80,6 +82,12 @@ def ImagesFromDataFrame( predictionHeaders = headers["predictionHeaders"] subjectIDHeader = headers["subjectIDHeader"] + if "logger_name" in parameters: + logger = logging.getLogger(parameters["logger_name"]) + else: + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + + resize_images_flag = False # if resize has been defined but resample is not (or is none) if not (preprocessing is None): @@ -118,141 +126,142 @@ def _save_resized_images( if not os.path.isfile(save_path): sitk.WriteImage(resized_image, save_path) - # iterating through the dataframe - for patient in tqdm( - range(num_row), desc="Constructing queue for " + loader_type + " data" - ): - # We need this dict for storing the meta data for each subject - # such as different image modalities, labels, any other data - subject_dict = {} - subject_dict["subject_id"] = str(dataframe[subjectIDHeader][patient]) - skip_subject = False - # iterating through the channels/modalities/timepoints of the subject - for channel in channelHeaders: - # sanity check for malformed csv - if not os.path.isfile(str(dataframe[channel][patient])): - skip_subject = True - - subject_dict[str(channel)] = torchio.ScalarImage( - dataframe[channel][patient] - ) - - # store image spacing information if not present - if "spacing" not in subject_dict: - file_reader = sitk.ImageFileReader() - file_reader.SetFileName(str(dataframe[channel][patient])) - file_reader.ReadImageInformation() - subject_dict["spacing"] = torch.Tensor(file_reader.GetSpacing()) - - # if resize_image is requested, the perform per-image resize with appropriate interpolator - if resize_images_flag: - img_resized = resize_image( - subject_dict[str(channel)].as_sitk(), preprocessing["resize_image"] + # iterating through the dataframe and sending logs to file + log_file = os.path.join(parameters["logs_dir"], "data_loop.log") + with open(log_file, 'a') as fl: + for patient in tqdm( + range(num_row), file = fl, desc="Constructing queue for " + loader_type + " data" + ): + # We need this dict for storing the meta data for each subject + # such as different image modalities, labels, any other data + subject_dict = {} + subject_dict["subject_id"] = str(dataframe[subjectIDHeader][patient]) + skip_subject = False + # iterating through the channels/modalities/timepoints of the subject + for channel in channelHeaders: + # sanity check for malformed csv + if not os.path.isfile(str(dataframe[channel][patient])): + skip_subject = True + + subject_dict[str(channel)] = torchio.ScalarImage( + dataframe[channel][patient] ) - if parameters["memory_save_mode"]: - _save_resized_images( - img_resized, - parameters["output_dir"], - subject_dict["subject_id"], - str(channel), - loader_type, - get_filename_extension_sanitized( - str(dataframe[channel][patient]) - ), + + # store image spacing information if not present + if "spacing" not in subject_dict: + file_reader = sitk.ImageFileReader() + file_reader.SetFileName(str(dataframe[channel][patient])) + file_reader.ReadImageInformation() + subject_dict["spacing"] = torch.Tensor(file_reader.GetSpacing()) + + # if resize_image is requested, the perform per-image resize with appropriate interpolator + if resize_images_flag: + img_resized = resize_image( + subject_dict[str(channel)].as_sitk(), preprocessing["resize_image"] ) - else: - # always ensure resized image spacing is used - subject_dict["spacing"] = torch.Tensor(img_resized.GetSpacing()) - subject_dict[str(channel)] = torchio.ScalarImage.from_sitk( - img_resized + if parameters["memory_save_mode"]: + _save_resized_images( + img_resized, + parameters["output_dir"], + subject_dict["subject_id"], + str(channel), + loader_type, + get_filename_extension_sanitized( + str(dataframe[channel][patient]) + ), + ) + else: + # always ensure resized image spacing is used + subject_dict["spacing"] = torch.Tensor(img_resized.GetSpacing()) + subject_dict[str(channel)] = torchio.ScalarImage.from_sitk( + img_resized + ) + + # # for regression -- this logic needs to be thought through + # if predictionHeaders: + # # get the mask + # if (subject_dict['label'] is None) and (class_list is not None): + # sys.exit('The \'class_list\' parameter has been defined but a label file is not present for patient: ', patient) + + if labelHeader is not None: + if not os.path.isfile(str(dataframe[labelHeader][patient])): + skip_subject = True + + subject_dict["label"] = torchio.LabelMap(dataframe[labelHeader][patient]) + subject_dict["path_to_metadata"] = str(dataframe[labelHeader][patient]) + + # if resize is requested, the perform per-image resize with appropriate interpolator + if resize_images_flag: + img_resized = resize_image( + subject_dict["label"].as_sitk(), + preprocessing["resize_image"], + sitk.sitkNearestNeighbor, ) - - # # for regression -- this logic needs to be thought through - # if predictionHeaders: - # # get the mask - # if (subject_dict['label'] is None) and (class_list is not None): - # sys.exit('The \'class_list\' parameter has been defined but a label file is not present for patient: ', patient) - - if labelHeader is not None: - if not os.path.isfile(str(dataframe[labelHeader][patient])): - skip_subject = True - - subject_dict["label"] = torchio.LabelMap(dataframe[labelHeader][patient]) - subject_dict["path_to_metadata"] = str(dataframe[labelHeader][patient]) - - # if resize is requested, the perform per-image resize with appropriate interpolator - if resize_images_flag: - img_resized = resize_image( - subject_dict["label"].as_sitk(), - preprocessing["resize_image"], - sitk.sitkNearestNeighbor, + if parameters["memory_save_mode"]: + _save_resized_images( + img_resized, + parameters["output_dir"], + subject_dict["subject_id"], + "label", + loader_type, + get_filename_extension_sanitized( + str(dataframe[channel][patient]) + ), + ) + else: + subject_dict["label"] = torchio.LabelMap.from_sitk(img_resized) + + else: + subject_dict["label"] = "NA" + subject_dict["path_to_metadata"] = str(dataframe[channel][patient]) + + # iterating through the values to predict of the subject + valueCounter = 0 + for values in predictionHeaders: + # assigning the dict key to the channel + subject_dict["value_" + str(valueCounter)] = np.array( + dataframe[values][patient] ) - if parameters["memory_save_mode"]: - _save_resized_images( - img_resized, - parameters["output_dir"], - subject_dict["subject_id"], - "label", - loader_type, - get_filename_extension_sanitized( - str(dataframe[channel][patient]) - ), - ) - else: - subject_dict["label"] = torchio.LabelMap.from_sitk(img_resized) - - else: - subject_dict["label"] = "NA" - subject_dict["path_to_metadata"] = str(dataframe[channel][patient]) - - # iterating through the values to predict of the subject - valueCounter = 0 - for values in predictionHeaders: - # assigning the dict key to the channel - subject_dict["value_" + str(valueCounter)] = np.array( - dataframe[values][patient] - ) - valueCounter += 1 - - # skip subject the condition was tripped - if not skip_subject: - # Initializing the subject object using the dict - subject = torchio.Subject(subject_dict) - # https://github.com/fepegar/torchio/discussions/587#discussioncomment-928834 - # this is causing memory usage to explode, see https://github.com/mlcommons/GaNDLF/issues/128 - if parameters["verbose"]: - print( + valueCounter += 1 + + # skip subject the condition was tripped + if not skip_subject: + # Initializing the subject object using the dict + subject = torchio.Subject(subject_dict) + # https://github.com/fepegar/torchio/discussions/587#discussioncomment-928834 + # this is causing memory usage to explode, see https://github.com/mlcommons/GaNDLF/issues/128 + logger.debug( "Checking consistency of images in subject '" + subject["subject_id"] + "'" ) - try: - perform_sanity_check_on_subject(subject, parameters) - except Exception as exception: - subjects_with_error.append(subject["subject_id"]) - print( - "Subject '" - + subject["subject_id"] - + "' could not be loaded due to the following exception: {}".format( - type(exception).__name__ + try: + perform_sanity_check_on_subject(subject, parameters) + except Exception as exception: + subjects_with_error.append(subject["subject_id"]) + print( + "Subject '" + + subject["subject_id"] + + "' could not be loaded due to the following exception: {}".format( + type(exception).__name__ + ) + + "; message: {}".format(exception) ) - + "; message: {}".format(exception) - ) - # # padding image, but only for label sampler, because we don't want to pad for uniform - if sampler["enable_padding"]: - psize_pad = get_correct_padding_size( - patch_size, parameters["model"]["dimension"] - ) - padder = Pad(psize_pad, padding_mode=sampler["padding_mode"]) - subject = padder(subject) + # # padding image, but only for label sampler, because we don't want to pad for uniform + if sampler["enable_padding"]: + psize_pad = get_correct_padding_size( + patch_size, parameters["model"]["dimension"] + ) + padder = Pad(psize_pad, padding_mode=sampler["padding_mode"]) + subject = padder(subject) - # load subject into memory: https://github.com/fepegar/torchio/discussions/568#discussioncomment-859027 - if in_memory: - subject.load() + # load subject into memory: https://github.com/fepegar/torchio/discussions/568#discussioncomment-859027 + if in_memory: + subject.load() - # Appending this subject to the list of subjects - subjects_list.append(subject) + # Appending this subject to the list of subjects + subjects_list.append(subject) assert ( subjects_with_error is not None diff --git a/GANDLF/metrics/generic.py b/GANDLF/metrics/generic.py index 2779de626..50f8f0b5a 100644 --- a/GANDLF/metrics/generic.py +++ b/GANDLF/metrics/generic.py @@ -1,4 +1,4 @@ -import torch +import torch, warnings from torchmetrics import ( Metric, F1Score, @@ -31,8 +31,8 @@ def generic_function_output_with_check( torch.Tensor: The output of the metric function. """ if torch.min(prediction) < 0: - print( - "WARNING: Negative values detected in prediction, cannot compute torchmetrics calculations." + warnings.warn( + "Negative values detected in prediction, cannot compute torchmetrics calculations." ) return torch.zeros((1), device=prediction.device) else: diff --git a/GANDLF/models/densenet.py b/GANDLF/models/densenet.py index b293f8830..43deaf40d 100644 --- a/GANDLF/models/densenet.py +++ b/GANDLF/models/densenet.py @@ -1,6 +1,7 @@ # adapted from https://github.com/kenshohara/3D-ResNets-PyTorch import sys +import warnings import torch import torch.nn as nn import torch.nn.functional as F @@ -174,8 +175,8 @@ def __init__(self, parameters: dict, block_config=(6, 12, 24, 16)): if not ("no_max_pool" in parameters): parameters["no_max_pool"] = False if self.Norm is None: - sys.stderr.write( - "Warning: densenet is not defined without a normalization layer" + warnings.warn( + "densenet is not defined without a normalization layer" ) self.Norm = self.BatchNorm diff --git a/GANDLF/models/efficientnet.py b/GANDLF/models/efficientnet.py index 57bba9dcb..db3a4b606 100644 --- a/GANDLF/models/efficientnet.py +++ b/GANDLF/models/efficientnet.py @@ -1,4 +1,5 @@ import sys +import warnings import torch.nn as nn import torch.nn.functional as F from collections import OrderedDict @@ -397,8 +398,8 @@ def __init__(self, parameters: dict, scale_params): # how to scale depth and wi else: sys.exit("Only 2D or 3D convolutions are supported.") if self.Norm is None: - sys.stderr.write( - "Warning: efficientnet is not defined without a normalization layer" + warnings.warn( + "efficientnet is not defined without a normalization layer" ) self.Norm = self.BatchNorm diff --git a/GANDLF/models/imagenet_unet.py b/GANDLF/models/imagenet_unet.py index 940987e1f..fa7f2b7ca 100644 --- a/GANDLF/models/imagenet_unet.py +++ b/GANDLF/models/imagenet_unet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # adapted from https://github.com/qubvel/segmentation_models.pytorch from typing import Optional, Union, List -import torch +import torch, warnings import torch.nn as nn from segmentation_models_pytorch.base import SegmentationHead, ClassificationHead @@ -142,8 +142,8 @@ def _get_fixed_encoder_channels(name, in_channels): out_channels = self.encoder.out_channels if "mit_" in encoder_name: # MixVision Transformers only support 3-channel inputs - print( - "WARNING: MixVision Transformers only support 3 channels, adding an extra Conv layer for compatibility." + warnings.warn( + "MixVision Transformers only support 3 channels, adding an extra Conv layer for compatibility." ) self.pre_encoder = nn.Conv2d(in_channels, 3, kernel_size=1, stride=1) modules = [] diff --git a/GANDLF/models/resnet.py b/GANDLF/models/resnet.py index e24dca221..81b1b3b00 100644 --- a/GANDLF/models/resnet.py +++ b/GANDLF/models/resnet.py @@ -1,4 +1,4 @@ -import sys +import sys, warnings import torch.nn as nn import torch.nn.functional as F from collections import OrderedDict @@ -27,7 +27,7 @@ def __init__(self, parameters: dict, blockType, block_config): # Display warning message if patch size is not large enough for desired number of layers if allowedLay != len(block_config) and allowedLay >= 1: - print( + warnings.warn( "The patch size is not large enough for the desired number of layers.", " It is expected that each dimension of the patch size is 2^(layers + 1)*i, where i is an integer greater than 2.", "Only the first %d layers will run." % allowedLay, @@ -56,8 +56,8 @@ def __init__(self, parameters: dict, blockType, block_config): # If normalization layer is not defined, use Batch Normalization if self.Norm is None: - sys.stderr.write( - "Warning: resnet is not defined without a normalization layer" + warnings.warn( + "resnet is not defined without a normalization layer" ) self.Norm = self.BatchNorm diff --git a/GANDLF/models/sdnet.py b/GANDLF/models/sdnet.py index 76968e344..8ba63adf1 100644 --- a/GANDLF/models/sdnet.py +++ b/GANDLF/models/sdnet.py @@ -1,4 +1,5 @@ -import typing, sys +import typing +import warnings import torch import torch.nn as nn import torch.nn.functional as F @@ -316,9 +317,8 @@ def __init__(self, parameters: dict): self.modality_factors = 8 if parameters["patch_size"] != [224, 224, 1]: - print( - "WARNING: The patch size is not 224x224, which is required for sdnet. Using default patch size instead", - file=sys.stderr, + warnings.warn( + "The patch size is not 224x224, which is required for sdnet. Using default patch size instead" ) parameters["patch_size"] = [224, 224, 1] diff --git a/GANDLF/training_manager.py b/GANDLF/training_manager.py index e605af6f6..8c58da6bb 100644 --- a/GANDLF/training_manager.py +++ b/GANDLF/training_manager.py @@ -1,9 +1,9 @@ import pandas as pd -import os, pickle, shutil +import os, pickle, shutil, logging from pathlib import Path from GANDLF.compute import training_loop -from GANDLF.utils import get_dataframe, split_data +from GANDLF.utils import get_dataframe, setup_logger, split_data def TrainingManager( @@ -25,6 +25,12 @@ def TrainingManager( resume (bool): Whether the previous run will be resumed or not. reset (bool): Whether the previous run will be reset or not. """ + + if "logger_name" in parameters: + logger = logging.getLogger(parameters["logger_name"]) + else: + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters["verbose"]) + if reset: shutil.rmtree(outputDir) Path(outputDir).mkdir(parents=True, exist_ok=True) @@ -36,10 +42,8 @@ def TrainingManager( pickle.dump(parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) else: if os.path.exists(currentModelConfigPickle): - print( - "Using previously saved parameter file", - currentModelConfigPickle, - flush=True, + logger.debug( + f"Using previously saved parameter file {currentModelConfigPickle}" ) parameters = pickle.load(open(currentModelConfigPickle, "rb")) diff --git a/GANDLF/utils/__init__.py b/GANDLF/utils/__init__.py index 66d830d3d..34450b1ad 100644 --- a/GANDLF/utils/__init__.py +++ b/GANDLF/utils/__init__.py @@ -68,3 +68,7 @@ ) from .data_splitter import split_data + +from .logger import ( + setup_logger, +) diff --git a/GANDLF/utils/logger.py b/GANDLF/utils/logger.py new file mode 100644 index 000000000..8c053c91f --- /dev/null +++ b/GANDLF/utils/logger.py @@ -0,0 +1,51 @@ +import logging, os, warnings +from typing import Optional, Tuple +from pathlib import Path + +def warning_on_one_line(message, category, filename, lineno, file=None, line=None): + """ + This function formats warning message according to its type + """ + if category == UserWarning: + return str(message) + else: + return '%s:%s: %s:%s' % (filename, lineno, category.__name__, message) + + +def setup_logger(output_dir: str, verbose: Optional[bool] = False) -> Tuple[logging.Logger, str, str]: + """ + This function setups a logger with severity level controlled by verbose parameter from a config file. + + Args: + logger_name (str): Name for a logger + logs_dir (str): Output directory for log files. + verbose (Optional[bool], optional): Used to setup the logging level. Defaults to False. + + Returns: + logger (logging.Logger) + logger_dir (str): directory for the logs + logger_name (str): name of the logger + """ + logs_dir = f'{output_dir}/logs' + logger_name = 'gandlf' + Path(logs_dir).mkdir(parents=True, exist_ok=True) + + warnings.formatwarning = warning_on_one_line + logging.captureWarnings(True) + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + + # create file handler which logs messages with severity determined by verbose param + fh = logging.FileHandler(os.path.join(logs_dir, "gandlf.log")) + fh.setLevel(logging.DEBUG) if verbose else fh.setLevel(logging.WARNING) + warnings_logger = logging.getLogger("py.warnings") + + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + + # add the handlers to logger + logger.addHandler(fh) + warnings_logger.addHandler(fh) + + return logger, logs_dir, logger_name diff --git a/GANDLF/utils/modelio.py b/GANDLF/utils/modelio.py index a83a6d952..2d18c71d7 100644 --- a/GANDLF/utils/modelio.py +++ b/GANDLF/utils/modelio.py @@ -1,10 +1,11 @@ import hashlib, os from typing import Any, Dict, Optional, Tuple -import torch +import torch, logging from ..version import __version__ from .generic import get_unique_timestamp, get_git_hash +from GANDLF.utils.logger import setup_logger # these are the base keys for the model dictionary to save model_dict_full = { @@ -37,6 +38,11 @@ def optimize_and_save_model( path (str): The path to save the model dictionary to. onnx_export (Optional[bool]): Whether to export to ONNX and OpenVINO. Defaults to True. """ + if "logger_name" in params: + logger = logging.getLogger(params["logger_name"]) + else: + logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", True)) + # Check if ONNX export is enabled in the parameter dictionary onnx_export = params["model"].get("onnx_export", onnx_export) @@ -50,12 +56,12 @@ def optimize_and_save_model( if not onnx_export: # Print a warning if ONNX export is disabled and not already warned if "onnx_print" not in params: - print("WARNING: Current model is not supported by ONNX/OpenVINO!") + logger.warning("Current model is not supported by ONNX/OpenVINO!") params["onnx_print"] = True return else: try: - print("Optimizing the best model.") + logger.debug("Optimizing the best model.") num_channel = params["model"]["num_channels"] model_dimension = params["model"]["dimension"] input_shape = params["patch_size"] @@ -85,7 +91,7 @@ def optimize_and_save_model( output_names=["output"], ) except RuntimeWarning: - print("WARNING: Cannot export to ONNX model.") + logger.warning("Cannot export to ONNX model.") return # Check if OpenVINO is present and try to convert the ONNX model @@ -100,7 +106,7 @@ def optimize_and_save_model( if "2023.0.1" in get_version(): openvino_present = True except ImportError: - print("WARNING: OpenVINO is not present.") + logger.warning("OpenVINO is not present.") if openvino_present: xml_path = onnx_path.replace("onnx", "xml") @@ -124,7 +130,7 @@ def optimize_and_save_model( ) ov.runtime.serialize(ov_model, xml_path=xml_path, bin_path=bin_path) except Exception as e: - print("WARNING: OpenVINO Model Optimizer IR conversion failed: " + e) + logger.warning("OpenVINO Model Optimizer IR conversion failed: " + e) def save_model( diff --git a/GANDLF/utils/tensor.py b/GANDLF/utils/tensor.py index 9dfd3cb9c..9bbe85368 100644 --- a/GANDLF/utils/tensor.py +++ b/GANDLF/utils/tensor.py @@ -1,4 +1,4 @@ -import os, sys +import os, sys, warnings from typing import List, Optional, Tuple, Union from pandas.util import hash_pandas_object import numpy as np @@ -279,8 +279,8 @@ def get_class_imbalance_weights_classification( for i in params["model"]["class_list"]: i = int(i) if i not in weight_dict: - print( - "WARNING: A class was found in 'class_list' that was not present in the training data, please re-check training data labels" + warnings.warn( + "A class was found in 'class_list' that was not present in the training data, please re-check training data labels" ) weight_dict[i] = sys.float_info.epsilon penalty_dict[i] = (1 + sys.float_info.epsilon) / weight_dict[i] diff --git a/GANDLF/utils/write_parse.py b/GANDLF/utils/write_parse.py index 03ee27552..47028dc4c 100644 --- a/GANDLF/utils/write_parse.py +++ b/GANDLF/utils/write_parse.py @@ -1,6 +1,6 @@ import os import pathlib -import sys +import warnings from typing import Optional, Tuple, Union import pandas as pd @@ -124,9 +124,8 @@ def parseTrainingCSV( if headers["labelHeader"] is None: headers["labelHeader"] = currentHeaderLoc else: - print( - "WARNING: Multiple label headers found in training CSV, only the first one will be used", - file=sys.stderr, + warnings.warn( + "Multiple label headers found in training CSV, only the first one will be used" ) convert_relative_paths_in_dataframe(data_full, headers, inputTrainingCSVFile) return data_full, headers @@ -153,13 +152,12 @@ def parseTestingCSV( # and a new mapping_csv was created to be used and write the location of the new mapping_csv # and the collision_csv to the user if collision_status: - print( - """WARNING: Some patients with colliding subject_id were found. + warnings.warn( + """Some patients with colliding subject_id were found. A new mapping_csv was created to be used and a collision_csv was created to be used to map the old subject_id to the new subject_id. The location of the updated_test_mapping.csv and the collision.csv are: """ - + output_dir, - file=sys.stderr, + + output_dir ) return collision_status, data_full, headers diff --git a/docs/usage.md b/docs/usage.md index 2dcb79a58..36f785550 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -310,6 +310,8 @@ ${architecture_name}_best.pth.tar # the best model in native PyTorch format ${architecture_name}_latest.pth.tar # the latest model in native PyTorch format ${architecture_name}_initial.pth.tar # the initial model in native PyTorch format ${architecture_name}_initial.{onnx/xml/bin} # [optional] if ${architecture_name} is supported, the graph-optimized best model in ONNX format +logs/gandlf.log # log file containing logs from the application +logs/data_loop.log # log file showing looping through data # other files dependent on if training/validation/testing output was enabled in configuration ``` @@ -319,7 +321,12 @@ ${architecture_name}_initial.{onnx/xml/bin} # [optional] if ${architecture_name} - The predictions will be saved in the same directory as the model if `outputdir` is not passed to `gandlf_run`. - For segmentation, a directory will be created per subject ID in the input CSV. - For classification/regression, the predictions will be generated in the `outputdir` or `modeldir` as a CSV file. +- Logs directory will be generated in the `outputdir` or `modeldir` if not specified +### Logging levels +Logging levels are compatible with the [Python logging library](https://docs.python.org/3/library/logging.html#levels). +- `std.err` is set to `ERROR` level +- `gandlf.log` file is set to `DEBUG` level if `verbose` pararameter is `True`, otherwise set to `WARNING`. The default logging level is `WARNING`; `verbose` parameter can be specified in a configuration file. ## Plot the final results diff --git a/gandlf_run b/gandlf_run index 0186b69e0..ab6d4d295 100644 --- a/gandlf_run +++ b/gandlf_run @@ -6,6 +6,7 @@ import argparse import ast import sys import traceback +import warnings from GANDLF import version from GANDLF.cli import main_run, copyrightMessage @@ -114,8 +115,8 @@ if __name__ == "__main__": args.reset, args.resume = False, False if args.reset and args.resume: - print( - "WARNING: 'reset' and 'resume' are mutually exclusive; 'resume' will be used." + warnings.warn( + "reset' and 'resume' are mutually exclusive; 'resume' will be used." ) args.reset = False diff --git a/setup.py b/setup.py index 5e7d74c4b..be205a2a4 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,7 @@ def run(self): "zarr", "keyring", "monai==1.3.0", + "testfixtures", ] if __name__ == "__main__": diff --git a/testing/test_full.py b/testing/test_full.py index 8417a0352..1bd81d7d0 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -1,15 +1,16 @@ from pathlib import Path -import gdown, zipfile, os, csv, random, copy, shutil, yaml, torch, pytest +import gdown, zipfile, os, csv, random, copy, shutil, yaml, torch, pytest, logging import SimpleITK as sitk import numpy as np import pandas as pd +from testfixtures import log_capture from pydicom.data import get_testdata_file import cv2 from GANDLF.data.ImagesFromDataFrame import ImagesFromDataFrame from GANDLF.utils import * -from GANDLF.utils import parseTestingCSV, get_tensor_from_image +from GANDLF.utils import parseTestingCSV, get_tensor_from_image, setup_logger from GANDLF.data.preprocessing import global_preprocessing_dict from GANDLF.data.augmentation import global_augs_dict from GANDLF.data.patch_miner.opm.utils import ( @@ -3096,6 +3097,8 @@ def test_generic_cli_function_metrics_cli_rad_nd(): sanitize_outputDir() + print("passed") + def test_generic_deploy_metrics_docker(): print("50: Testing deployment of a metrics generator to Docker") @@ -3111,11 +3114,11 @@ def test_generic_deploy_metrics_docker(): ) assert result, "run_deployment returned false" + sanitize_outputDir() - + print("passed") - def test_generic_data_split(): print("51: Starting test for splitting and saving CSVs") # read and initialize parameters for specific data dimension @@ -3147,5 +3150,26 @@ def test_generic_data_split(): assert len(files_in_outputDir) == 15, "CSVs were not split correctly" sanitize_outputDir() + + print("passed") + +@log_capture() +def test_generic_logging(capture): + print("52: Testing logging setup") + logger, logs_dir, logger_name = setup_logger(outputDir, verbose=True) + + assert os.path.isdir(logs_dir), 'Directory for logs was not generated' + assert logger_name in logging.root.manager.loggerDict, f"Logger with name {logger_name} was not created" + + logger.info('a message') + logger.error('an error') + + capture.check( + (logger_name, "INFO", "a message"), + (logger_name, "ERROR", "an error") + ) + + sanitize_outputDir() + print("passed") From 083006726281fd1ab6eda61b6276149e6653768c Mon Sep 17 00:00:00 2001 From: sylwiamm Date: Wed, 3 Apr 2024 03:20:11 -0400 Subject: [PATCH 2/5] Slight improvements --- GANDLF/cli/main_run.py | 31 +++++++++++++------------------ GANDLF/compute/training_loop.py | 15 ++++++++++----- docs/usage.md | 14 ++++++++++++-- testing/test_full.py | 2 +- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/GANDLF/cli/main_run.py b/GANDLF/cli/main_run.py index e0a5b8dce..298df9ef8 100644 --- a/GANDLF/cli/main_run.py +++ b/GANDLF/cli/main_run.py @@ -1,4 +1,3 @@ -import yaml from typing import Optional from pathlib import Path @@ -42,6 +41,8 @@ def main_run( file_data_full = data_csv model_parameters = config_file device = device + parameters = ConfigManager(model_parameters) + parameters["device_id"] = -1 if train_mode: if resume: @@ -49,24 +50,18 @@ def main_run( "Trying to resume training without changing any parameters from previous run.", flush=True, ) - output_dir = model_dir - else: - if output_dir is None: - output_dir = model_dir - - Path(output_dir).mkdir(parents=True, exist_ok=True) + parameters["output_dir"] = model_dir + Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) - with open(config_file, 'r') as f: - config_params = yaml.safe_load(f) + # if the output directory is not specified, then use the model directory even for the testing data + # default behavior + parameters["output_dir"] = parameters.get("output_dir", output_dir) + if parameters["output_dir"] is None: + parameters["output_dir"] = model_dir + Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) # setup logger - logger, logs_dir, logger_name = setup_logger(output_dir=output_dir, verbose=config_params.get("verbose", False)) - - parameters = ConfigManager(model_parameters) - parameters["device_id"] = -1 - parameters["output_dir"] = output_dir - parameters["logs_dir"] = logs_dir - parameters["logger_name"] = logger_name + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=output_dir, verbose=parameters.get("verbose", False)) if "-1" in device: device = "cpu" @@ -82,7 +77,7 @@ def main_run( ) assert ( headers_train == headers_validation - ), logger.error("The training and validation CSVs do not have the same header information.") + ), "The training and validation CSVs do not have the same header information." # testing data is present data_testing = None @@ -93,7 +88,7 @@ def main_run( ) assert ( headers_train == headers_testing - ), logger.error("The training and testing CSVs do not have the same header information.") + ), "The training and testing CSVs do not have the same header information." parameters = populate_header_in_parameters(parameters, headers_train) logger.debug("if we are here, it is assumed that the user wants to do training") diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 7bd7184e1..93dae48b6 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -113,8 +113,8 @@ def train_network( label = subject["label"][torchio.DATA] label = label.to(params["device"]) - if params["save_training"]: - write_training_patches(subject, params) + if params["save_training"]: + write_training_patches(subject, params) # ensure spacing is always present in params and is always subject-specific if "spacing" in subject: @@ -195,7 +195,7 @@ def train_network( ) average_epoch_train_loss = total_epoch_train_loss / len(train_dataloader) - logger.info(f"Epoch Final train loss : {average_epoch_train_loss}") + print(" Epoch Final train loss : ", average_epoch_train_loss) # get overall stats for classification if calculate_overall_metrics: @@ -334,7 +334,7 @@ def training_loop( logger.debug(f"Hostname : {os.environ.get('HOSTNAME')}") # datetime object containing current date and time - print(f"Initializing training at : {get_date_time()}") + print("Initializing training at :", get_date_time(), flush=True) calculate_overall_metrics = (params["problem_type"] == "classification") or ( params["problem_type"] == "regression" @@ -463,7 +463,12 @@ def training_loop( logger.debug(f"Epoch end time : {get_date_time()}") epoch_end_time = time.time() - logger.info(f"Time taken for epoch : {(epoch_end_time - epoch_start_time) / 60} mins") + print( + "Time taken for epoch : ", + (epoch_end_time - epoch_start_time) / 60, + " mins", + flush=True, + ) model_dict = get_model_dict(model, params["device_id"]) diff --git a/docs/usage.md b/docs/usage.md index 36f785550..171357e8b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -321,10 +321,20 @@ logs/data_loop.log # log file showing looping through data - The predictions will be saved in the same directory as the model if `outputdir` is not passed to `gandlf_run`. - For segmentation, a directory will be created per subject ID in the input CSV. - For classification/regression, the predictions will be generated in the `outputdir` or `modeldir` as a CSV file. -- Logs directory will be generated in the `outputdir` or `modeldir` if not specified +- Logs directory will be generated in the same directory as the model if `outputdir` is not specified. ### Logging levels -Logging levels are compatible with the [Python logging library](https://docs.python.org/3/library/logging.html#levels). +The log messages produced by a specific component each have their own designated log levels. If a message's log level is equal to or higher in priority than the log level setting of its component, it will be displayed. Otherwise, it will be suppressed. This feature enables you to selectively mute extensive sets of log messages that aren't pertinent and enhance the detail of logs for relevant components. The anticipated log level values, ranked from most to least critical, are as follows: +* `logging.CRITICAL` +* `logging.ERROR` +* `logging.WARNING` +* `logging.INFO` +* `logging.DEBUG` +* `logging.NOTSET` + +See documentation for the Python logging module for more information on log levels: https://docs.python.org/3/library/logging.html#levels. + +In GaNDLF logs levels are set as follows: - `std.err` is set to `ERROR` level - `gandlf.log` file is set to `DEBUG` level if `verbose` pararameter is `True`, otherwise set to `WARNING`. The default logging level is `WARNING`; `verbose` parameter can be specified in a configuration file. diff --git a/testing/test_full.py b/testing/test_full.py index 1bd81d7d0..fdab6969d 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -3096,7 +3096,7 @@ def test_generic_cli_function_metrics_cli_rad_nd(): assert os.path.isfile(output_file), "Metrics output file was not generated" sanitize_outputDir() - + print("passed") From 611a209d4e228f65e620d2a5c7a25fa5759ef49b Mon Sep 17 00:00:00 2001 From: sylwiamm Date: Wed, 3 Apr 2024 04:50:46 -0400 Subject: [PATCH 3/5] Reformatting --- GANDLF/cli/main_run.py | 6 ++++-- GANDLF/compute/forward_pass.py | 18 +++++++++--------- GANDLF/compute/generic.py | 9 ++++++--- GANDLF/compute/inference_loop.py | 7 +++++-- GANDLF/compute/loss_and_metric.py | 3 +-- GANDLF/compute/step.py | 5 ++++- GANDLF/compute/training_loop.py | 21 +++++++++++++-------- GANDLF/config_manager.py | 4 +--- GANDLF/data/ImagesFromDataFrame.py | 20 ++++++++++++++------ GANDLF/models/densenet.py | 4 +--- GANDLF/models/efficientnet.py | 4 +--- GANDLF/models/resnet.py | 4 +--- GANDLF/training_manager.py | 5 ++++- GANDLF/utils/__init__.py | 4 +--- GANDLF/utils/logger.py | 15 ++++++++++----- GANDLF/utils/modelio.py | 5 ++++- testing/test_full.py | 23 ++++++++++++----------- 17 files changed, 91 insertions(+), 66 deletions(-) diff --git a/GANDLF/cli/main_run.py b/GANDLF/cli/main_run.py index 298df9ef8..4a1b0f6d3 100644 --- a/GANDLF/cli/main_run.py +++ b/GANDLF/cli/main_run.py @@ -52,7 +52,7 @@ def main_run( ) parameters["output_dir"] = model_dir Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) - + # if the output directory is not specified, then use the model directory even for the testing data # default behavior parameters["output_dir"] = parameters.get("output_dir", output_dir) @@ -61,7 +61,9 @@ def main_run( Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True) # setup logger - logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=output_dir, verbose=parameters.get("verbose", False)) + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( + output_dir=output_dir, verbose=parameters.get("verbose", False) + ) if "-1" in device: device = "cpu" diff --git a/GANDLF/compute/forward_pass.py b/GANDLF/compute/forward_pass.py index c403fa22f..a45f6a970 100644 --- a/GANDLF/compute/forward_pass.py +++ b/GANDLF/compute/forward_pass.py @@ -50,7 +50,10 @@ def validate_network( if "logger_name" in params: logger = logging.getLogger(params["logger_name"]) else: - logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", False)) + logger, params["logs_dir"], params["logger_name"] = setup_logger( + output_dir=params["output_dir"], + verbose=params.get("verbose", False), + ) print("*" * 20) print("Starting " + mode + " : ") @@ -257,7 +260,7 @@ def validate_network( ", time : ", get_date_time(), ", location :", - patches_batch[torchio.LOCATION] + patches_batch[torchio.LOCATION], ) current_patch += 1 image = ( @@ -285,7 +288,7 @@ def validate_network( "=== Validation shapes : label:", label.shape, ", image:", - image.shape + image.shape, ) if is_inference: @@ -431,7 +434,7 @@ def validate_network( "Full image " + mode + ":: Loss: ", final_loss, "; Metric: ", - final_metric + final_metric, ) # # Non network validing related @@ -465,12 +468,9 @@ def validate_network( total_epoch_valid_metric[metric] / (batch_idx + 1) ).tolist() else: - to_print = total_epoch_valid_metric[metric] / ( - batch_idx + 1 - ) + to_print = total_epoch_valid_metric[metric] / (batch_idx + 1) logger.debug( - "Half-Epoch Average " + mode + " " + metric + " : ", - to_print, + "Half-Epoch Average " + mode + " " + metric + " : ", to_print ) if params["medcam_enabled"] and params["model"]["type"] == "torch": diff --git a/GANDLF/compute/generic.py b/GANDLF/compute/generic.py index 8b1b2cd5e..b179d9407 100644 --- a/GANDLF/compute/generic.py +++ b/GANDLF/compute/generic.py @@ -12,7 +12,7 @@ parseTrainingCSV, send_model_to_device, get_class_imbalance_weights, - setup_logger + setup_logger, ) @@ -40,11 +40,14 @@ def create_pytorch_objects( Returns: Tuple[ torch.nn.Module, torch.optim.Optimizer, DataLoader, DataLoader, torch.optim.lr_scheduler.LRScheduler, dict, ]: The model, optimizer, train loader, validation loader, scheduler, and parameters. - """ + """ if "logger_name" in parameters: logger = logging.getLogger(parameters["logger_name"]) else: - logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( + output_dir=parameters["output_dir"], + verbose=parameters.get("verbose", False), + ) # initialize train and val loaders train_loader, val_loader = None, None diff --git a/GANDLF/compute/inference_loop.py b/GANDLF/compute/inference_loop.py index 3f63f6ebe..e1861442c 100644 --- a/GANDLF/compute/inference_loop.py +++ b/GANDLF/compute/inference_loop.py @@ -24,7 +24,7 @@ load_ov_model, print_model_summary, applyCustomColorMap, - setup_logger + setup_logger, ) from GANDLF.data.inference_dataloader_histopath import InferTumorSegDataset @@ -51,7 +51,10 @@ def inference_loop( if "logger_name" in parameters: logger = logging.getLogger(parameters["logger_name"]) else: - logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( + output_dir=parameters["output_dir"], + verbose=parameters.get("verbose", False), + ) # Defining our model here according to parameters mentioned in the configuration file logger.debug("Current model type : ", parameters["model"]["type"]) diff --git a/GANDLF/compute/loss_and_metric.py b/GANDLF/compute/loss_and_metric.py index f2247ba3b..d709bf2ae 100644 --- a/GANDLF/compute/loss_and_metric.py +++ b/GANDLF/compute/loss_and_metric.py @@ -64,8 +64,7 @@ def get_loss_and_metrics( loss_function = global_losses_dict[loss_str_lower] else: warnings.warn( - "Could not find the requested loss function '" - + params["loss_function"] + "Could not find the requested loss function '" + params["loss_function"] ) loss = 0 diff --git a/GANDLF/compute/step.py b/GANDLF/compute/step.py index cb9c41a15..5f2d11db7 100644 --- a/GANDLF/compute/step.py +++ b/GANDLF/compute/step.py @@ -30,7 +30,10 @@ def step( if "logger_name" in params: logger = logging.getLogger(params["logger_name"]) else: - logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params["verbose"]) + logger, params["logs_dir"], params["logger_name"] = setup_logger( + output_dir=params["output_dir"], + verbose=params.get("verbose", False), + ) if torch.cuda.is_available(): diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 93dae48b6..4358d84e5 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -58,7 +58,10 @@ def train_network( if "logger_name" in params: logger = logging.getLogger(params["logger_name"]) else: - logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", False)) + logger, params["logs_dir"], params["logger_name"] = setup_logger( + output_dir=params["output_dir"], + verbose=params.get("verbose", False), + ) print("*" * 20) print("Starting Training : ") @@ -91,14 +94,15 @@ def train_network( # Set the model to train model.train() log_file = os.path.join(parameters["logs_dir"], "data_loop.log") - with open(log_file, 'a') as fl: + with open(log_file, "a") as fl: for batch_idx, (subject) in enumerate( tqdm(train_dataloader, file = fl, desc="Looping over training data") ): optimizer.zero_grad() image = ( torch.cat( - [subject[key][torchio.DATA] for key in params["channel_keys"]], dim=1 + [subject[key][torchio.DATA] for key in params["channel_keys"]], + dim=1, ) .float() .to(params["device"]) @@ -107,7 +111,7 @@ def train_network( label = torch.cat([subject[key] for key in params["value_keys"]], dim=0) # min is needed because for certain cases, batch size becomes smaller than the total remaining labels label = label.reshape( - min(params["batch_size"], len(label)), len(params["value_keys"]), + min(params["batch_size"], len(label)), len(params["value_keys"]) ) else: label = subject["label"][torchio.DATA] @@ -190,9 +194,7 @@ def train_network( ).tolist() else: to_print = total_epoch_train_metric[metric] / (batch_idx + 1) - logger.debug( - f"Half-Epoch Average train {metric}: {to_print}" - ) + logger.debug(f"Half-Epoch Average train {metric}: {to_print}") average_epoch_train_loss = total_epoch_train_loss / len(train_dataloader) print(" Epoch Final train loss : ", average_epoch_train_loss) @@ -237,7 +239,10 @@ def training_loop( if "logger_name" in params: logger = logging.getLogger(params["logger_name"]) else: - logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params["verbose"]) + logger, params["logs_dir"], params["logger_name"] = setup_logger( + output_dir=params["output_dir"], + verbose=params["verbose"], + ) # Some autodetermined factors if epochs is None: diff --git a/GANDLF/config_manager.py b/GANDLF/config_manager.py index 06d641454..ae1482ab7 100644 --- a/GANDLF/config_manager.py +++ b/GANDLF/config_manager.py @@ -70,9 +70,7 @@ def initialize_parameter( params[parameter_to_initialize] ) else: - warnings.warn( - "Initializing '" + parameter_to_initialize + "' as " + str(value) - ) + warnings.warn("Initializing '" + parameter_to_initialize + "' as " + str(value)) params[parameter_to_initialize] = value return params diff --git a/GANDLF/data/ImagesFromDataFrame.py b/GANDLF/data/ImagesFromDataFrame.py index c85ec1b4f..8e7917715 100644 --- a/GANDLF/data/ImagesFromDataFrame.py +++ b/GANDLF/data/ImagesFromDataFrame.py @@ -16,7 +16,7 @@ resize_image, get_filename_extension_sanitized, get_correct_padding_size, - setup_logger + setup_logger, ) from .preprocessing import get_transforms_for_preprocessing from .augmentation import global_augs_dict @@ -85,7 +85,10 @@ def ImagesFromDataFrame( if "logger_name" in parameters: logger = logging.getLogger(parameters["logger_name"]) else: - logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False)) + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( + output_dir=parameters["output_dir"], + verbose=parameters.get("verbose", False), + ) resize_images_flag = False @@ -128,9 +131,11 @@ def _save_resized_images( # iterating through the dataframe and sending logs to file log_file = os.path.join(parameters["logs_dir"], "data_loop.log") - with open(log_file, 'a') as fl: + with open(log_file, "a") as fl: for patient in tqdm( - range(num_row), file = fl, desc="Constructing queue for " + loader_type + " data" + range(num_row), + file = fl, + desc="Constructing queue for " + loader_type + " data", ): # We need this dict for storing the meta data for each subject # such as different image modalities, labels, any other data @@ -157,7 +162,8 @@ def _save_resized_images( # if resize_image is requested, the perform per-image resize with appropriate interpolator if resize_images_flag: img_resized = resize_image( - subject_dict[str(channel)].as_sitk(), preprocessing["resize_image"] + subject_dict[str(channel)].as_sitk(), + preprocessing["resize_image"], ) if parameters["memory_save_mode"]: _save_resized_images( @@ -187,7 +193,9 @@ def _save_resized_images( if not os.path.isfile(str(dataframe[labelHeader][patient])): skip_subject = True - subject_dict["label"] = torchio.LabelMap(dataframe[labelHeader][patient]) + subject_dict["label"] = torchio.LabelMap( + dataframe[labelHeader][patient] + ) subject_dict["path_to_metadata"] = str(dataframe[labelHeader][patient]) # if resize is requested, the perform per-image resize with appropriate interpolator diff --git a/GANDLF/models/densenet.py b/GANDLF/models/densenet.py index 43deaf40d..95a4e0626 100644 --- a/GANDLF/models/densenet.py +++ b/GANDLF/models/densenet.py @@ -175,9 +175,7 @@ def __init__(self, parameters: dict, block_config=(6, 12, 24, 16)): if not ("no_max_pool" in parameters): parameters["no_max_pool"] = False if self.Norm is None: - warnings.warn( - "densenet is not defined without a normalization layer" - ) + warnings.warn("densenet is not defined without a normalization layer") self.Norm = self.BatchNorm if self.n_dimensions == 2: diff --git a/GANDLF/models/efficientnet.py b/GANDLF/models/efficientnet.py index db3a4b606..560d1aa98 100644 --- a/GANDLF/models/efficientnet.py +++ b/GANDLF/models/efficientnet.py @@ -398,9 +398,7 @@ def __init__(self, parameters: dict, scale_params): # how to scale depth and wi else: sys.exit("Only 2D or 3D convolutions are supported.") if self.Norm is None: - warnings.warn( - "efficientnet is not defined without a normalization layer" - ) + warnings.warn("efficientnet is not defined without a normalization layer") self.Norm = self.BatchNorm patch_check = checkPatchDimensions(parameters["patch_size"], numlay=5) diff --git a/GANDLF/models/resnet.py b/GANDLF/models/resnet.py index 81b1b3b00..a12e3fdce 100644 --- a/GANDLF/models/resnet.py +++ b/GANDLF/models/resnet.py @@ -56,9 +56,7 @@ def __init__(self, parameters: dict, blockType, block_config): # If normalization layer is not defined, use Batch Normalization if self.Norm is None: - warnings.warn( - "resnet is not defined without a normalization layer" - ) + warnings.warn("resnet is not defined without a normalization layer") self.Norm = self.BatchNorm # Define first convolution layer with 7x7 conv stride 2, 2x2 pool stride 2 diff --git a/GANDLF/training_manager.py b/GANDLF/training_manager.py index 8c58da6bb..d38ba3d65 100644 --- a/GANDLF/training_manager.py +++ b/GANDLF/training_manager.py @@ -29,7 +29,10 @@ def TrainingManager( if "logger_name" in parameters: logger = logging.getLogger(parameters["logger_name"]) else: - logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger(output_dir=parameters["output_dir"], verbose=parameters["verbose"]) + logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( + output_dir=parameters["output_dir"], + verbose=parameters.get("verbose", False), + ) if reset: shutil.rmtree(outputDir) diff --git a/GANDLF/utils/__init__.py b/GANDLF/utils/__init__.py index 34450b1ad..89b654286 100644 --- a/GANDLF/utils/__init__.py +++ b/GANDLF/utils/__init__.py @@ -69,6 +69,4 @@ from .data_splitter import split_data -from .logger import ( - setup_logger, -) +from .logger import setup_logger diff --git a/GANDLF/utils/logger.py b/GANDLF/utils/logger.py index 8c053c91f..9ab271c72 100644 --- a/GANDLF/utils/logger.py +++ b/GANDLF/utils/logger.py @@ -2,6 +2,7 @@ from typing import Optional, Tuple from pathlib import Path + def warning_on_one_line(message, category, filename, lineno, file=None, line=None): """ This function formats warning message according to its type @@ -9,10 +10,12 @@ def warning_on_one_line(message, category, filename, lineno, file=None, line=Non if category == UserWarning: return str(message) else: - return '%s:%s: %s:%s' % (filename, lineno, category.__name__, message) + return "%s:%s: %s:%s" % (filename, lineno, category.__name__, message) -def setup_logger(output_dir: str, verbose: Optional[bool] = False) -> Tuple[logging.Logger, str, str]: +def setup_logger( + output_dir: str, verbose: Optional[bool] = False + ) -> Tuple[logging.Logger, str, str]: """ This function setups a logger with severity level controlled by verbose parameter from a config file. @@ -26,8 +29,8 @@ def setup_logger(output_dir: str, verbose: Optional[bool] = False) -> Tuple[logg logger_dir (str): directory for the logs logger_name (str): name of the logger """ - logs_dir = f'{output_dir}/logs' - logger_name = 'gandlf' + logs_dir = f"{output_dir}/logs" + logger_name = "gandlf" Path(logs_dir).mkdir(parents=True, exist_ok=True) warnings.formatwarning = warning_on_one_line @@ -41,7 +44,9 @@ def setup_logger(output_dir: str, verbose: Optional[bool] = False) -> Tuple[logg warnings_logger = logging.getLogger("py.warnings") # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) fh.setFormatter(formatter) # add the handlers to logger diff --git a/GANDLF/utils/modelio.py b/GANDLF/utils/modelio.py index 2d18c71d7..e4543dc5c 100644 --- a/GANDLF/utils/modelio.py +++ b/GANDLF/utils/modelio.py @@ -41,7 +41,10 @@ def optimize_and_save_model( if "logger_name" in params: logger = logging.getLogger(params["logger_name"]) else: - logger, params["logs_dir"], params["logger_name"] = setup_logger(output_dir=params["output_dir"], verbose=params.get("verbose", True)) + logger, params["logs_dir"], params["logger_name"] = setup_logger( + output_dir=params["output_dir"], + verbose=params.get("verbose", False), + ) # Check if ONNX export is enabled in the parameter dictionary onnx_export = params["model"].get("onnx_export", onnx_export) diff --git a/testing/test_full.py b/testing/test_full.py index fdab6969d..9ff310c59 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -3096,7 +3096,7 @@ def test_generic_cli_function_metrics_cli_rad_nd(): assert os.path.isfile(output_file), "Metrics output file was not generated" sanitize_outputDir() - + print("passed") @@ -3114,9 +3114,9 @@ def test_generic_deploy_metrics_docker(): ) assert result, "run_deployment returned false" - + sanitize_outputDir() - + print("passed") def test_generic_data_split(): @@ -3150,7 +3150,7 @@ def test_generic_data_split(): assert len(files_in_outputDir) == 15, "CSVs were not split correctly" sanitize_outputDir() - + print("passed") @log_capture() @@ -3159,17 +3159,18 @@ def test_generic_logging(capture): logger, logs_dir, logger_name = setup_logger(outputDir, verbose=True) - assert os.path.isdir(logs_dir), 'Directory for logs was not generated' - assert logger_name in logging.root.manager.loggerDict, f"Logger with name {logger_name} was not created" + assert os.path.isdir(logs_dir), "Directory for logs was not generated" + assert ( + logger_name in logging.root.manager.loggerDict + ), f"Logger with name {logger_name} was not created" - logger.info('a message') - logger.error('an error') + logger.info("a message") + logger.error("an error") capture.check( - (logger_name, "INFO", "a message"), - (logger_name, "ERROR", "an error") + (logger_name, "INFO", "a message"), (logger_name, "ERROR", "an error") ) sanitize_outputDir() - + print("passed") From 4e3688abb84d26454a6ab1fe6fa340470dd7eb21 Mon Sep 17 00:00:00 2001 From: sylwiamm Date: Wed, 3 Apr 2024 05:04:47 -0400 Subject: [PATCH 4/5] Reformatting pt2 --- GANDLF/compute/forward_pass.py | 3 +-- GANDLF/compute/generic.py | 2 +- GANDLF/compute/inference_loop.py | 2 +- GANDLF/compute/step.py | 3 +-- GANDLF/compute/training_loop.py | 8 +++----- GANDLF/data/ImagesFromDataFrame.py | 8 ++++---- GANDLF/training_manager.py | 4 ++-- GANDLF/utils/logger.py | 4 ++-- GANDLF/utils/modelio.py | 5 ++--- testing/test_full.py | 4 +++- 10 files changed, 20 insertions(+), 23 deletions(-) diff --git a/GANDLF/compute/forward_pass.py b/GANDLF/compute/forward_pass.py index a45f6a970..59f26001f 100644 --- a/GANDLF/compute/forward_pass.py +++ b/GANDLF/compute/forward_pass.py @@ -51,8 +51,7 @@ def validate_network( logger = logging.getLogger(params["logger_name"]) else: logger, params["logs_dir"], params["logger_name"] = setup_logger( - output_dir=params["output_dir"], - verbose=params.get("verbose", False), + output_dir=params["output_dir"], verbose=params.get("verbose", False) ) print("*" * 20) diff --git a/GANDLF/compute/generic.py b/GANDLF/compute/generic.py index b179d9407..137514ad8 100644 --- a/GANDLF/compute/generic.py +++ b/GANDLF/compute/generic.py @@ -45,7 +45,7 @@ def create_pytorch_objects( logger = logging.getLogger(parameters["logger_name"]) else: logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( - output_dir=parameters["output_dir"], + output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False), ) diff --git a/GANDLF/compute/inference_loop.py b/GANDLF/compute/inference_loop.py index e1861442c..fa5c84112 100644 --- a/GANDLF/compute/inference_loop.py +++ b/GANDLF/compute/inference_loop.py @@ -52,7 +52,7 @@ def inference_loop( logger = logging.getLogger(parameters["logger_name"]) else: logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( - output_dir=parameters["output_dir"], + output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False), ) diff --git a/GANDLF/compute/step.py b/GANDLF/compute/step.py index 5f2d11db7..e6e0d7eec 100644 --- a/GANDLF/compute/step.py +++ b/GANDLF/compute/step.py @@ -31,8 +31,7 @@ def step( logger = logging.getLogger(params["logger_name"]) else: logger, params["logs_dir"], params["logger_name"] = setup_logger( - output_dir=params["output_dir"], - verbose=params.get("verbose", False), + output_dir=params["output_dir"], verbose=params.get("verbose", False) ) diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 4358d84e5..546c63c41 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -59,8 +59,7 @@ def train_network( logger = logging.getLogger(params["logger_name"]) else: logger, params["logs_dir"], params["logger_name"] = setup_logger( - output_dir=params["output_dir"], - verbose=params.get("verbose", False), + output_dir=params["output_dir"], verbose=params.get("verbose", False) ) print("*" * 20) @@ -101,7 +100,7 @@ def train_network( optimizer.zero_grad() image = ( torch.cat( - [subject[key][torchio.DATA] for key in params["channel_keys"]], + [subject[key][torchio.DATA] for key in params["channel_keys"]], dim=1, ) .float() @@ -240,8 +239,7 @@ def training_loop( logger = logging.getLogger(params["logger_name"]) else: logger, params["logs_dir"], params["logger_name"] = setup_logger( - output_dir=params["output_dir"], - verbose=params["verbose"], + output_dir=params["output_dir"], verbose=params.get("verbose", False) ) # Some autodetermined factors diff --git a/GANDLF/data/ImagesFromDataFrame.py b/GANDLF/data/ImagesFromDataFrame.py index 8e7917715..3b50d58f9 100644 --- a/GANDLF/data/ImagesFromDataFrame.py +++ b/GANDLF/data/ImagesFromDataFrame.py @@ -86,7 +86,7 @@ def ImagesFromDataFrame( logger = logging.getLogger(parameters["logger_name"]) else: logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( - output_dir=parameters["output_dir"], + output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False), ) @@ -133,8 +133,8 @@ def _save_resized_images( log_file = os.path.join(parameters["logs_dir"], "data_loop.log") with open(log_file, "a") as fl: for patient in tqdm( - range(num_row), - file = fl, + range(num_row), + file = fl, desc="Constructing queue for " + loader_type + " data", ): # We need this dict for storing the meta data for each subject @@ -162,7 +162,7 @@ def _save_resized_images( # if resize_image is requested, the perform per-image resize with appropriate interpolator if resize_images_flag: img_resized = resize_image( - subject_dict[str(channel)].as_sitk(), + subject_dict[str(channel)].as_sitk(), preprocessing["resize_image"], ) if parameters["memory_save_mode"]: diff --git a/GANDLF/training_manager.py b/GANDLF/training_manager.py index d38ba3d65..1ca72655c 100644 --- a/GANDLF/training_manager.py +++ b/GANDLF/training_manager.py @@ -25,12 +25,12 @@ def TrainingManager( resume (bool): Whether the previous run will be resumed or not. reset (bool): Whether the previous run will be reset or not. """ - + if "logger_name" in parameters: logger = logging.getLogger(parameters["logger_name"]) else: logger, parameters["logs_dir"], parameters["logger_name"] = setup_logger( - output_dir=parameters["output_dir"], + output_dir=parameters["output_dir"], verbose=parameters.get("verbose", False), ) diff --git a/GANDLF/utils/logger.py b/GANDLF/utils/logger.py index 9ab271c72..1c5dec0da 100644 --- a/GANDLF/utils/logger.py +++ b/GANDLF/utils/logger.py @@ -14,8 +14,8 @@ def warning_on_one_line(message, category, filename, lineno, file=None, line=Non def setup_logger( - output_dir: str, verbose: Optional[bool] = False - ) -> Tuple[logging.Logger, str, str]: + output_dir: str, verbose: Optional[bool] = False +) -> Tuple[logging.Logger, str, str]: """ This function setups a logger with severity level controlled by verbose parameter from a config file. diff --git a/GANDLF/utils/modelio.py b/GANDLF/utils/modelio.py index e4543dc5c..ba2306f4d 100644 --- a/GANDLF/utils/modelio.py +++ b/GANDLF/utils/modelio.py @@ -42,10 +42,9 @@ def optimize_and_save_model( logger = logging.getLogger(params["logger_name"]) else: logger, params["logs_dir"], params["logger_name"] = setup_logger( - output_dir=params["output_dir"], - verbose=params.get("verbose", False), + output_dir=params["output_dir"], verbose=params.get("verbose", False) ) - + # Check if ONNX export is enabled in the parameter dictionary onnx_export = params["model"].get("onnx_export", onnx_export) diff --git a/testing/test_full.py b/testing/test_full.py index 9ff310c59..54f4bdb64 100644 --- a/testing/test_full.py +++ b/testing/test_full.py @@ -3119,6 +3119,7 @@ def test_generic_deploy_metrics_docker(): print("passed") + def test_generic_data_split(): print("51: Starting test for splitting and saving CSVs") # read and initialize parameters for specific data dimension @@ -3153,6 +3154,7 @@ def test_generic_data_split(): print("passed") + @log_capture() def test_generic_logging(capture): print("52: Testing logging setup") @@ -3163,7 +3165,7 @@ def test_generic_logging(capture): assert ( logger_name in logging.root.manager.loggerDict ), f"Logger with name {logger_name} was not created" - + logger.info("a message") logger.error("an error") From 8d5f8c62c6560bd74a7e974feb19591009e2d26e Mon Sep 17 00:00:00 2001 From: sylwiamm Date: Wed, 3 Apr 2024 05:08:09 -0400 Subject: [PATCH 5/5] Reformatting pt3 --- GANDLF/compute/step.py | 1 - GANDLF/compute/training_loop.py | 2 +- GANDLF/data/ImagesFromDataFrame.py | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/GANDLF/compute/step.py b/GANDLF/compute/step.py index e6e0d7eec..55b65850e 100644 --- a/GANDLF/compute/step.py +++ b/GANDLF/compute/step.py @@ -34,7 +34,6 @@ def step( output_dir=params["output_dir"], verbose=params.get("verbose", False) ) - if torch.cuda.is_available(): logger.debug(torch.cuda.memory_summary()) logger.debug( diff --git a/GANDLF/compute/training_loop.py b/GANDLF/compute/training_loop.py index 546c63c41..960ca6e16 100644 --- a/GANDLF/compute/training_loop.py +++ b/GANDLF/compute/training_loop.py @@ -95,7 +95,7 @@ def train_network( log_file = os.path.join(parameters["logs_dir"], "data_loop.log") with open(log_file, "a") as fl: for batch_idx, (subject) in enumerate( - tqdm(train_dataloader, file = fl, desc="Looping over training data") + tqdm(train_dataloader, file=fl, desc="Looping over training data") ): optimizer.zero_grad() image = ( diff --git a/GANDLF/data/ImagesFromDataFrame.py b/GANDLF/data/ImagesFromDataFrame.py index 3b50d58f9..1b6669bab 100644 --- a/GANDLF/data/ImagesFromDataFrame.py +++ b/GANDLF/data/ImagesFromDataFrame.py @@ -90,7 +90,6 @@ def ImagesFromDataFrame( verbose=parameters.get("verbose", False), ) - resize_images_flag = False # if resize has been defined but resample is not (or is none) if not (preprocessing is None): @@ -134,7 +133,7 @@ def _save_resized_images( with open(log_file, "a") as fl: for patient in tqdm( range(num_row), - file = fl, + file=fl, desc="Constructing queue for " + loader_type + " data", ): # We need this dict for storing the meta data for each subject