From e9e1815fed3f89fea6289e211a1c80ca9743e74a Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Fri, 18 Oct 2019 00:55:21 +0530 Subject: [PATCH 1/8] Refactor logging setup to support distributed attrs * `cleanup_logging()` is replaced with stdlib's `logging.shutdown()` * Remove `TeeLogger` and use standard log handlers * Remove `replace_cr_with_newline` and use the standard logging practice of using `logging.Filter` * Introduce `rank` and `world_size` optional attributes to support distributed workers --- allennlp/commands/train.py | 4 +- allennlp/common/__init__.py | 1 - allennlp/common/tee_logger.py | 63 --------------- allennlp/common/util.py | 144 ++++++++++++++++++++-------------- 4 files changed, 89 insertions(+), 123 deletions(-) delete mode 100644 allennlp/common/tee_logger.py diff --git a/allennlp/commands/train.py b/allennlp/commands/train.py index 1a87da0b5dd..07b4d4a776b 100644 --- a/allennlp/commands/train.py +++ b/allennlp/commands/train.py @@ -241,7 +241,7 @@ def train_model( create_serialization_dir(params, serialization_dir, recover, force) params.to_file(os.path.join(serialization_dir, CONFIG_NAME)) - stdout_handler = prepare_global_logging(serialization_dir, file_friendly_logging) + prepare_global_logging(serialization_dir, file_friendly_logging) prepare_environment(params) cuda_device = params.params.get("trainer").get("cuda_device", -1) @@ -318,7 +318,7 @@ def train_model( "'evaluate_on_test' flag, or use the 'allennlp evaluate' command." ) - cleanup_global_logging(stdout_handler) + logging.shutdown() # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) diff --git a/allennlp/common/__init__.py b/allennlp/common/__init__.py index bde38d57dfb..690166e68e0 100644 --- a/allennlp/common/__init__.py +++ b/allennlp/common/__init__.py @@ -1,6 +1,5 @@ from allennlp.common.from_params import FromParams from allennlp.common.params import Params from allennlp.common.registrable import Registrable -from allennlp.common.tee_logger import TeeLogger from allennlp.common.tqdm import Tqdm from allennlp.common.util import JsonDict diff --git a/allennlp/common/tee_logger.py b/allennlp/common/tee_logger.py deleted file mode 100644 index 7d74e2b6df9..00000000000 --- a/allennlp/common/tee_logger.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -A logger that maintains logs of both stdout and stderr when models are run. -""" - -from typing import TextIO -import os - - -def replace_cr_with_newline(message: str): - """ - TQDM and requests use carriage returns to get the training line to update for each batch - without adding more lines to the terminal output. Displaying those in a file won't work - correctly, so we'll just make sure that each batch shows up on its one line. - :param message: the message to permute - :return: the message with carriage returns replaced with newlines - """ - if "\r" in message: - message = message.replace("\r", "") - if not message or message[-1] != "\n": - message += "\n" - return message - - -class TeeLogger: - """ - This class is an attempt to maintain logs of both stdout and stderr for when models are run. - To use this class, at the beginning of your script insert these lines:: - - sys.stdout = TeeLogger("stdout.log", sys.stdout) - sys.stderr = TeeLogger("stdout.log", sys.stderr) - """ - - def __init__( - self, filename: str, terminal: TextIO, file_friendly_terminal_output: bool - ) -> None: - self.terminal = terminal - self.file_friendly_terminal_output = file_friendly_terminal_output - parent_directory = os.path.dirname(filename) - os.makedirs(parent_directory, exist_ok=True) - self.log = open(filename, "a") - - def write(self, message): - cleaned = replace_cr_with_newline(message) - - if self.file_friendly_terminal_output: - self.terminal.write(cleaned) - else: - self.terminal.write(message) - - self.log.write(cleaned) - - def flush(self): - self.terminal.flush() - self.log.flush() - - def isatty(self): - # Mirror the API of sys.stdout so that we can - # check for the presence of a terminal easily. - return not self.file_friendly_terminal_output - - def cleanup(self) -> TextIO: - self.log.close() - return self.terminal diff --git a/allennlp/common/util.py b/allennlp/common/util.py index 9a52d5a1cda..e43c657989c 100644 --- a/allennlp/common/util.py +++ b/allennlp/common/util.py @@ -1,16 +1,17 @@ """ Various utilities that don't fit anwhere else. """ -from itertools import zip_longest, islice -from typing import Any, Callable, Dict, List, Tuple, TypeVar, Iterable, Iterator import importlib import json import logging +import os import pkgutil import random import subprocess import sys -import os +from itertools import zip_longest, islice +from logging import Filter +from typing import Any, Callable, Dict, List, Tuple, TypeVar, Iterable, Iterator try: import resource @@ -29,7 +30,6 @@ from allennlp.common.checks import log_pytorch_version_info from allennlp.common.params import Params from allennlp.common.tqdm import Tqdm -from allennlp.common.tee_logger import TeeLogger logger = logging.getLogger(__name__) @@ -115,10 +115,10 @@ def lazy_groups_of(iterator: Iterator[A], group_size: int) -> Iterator[List[A]]: def pad_sequence_to_length( - sequence: List, - desired_length: int, - default_value: Callable[[], Any] = lambda: 0, - padding_on_right: bool = True, + sequence: List, + desired_length: int, + default_value: Callable[[], Any] = lambda: 0, + padding_on_right: bool = True, ) -> List: """ Take a list of objects and pads it to the desired length, returning the padded list. The @@ -219,78 +219,108 @@ def prepare_environment(params: Params): log_pytorch_version_info() -def prepare_global_logging( - serialization_dir: str, file_friendly_logging: bool -) -> logging.FileHandler: +class FileFriendlyLogFilter(Filter): + """ + TQDM and requests use carriage returns to get the training line to update for each batch + without adding more lines to the terminal output. Displaying those in a file won't work + correctly, so we'll just make sure that each batch shows up on its one line. """ - This function configures 3 global logging attributes - streaming stdout and stderr - to a file as well as the terminal, setting the formatting for the python logging - library and setting the interval frequency for the Tqdm progress bar. + def filter(self, record): + if "\r" in record.msg: + record.msg = record.msg.replace("\r", "") + if not record.msg or record.msg[-1] != "\n": + record.msg += "\n" + return True - Note that this function does not set the logging level, which is set in ``allennlp/run.py``. - Parameters - ---------- - serialization_dir : ``str``, required. - The directory to stream logs to. - file_friendly_logging : ``bool``, required. - Whether logs should clean the output to prevent carriage returns - (used to update progress bars on a single terminal line). This - option is typically only used if you are running in an environment - without a terminal. +class WorkerLogFilter(Filter): + def __init__(self, rank=-1): + super().__init__() + self._rank = rank - Returns - ------- - ``logging.FileHandler`` - A logging file handler that can later be closed and removed from the global logger. - """ + def filter(self, record): + if self._rank != -1: + record.msg = f"Rank {self._rank} | {record.msg}" + return True + +def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, rank: int = 0, + world_size: int = 1) -> None: # If we don't have a terminal as stdout, # force tqdm to be nicer. if not sys.stdout.isatty(): file_friendly_logging = True Tqdm.set_slower_interval(file_friendly_logging) - std_out_file = os.path.join(serialization_dir, "stdout.log") - sys.stdout = TeeLogger( # type: ignore - std_out_file, sys.stdout, file_friendly_logging - ) - sys.stderr = TeeLogger( # type: ignore - os.path.join(serialization_dir, "stderr.log"), sys.stderr, file_friendly_logging - ) - stdout_handler = logging.FileHandler(std_out_file) - stdout_handler.setFormatter( - logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") - ) - logging.getLogger().addHandler(stdout_handler) + # Handlers for stdout/err logging + output_stream_log_handler = logging.StreamHandler(sys.stdout) + error_stream_log_handler = logging.StreamHandler(sys.stderr) + + if world_size == 1: + # This case is not distributed training and hence will stick to the older + # log file names + output_file_log_handler = logging.FileHandler( + filename=os.path.join(serialization_dir, f"stdout.log")) + error_file_log_handler = logging.FileHandler( + filename=os.path.join(serialization_dir, f"stderr.log")) + else: + # Create log files with worker ids + output_file_log_handler = logging.FileHandler( + filename=os.path.join(serialization_dir, f"stdout_worker{rank}.log")) + error_file_log_handler = logging.FileHandler( + filename=os.path.join(serialization_dir, f"stderr_worker{rank}.log")) - return stdout_handler + # This adds the worker's rank to messages being logged to files. + # This will help when combining multiple worker log files using `less` command. + worker_filter = WorkerLogFilter(rank) + output_file_log_handler.addFilter(worker_filter) + error_file_log_handler.addFilter(worker_filter) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') -def cleanup_global_logging(stdout_handler: logging.FileHandler) -> None: - """ - This function closes any open file handles and logs set up by `prepare_global_logging`. + root_logger = logging.getLogger() - Parameters - ---------- - stdout_handler : ``logging.FileHandler``, required. - The file handler returned from `prepare_global_logging`, attached to the global logger. - """ - stdout_handler.close() - logging.getLogger().removeHandler(stdout_handler) + # file handlers need to be handled for tqdm's \r char + file_friendly_log_filter = FileFriendlyLogFilter() + + if rank == 0: + # stdout/stderr handlers are added only for the + # master worker. This is to avoid cluttering the console + # screen with too many log messages from all workers. + output_stream_log_handler.setFormatter(formatter) + error_stream_log_handler.setFormatter(formatter) + + output_stream_log_handler.setLevel(logging.INFO) + error_stream_log_handler.setLevel(logging.ERROR) + + if file_friendly_logging: + output_stream_log_handler.addFilter(file_friendly_log_filter) + error_stream_log_handler.addFilter(file_friendly_log_filter) + + root_logger.addHandler(output_stream_log_handler) + root_logger.addHandler(error_stream_log_handler) + + output_file_log_handler.addFilter(file_friendly_log_filter) + error_file_log_handler.addFilter(file_friendly_log_filter) + + output_file_log_handler.setFormatter(formatter) + error_file_log_handler.setFormatter(formatter) + + output_file_log_handler.setLevel(logging.INFO) + error_file_log_handler.setLevel(logging.ERROR) + + root_logger.addHandler(output_file_log_handler) + root_logger.addHandler(error_file_log_handler) - if isinstance(sys.stdout, TeeLogger): - sys.stdout = sys.stdout.cleanup() - if isinstance(sys.stderr, TeeLogger): - sys.stderr = sys.stderr.cleanup() + root_logger.setLevel(logging.INFO) LOADED_SPACY_MODELS: Dict[Tuple[str, bool, bool, bool], SpacyModelType] = {} def get_spacy_model( - spacy_model_name: str, pos_tags: bool, parse: bool, ner: bool + spacy_model_name: str, pos_tags: bool, parse: bool, ner: bool ) -> SpacyModelType: """ In order to avoid loading spacy models a whole bunch of times, we'll save references to them, From 98f2b6f1735f8bedee1a71bc5ff1b712868d07da Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Fri, 18 Oct 2019 01:04:45 +0530 Subject: [PATCH 2/8] Support for distributed training in `get_metrics` --- allennlp/training/util.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/allennlp/training/util.py b/allennlp/training/util.py index 37c600f3440..37594ea5a08 100644 --- a/allennlp/training/util.py +++ b/allennlp/training/util.py @@ -1,6 +1,7 @@ """ Helper functions for Trainers """ +import torch.distributed as dist from typing import Any, Union, Dict, Iterable, List, Optional, Tuple import datetime import json @@ -378,9 +379,12 @@ def rescale_gradients(model: Model, grad_norm: Optional[float] = None) -> Option return None -def get_metrics( - model: Model, total_loss: float, num_batches: int, reset: bool = False -) -> Dict[str, float]: +def get_metrics(model: Model, + total_loss: float, + num_batches: int, + reset: bool = False, + world_size: int = 1, + rank: int = 0) -> Dict[str, float]: """ Gets the metrics but sets ``"loss"`` to the total loss divided by the ``num_batches`` so that @@ -388,7 +392,18 @@ def get_metrics( """ metrics = model.get_metrics(reset=reset) metrics["loss"] = float(total_loss / num_batches) if num_batches > 0 else 0.0 - return metrics + + if world_size > 1: + # In distributed mode, average out all metrics across GPUs + aggregated_metrics = {} + for metric_name, metric_val in metrics.items(): + metric_tensor = torch.tensor(metric_val).to(torch.device(rank)) + dist.all_reduce(metric_tensor, op=dist.ReduceOp.SUM) + reduced_metric = metric_tensor.item() / world_size + aggregated_metrics[metric_name] = reduced_metric + return aggregated_metrics + else: + return metrics def evaluate( From 0dc45311489d49bb8752bf119d8ebf04a584fe13 Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Fri, 18 Oct 2019 01:20:18 +0530 Subject: [PATCH 3/8] Remove bad import --- allennlp/commands/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/allennlp/commands/train.py b/allennlp/commands/train.py index 07b4d4a776b..83ce1c1efa4 100644 --- a/allennlp/commands/train.py +++ b/allennlp/commands/train.py @@ -51,7 +51,6 @@ from allennlp.common.util import ( prepare_environment, prepare_global_logging, - cleanup_global_logging, dump_metrics, ) from allennlp.models.archival import archive_model, CONFIG_NAME From 7d2642350bd4cecd59980c72a53137474987bbb3 Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Fri, 18 Oct 2019 01:35:38 +0530 Subject: [PATCH 4/8] Fix duplicate log messages in stdout --- allennlp/common/util.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/allennlp/common/util.py b/allennlp/common/util.py index e43c657989c..6df12f74e4b 100644 --- a/allennlp/common/util.py +++ b/allennlp/common/util.py @@ -261,9 +261,9 @@ def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, # This case is not distributed training and hence will stick to the older # log file names output_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, f"stdout.log")) + filename=os.path.join(serialization_dir, "stdout.log")) error_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, f"stderr.log")) + filename=os.path.join(serialization_dir, "stderr.log")) else: # Create log files with worker ids output_file_log_handler = logging.FileHandler( @@ -281,6 +281,11 @@ def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, root_logger = logging.getLogger() + # Remove the already set stream handler in root logger. + # Not doing this will result in duplicate log messages + # printed in the console + root_logger.removeHandler(root_logger.handlers[0]) + # file handlers need to be handled for tqdm's \r char file_friendly_log_filter = FileFriendlyLogFilter() From 7264e05b7a7f82727a605d57593b82a742f95772 Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Tue, 22 Oct 2019 11:09:04 +0530 Subject: [PATCH 5/8] Remove preemptive `logging.shutdown` `logging.shutdown` is called by the logging module by default during exit which makes it unnecessary to be called from `train_model` --- allennlp/commands/train.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/allennlp/commands/train.py b/allennlp/commands/train.py index 83ce1c1efa4..fcba96b43bb 100644 --- a/allennlp/commands/train.py +++ b/allennlp/commands/train.py @@ -317,8 +317,6 @@ def train_model( "'evaluate_on_test' flag, or use the 'allennlp evaluate' command." ) - logging.shutdown() - # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) dump_metrics(os.path.join(serialization_dir, "metrics.json"), metrics, log=True) From 9a8f2398c8b9955e66367c30e02e1b0e051b4f28 Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Tue, 22 Oct 2019 11:29:49 +0530 Subject: [PATCH 6/8] Fix black formatting issues --- allennlp/commands/train.py | 6 +----- allennlp/common/util.py | 30 ++++++++++++++++++------------ allennlp/training/util.py | 14 ++++++++------ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/allennlp/commands/train.py b/allennlp/commands/train.py index fcba96b43bb..cfae9dda0a0 100644 --- a/allennlp/commands/train.py +++ b/allennlp/commands/train.py @@ -48,11 +48,7 @@ from allennlp.commands.subcommand import Subcommand from allennlp.common.checks import check_for_gpu from allennlp.common import Params -from allennlp.common.util import ( - prepare_environment, - prepare_global_logging, - dump_metrics, -) +from allennlp.common.util import prepare_environment, prepare_global_logging, dump_metrics from allennlp.models.archival import archive_model, CONFIG_NAME from allennlp.models.model import Model, _DEFAULT_WEIGHTS from allennlp.training.trainer import Trainer diff --git a/allennlp/common/util.py b/allennlp/common/util.py index 6df12f74e4b..1d3dfdf81f2 100644 --- a/allennlp/common/util.py +++ b/allennlp/common/util.py @@ -115,10 +115,10 @@ def lazy_groups_of(iterator: Iterator[A], group_size: int) -> Iterator[List[A]]: def pad_sequence_to_length( - sequence: List, - desired_length: int, - default_value: Callable[[], Any] = lambda: 0, - padding_on_right: bool = True, + sequence: List, + desired_length: int, + default_value: Callable[[], Any] = lambda: 0, + padding_on_right: bool = True, ) -> List: """ Take a list of objects and pads it to the desired length, returning the padded list. The @@ -225,6 +225,7 @@ class FileFriendlyLogFilter(Filter): without adding more lines to the terminal output. Displaying those in a file won't work correctly, so we'll just make sure that each batch shows up on its one line. """ + def filter(self, record): if "\r" in record.msg: record.msg = record.msg.replace("\r", "") @@ -244,8 +245,9 @@ def filter(self, record): return True -def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, rank: int = 0, - world_size: int = 1) -> None: +def prepare_global_logging( + serialization_dir: str, file_friendly_logging: bool, rank: int = 0, world_size: int = 1 +) -> None: # If we don't have a terminal as stdout, # force tqdm to be nicer. if not sys.stdout.isatty(): @@ -261,15 +263,19 @@ def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, # This case is not distributed training and hence will stick to the older # log file names output_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, "stdout.log")) + filename=os.path.join(serialization_dir, "stdout.log") + ) error_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, "stderr.log")) + filename=os.path.join(serialization_dir, "stderr.log") + ) else: # Create log files with worker ids output_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, f"stdout_worker{rank}.log")) + filename=os.path.join(serialization_dir, f"stdout_worker{rank}.log") + ) error_file_log_handler = logging.FileHandler( - filename=os.path.join(serialization_dir, f"stderr_worker{rank}.log")) + filename=os.path.join(serialization_dir, f"stderr_worker{rank}.log") + ) # This adds the worker's rank to messages being logged to files. # This will help when combining multiple worker log files using `less` command. @@ -277,7 +283,7 @@ def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, output_file_log_handler.addFilter(worker_filter) error_file_log_handler.addFilter(worker_filter) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") root_logger = logging.getLogger() @@ -325,7 +331,7 @@ def prepare_global_logging(serialization_dir: str, file_friendly_logging: bool, def get_spacy_model( - spacy_model_name: str, pos_tags: bool, parse: bool, ner: bool + spacy_model_name: str, pos_tags: bool, parse: bool, ner: bool ) -> SpacyModelType: """ In order to avoid loading spacy models a whole bunch of times, we'll save references to them, diff --git a/allennlp/training/util.py b/allennlp/training/util.py index 37594ea5a08..fcac33000a4 100644 --- a/allennlp/training/util.py +++ b/allennlp/training/util.py @@ -379,12 +379,14 @@ def rescale_gradients(model: Model, grad_norm: Optional[float] = None) -> Option return None -def get_metrics(model: Model, - total_loss: float, - num_batches: int, - reset: bool = False, - world_size: int = 1, - rank: int = 0) -> Dict[str, float]: +def get_metrics( + model: Model, + total_loss: float, + num_batches: int, + reset: bool = False, + world_size: int = 1, + rank: int = 0, +) -> Dict[str, float]: """ Gets the metrics but sets ``"loss"`` to the total loss divided by the ``num_batches`` so that From dfb38cb3d74f832991cf96e983ee16daee42b6fa Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Tue, 22 Oct 2019 11:37:56 +0530 Subject: [PATCH 7/8] Remove `tee_logger` references in API doc --- doc/api/allennlp.common.rst | 1 - doc/api/allennlp.common.tee_logger.rst | 7 ------- 2 files changed, 8 deletions(-) delete mode 100644 doc/api/allennlp.common.tee_logger.rst diff --git a/doc/api/allennlp.common.rst b/doc/api/allennlp.common.rst index 8a3bdd0b342..b6d01e21b34 100644 --- a/doc/api/allennlp.common.rst +++ b/doc/api/allennlp.common.rst @@ -12,7 +12,6 @@ that's used by datasets, models, trainers, and so on. allennlp.common.from_params allennlp.common.params allennlp.common.registrable - allennlp.common.tee_logger allennlp.common.testing allennlp.common.tqdm allennlp.common.util diff --git a/doc/api/allennlp.common.tee_logger.rst b/doc/api/allennlp.common.tee_logger.rst deleted file mode 100644 index 2535a078c33..00000000000 --- a/doc/api/allennlp.common.tee_logger.rst +++ /dev/null @@ -1,7 +0,0 @@ -allennlp.common.tee_logger -=============================== - -.. automodule:: allennlp.common.tee_logger - :members: - :undoc-members: - :show-inheritance: From fbaf7699e89d92085116d5cad23d0ab18f421e98 Mon Sep 17 00:00:00 2001 From: scarecrow1123 Date: Wed, 23 Oct 2019 08:31:41 +0530 Subject: [PATCH 8/8] Set log level from `ALLENNLP_DEBUG` env --- allennlp/common/util.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/allennlp/common/util.py b/allennlp/common/util.py index 1d3dfdf81f2..e922f2efb45 100644 --- a/allennlp/common/util.py +++ b/allennlp/common/util.py @@ -295,6 +295,11 @@ def prepare_global_logging( # file handlers need to be handled for tqdm's \r char file_friendly_log_filter = FileFriendlyLogFilter() + if os.environ.get("ALLENNLP_DEBUG"): + LEVEL = logging.DEBUG + else: + LEVEL = logging.INFO + if rank == 0: # stdout/stderr handlers are added only for the # master worker. This is to avoid cluttering the console @@ -302,7 +307,7 @@ def prepare_global_logging( output_stream_log_handler.setFormatter(formatter) error_stream_log_handler.setFormatter(formatter) - output_stream_log_handler.setLevel(logging.INFO) + output_stream_log_handler.setLevel(LEVEL) error_stream_log_handler.setLevel(logging.ERROR) if file_friendly_logging: @@ -318,13 +323,13 @@ def prepare_global_logging( output_file_log_handler.setFormatter(formatter) error_file_log_handler.setFormatter(formatter) - output_file_log_handler.setLevel(logging.INFO) + output_file_log_handler.setLevel(LEVEL) error_file_log_handler.setLevel(logging.ERROR) root_logger.addHandler(output_file_log_handler) root_logger.addHandler(error_file_log_handler) - root_logger.setLevel(logging.INFO) + root_logger.setLevel(LEVEL) LOADED_SPACY_MODELS: Dict[Tuple[str, bool, bool, bool], SpacyModelType] = {}