diff --git a/docs/api.rst b/docs/api.rst index 4921f39aa..1c126331c 100755 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,8 +8,9 @@ Logging ``pyaerocom`` initializes logging automatically on import in the following way. -1. ``info``-messages or worse are logged to ``pyaerocom.log.$PID`` or +1. ``info``-messages or worse are logged to ``logs/pyaerocom.log.$PID`` or (dynamic feature) the file given in the environment variable ``PYAEROCOM_LOG_FILE`` + - (dynamic feature) these log-files will be deleted after 7 days. 2. ``warning``-messages or worse are also printed on stdout. (dynamic feature) Output to stdout is disabled if the script is called non-interactive. @@ -46,7 +47,7 @@ provided here: class=FileHandler formatter=detailed level=DEBUG - file_name=pyaerocom.log.%(pid)s + file_name=logs/pyaerocom.log.%(pid)s args=('%(file_name)s', "w") diff --git a/pyaerocom/_logging.py b/pyaerocom/_logging.py index cf7313da9..68a9e549d 100644 --- a/pyaerocom/_logging.py +++ b/pyaerocom/_logging.py @@ -15,6 +15,7 @@ import os import pathlib import sys +import time from logging.config import fileConfig from pyaerocom.data import resources @@ -51,14 +52,30 @@ def change_verbosity(level: str | int) -> None: LOGGING_CONFIG = dict( # root logger - file_name=os.getenv("PYAEROCOM_LOG_FILE", default=f"pyaerocom.log.{os.getpid()}"), + file_name=os.getenv("PYAEROCOM_LOG_FILE", default=f"logs/pyaerocom.log.{os.getpid()}"), pid=os.getpid(), ) cwd_log_path = pathlib.Path.cwd() / "logging.ini" if cwd_log_path.exists(): fileConfig(cwd_log_path, defaults=LOGGING_CONFIG, disable_existing_loggers=True) else: + file_name = pathlib.Path(LOGGING_CONFIG["file_name"]) + log_path = file_name.parent + log_path.mkdir(exist_ok=True, parents=True) with resources.path("pyaerocom", "logging.ini") as path: fileConfig(path, defaults=LOGGING_CONFIG, disable_existing_loggers=False) if not sys.stdout.isatty(): # disable stdout when non-interactive change_verbosity(logging.CRITICAL) + # cleanup of old default logging files + now = time.time() + logger = logging.getLogger(__name__) + for f in log_path.glob("pyaerocom.log.*"): + age = now - f.lstat().st_mtime + if age > (7 * 24 * 60 * 60): + logger.info(f"deleting log-file older than 7 days: {f}") + f.unlink() + old_logfile = pathlib.Path("pyaerocom.log") + if old_logfile.exists(): + logger.warn( + f"no longer used old default logfile '{old_logfile}' exist, please consider deleting" + ) diff --git a/pyaerocom/io/read_airnow.py b/pyaerocom/io/read_airnow.py index 15f021250..5ccc44da5 100644 --- a/pyaerocom/io/read_airnow.py +++ b/pyaerocom/io/read_airnow.py @@ -335,7 +335,7 @@ def _read_files(self, files, vars_to_retrieve): varcol = self.FILE_COL_NAMES.index("variable") arrs = [] unique_stat_ids = None - for i in tqdm(range(len(files))): + for i in tqdm(range(len(files)), disable=None): fp = files[i] filedata = self._read_file(fp) for i, filevar in enumerate(file_vars_to_retrieve): @@ -421,7 +421,7 @@ def _filedata_to_statlist( statlist = np.unique((subset[:, statcol])) else: statlist = unique_stat_ids - for stat_id in tqdm(statlist, desc=var): + for stat_id in tqdm(statlist, desc=var, disable=None): if not stat_id in stat_ids: continue statmask = subset[:, statcol] == stat_id diff --git a/pyaerocom/io/read_ebas.py b/pyaerocom/io/read_ebas.py index 44a93dc1b..f5b00ed16 100644 --- a/pyaerocom/io/read_ebas.py +++ b/pyaerocom/io/read_ebas.py @@ -1830,7 +1830,7 @@ def _read_files(self, files, vars_to_retrieve, files_contain, constraints): var_count_glob = -1 logger.info(f"Reading EBAS data from {self.file_dir}") num_files = len(files) - for i in tqdm(range(num_files)): + for i in tqdm(range(num_files), disable=None): _file = files[i] contains = files_contain[i] try: diff --git a/pyaerocom/io/read_eea_aqerep_base.py b/pyaerocom/io/read_eea_aqerep_base.py index 5df1e59a3..b1bf4e986 100644 --- a/pyaerocom/io/read_eea_aqerep_base.py +++ b/pyaerocom/io/read_eea_aqerep_base.py @@ -653,7 +653,7 @@ def read( _country_dict = get_country_name_from_iso() logger.info("Reading files...") - for i in tqdm(range(len(files))): + for i in tqdm(range(len(files)), disable=None): _file = files[i] try: station_data = self.read_file(_file, var_name=var_name) diff --git a/pyaerocom/io/readaeronetbase.py b/pyaerocom/io/readaeronetbase.py index 1c13486f0..1f04bfc74 100644 --- a/pyaerocom/io/readaeronetbase.py +++ b/pyaerocom/io/readaeronetbase.py @@ -368,7 +368,7 @@ def read( num_files = len(files) logger.info("Reading AERONET data") skipped = 0 - for i in tqdm(range(num_files)): + for i in tqdm(range(num_files), disable=None): _file = files[i] try: station_data = self.read_file(_file, vars_to_retrieve=vars_to_retrieve)