Move default logs to default directory and cleanup of old logs

docs/api.rst

@@ -8,8 +8,9 @@ Logging
 
 ``pyaerocom`` initializes logging automatically on import in the following way.
 
-1. ``info``-messages or worse are logged to ``pyaerocom.log.$PID`` or
+1. ``info``-messages or worse are logged to ``logs/pyaerocom.log.$PID`` or
    (dynamic feature) the file given in the environment variable ``PYAEROCOM_LOG_FILE``
+   - (dynamic feature) these log-files will be deleted after 7 days.
 2. ``warning``-messages or worse are also printed on stdout.
    (dynamic feature) Output to stdout is disabled if the script is called non-interactive.
 
@@ -46,7 +47,7 @@ provided here:
    class=FileHandler
    formatter=detailed
    level=DEBUG
-   file_name=pyaerocom.log.%(pid)s
+   file_name=logs/pyaerocom.log.%(pid)s
    args=('%(file_name)s', "w")
 
 

pyaerocom/_logging.py

@@ -15,6 +15,7 @@
 import os
 import pathlib
 import sys
+import time
 from logging.config import fileConfig
 
 from pyaerocom.data import resources
@@ -51,14 +52,30 @@
 
 LOGGING_CONFIG = dict(
     # root logger
-    file_name=os.getenv("PYAEROCOM_LOG_FILE", default=f"pyaerocom.log.{os.getpid()}"),
+    file_name=os.getenv("PYAEROCOM_LOG_FILE", default=f"logs/pyaerocom.log.{os.getpid()}"),
     pid=os.getpid(),
 )
 cwd_log_path = pathlib.Path.cwd() / "logging.ini"
 if cwd_log_path.exists():
     fileConfig(cwd_log_path, defaults=LOGGING_CONFIG, disable_existing_loggers=True)
 else:
+    file_name = pathlib.Path(LOGGING_CONFIG["file_name"])
+    log_path = file_name.parent
+    log_path.mkdir(exist_ok=True, parents=True)
     with resources.path("pyaerocom", "logging.ini") as path:
         fileConfig(path, defaults=LOGGING_CONFIG, disable_existing_loggers=False)
     if not sys.stdout.isatty():  # disable stdout when non-interactive
         change_verbosity(logging.CRITICAL)
+    # cleanup of old default logging files
+    now = time.time()
+    logger = logging.getLogger(__name__)
+    for f in log_path.glob("pyaerocom.log.*"):
+        age = now - f.lstat().st_mtime
+        if age > (7 * 24 * 60 * 60):
+            logger.info(f"deleting log-file older than 7 days: {f}")
+            f.unlink()
+    old_logfile = pathlib.Path("pyaerocom.log")
+    if old_logfile.exists():
+        logger.warn(
+            f"no longer used old default logfile '{old_logfile}' exist, please consider deleting"
+        )

pyaerocom/io/read_airnow.py

@@ -335,7 +335,7 @@ def _read_files(self, files, vars_to_retrieve):
         varcol = self.FILE_COL_NAMES.index("variable")
         arrs = []
         unique_stat_ids = None
-        for i in tqdm(range(len(files))):
+        for i in tqdm(range(len(files)), disable=None):
             fp = files[i]
             filedata = self._read_file(fp)
             for i, filevar in enumerate(file_vars_to_retrieve):
@@ -421,7 +421,7 @@ def _filedata_to_statlist(
                 statlist = np.unique((subset[:, statcol]))
             else:
                 statlist = unique_stat_ids
-            for stat_id in tqdm(statlist, desc=var):
+            for stat_id in tqdm(statlist, desc=var, disable=None):
                 if not stat_id in stat_ids:
                     continue
                 statmask = subset[:, statcol] == stat_id

pyaerocom/io/read_ebas.py

@@ -1830,7 +1830,7 @@ def _read_files(self, files, vars_to_retrieve, files_contain, constraints):
         var_count_glob = -1
         logger.info(f"Reading EBAS data from {self.file_dir}")
         num_files = len(files)
-        for i in tqdm(range(num_files)):
+        for i in tqdm(range(num_files), disable=None):
             _file = files[i]
             contains = files_contain[i]
             try:

pyaerocom/io/read_eea_aqerep_base.py

@@ -653,7 +653,7 @@ def read(
         _country_dict = get_country_name_from_iso()
         logger.info("Reading files...")
 
-        for i in tqdm(range(len(files))):
+        for i in tqdm(range(len(files)), disable=None):
             _file = files[i]
             try:
                 station_data = self.read_file(_file, var_name=var_name)

pyaerocom/io/readaeronetbase.py

@@ -368,7 +368,7 @@ def read(
         num_files = len(files)
         logger.info("Reading AERONET data")
         skipped = 0
-        for i in tqdm(range(num_files)):
+        for i in tqdm(range(num_files), disable=None):
             _file = files[i]
             try:
                 station_data = self.read_file(_file, vars_to_retrieve=vars_to_retrieve)