feat: configure logging using settings system

docs/source/api.rst

@@ -115,6 +115,13 @@ Caching
     :show-inheritance:
 
 
+Logging
+~~~~~~~
+
+.. automodule:: vaex.logging
+    :members:
+
+
 Extensions
 ----------
 

packages/vaex-core/vaex/__init__.py

@@ -32,12 +32,16 @@
 Follow the tutorial at https://docs.vaex.io/en/latest/tutorial.html to learn how to use vaex.
 
 """  # -*- coding: utf-8 -*-
-from __future__ import print_function
-import glob
-import re
+import logging as root_logging
+import os
+import pkg_resources
 from typing import Dict, List
-from numpy.lib.function_base import copy
-import six
+from urllib.parse import urlparse, parse_qs
+
+# first configure logging, which also imports vaex.settings
+import vaex.logging
+# import this to be explicit
+import vaex.settings
 
 import vaex.dataframe
 import vaex.dataset
@@ -51,16 +55,11 @@
 from .groupby import *
 from . import agg
 import vaex.datasets
-# import vaex.plot
-# from vaex.dataframe import DataFrame
-# del ServerRest, DataFrame
 
-import vaex.settings
+
+
+
 import vaex.progress
-import logging
-import pkg_resources
-import os
-from functools import reduce
 
 try:
     from . import version
@@ -71,6 +70,8 @@
 
 vaex = vaex.utils.optional_import("vaex", modules=["vaex.ml"])
 
+logger = root_logging.getLogger('vaex')
+DEBUG_MODE = os.environ.get('VAEX_DEBUG', '')
 __version__ = version.get_versions()
 
 
@@ -257,7 +258,7 @@ def open(path, convert=False, progress=None, shuffle=False, fs_options={}, fs=No
             raise IOError('Unknown error opening: {}'.format(path))
         return df
     except:
-        logging.getLogger("vaex").exception("error opening %r" % path)
+        logger.exception("error opening %r" % path)
         raise
 
 
@@ -596,12 +597,6 @@ def read_csv(filepath_or_buffer, **kwargs):
 
 aliases = vaex.settings.aliases
 
-# py2/p3 compatibility
-try:
-    from urllib.parse import urlparse, parse_qs
-except ImportError:
-    from urlparse import urlparse, parse_qs
-
 
 def connect(url, **kwargs):
     """Connect to hostname supporting the vaex web api.
@@ -624,60 +619,35 @@ def example():
     '''
     return vaex.datasets.helmi_simulation_data()
 
-# create named logger, for all loglevels
-logger = logging.getLogger('vaex')
-logger.setLevel(logging.DEBUG)
-
-# create console handler and accept all loglevels
-ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
-
-# create formatter
-formatter = logging.Formatter('%(levelname)s:%(threadName)s:%(name)s:%(message)s')
 
-# add formatter to console handler
-ch.setFormatter(formatter)
-
-# add console handler to logger
-logger.addHandler(ch)
 
+# there are kept for backwards compatibility
+# TODO: remove in vaex v5?
 
 def set_log_level_debug(loggers=["vaex"]):
     """set log level to debug"""
-    for logger in loggers:
-        logging.getLogger(logger).setLevel(logging.DEBUG)
+    vaex.logging.set_log_level_debug(loggers)
 
 
-def set_log_level_info():
+def set_log_level_info(loggers=["vaex"]):
     """set log level to info"""
-    logging.getLogger("vaex").setLevel(logging.INFO)
+    vaex.logging.set_log_level_info(loggers)
 
 
-def set_log_level_warning():
+def set_log_level_warning(loggers=["vaex"]):
     """set log level to warning"""
-    logging.getLogger("vaex").setLevel(logging.WARNING)
+    vaex.logging.set_log_level_warning(loggers)
 
 
-def set_log_level_exception():
-    """set log level to exception"""
-    logging.getLogger("vaex").setLevel(logging.ERROR)
+def set_log_level_exception(loggers=["vaex"]):
+    """set log level to exception/error"""
+    vaex.logging.set_log_level_error(loggers)
 
 
 def set_log_level_off():
     """Disabled logging"""
-    logging.getLogger('vaex').removeHandler(ch)
-    logging.getLogger('vaex').addHandler(logging.NullHandler())
-
+    vaex.logging.set_log_level_off()
 
-DEBUG_MODE = os.environ.get('VAEX_DEBUG', '')
-if DEBUG_MODE:
-    set_log_level_warning()
-    if DEBUG_MODE.startswith('vaex'):
-        set_log_level_debug(DEBUG_MODE.split(","))
-    else:
-        set_log_level_debug()
-else:
-    set_log_level_warning()
 
 import_script = os.path.expanduser("~/.vaex/vaex_import.py")
 if os.path.exists(import_script):

packages/vaex-core/vaex/logging.py

@@ -0,0 +1,95 @@
+"""Sets up logging for vaex.
+
+See `configuration of logging <conf.html#logging>`_ how to configure logging.
+
+"""
+import os
+import vaex.settings
+import logging
+
+logger = logging.getLogger('vaex')
+log_handler : logging.Handler = None
+
+
+def set_log_level(loggers=["vaex"], level=logging.DEBUG):
+    """set log level to debug"""
+    for logger in loggers:
+        logging.getLogger(logger).setLevel(level)
+
+
+def set_log_level_debug(loggers=["vaex"]):
+    """set log level to debug"""
+    set_log_level(loggers, logging.DEBUG)
+
+
+def set_log_level_info(loggers=["vaex"]):
+    """set log level to info"""
+    set_log_level(loggers, logging.INFO)
+
+
+def set_log_level_warning(loggers=["vaex"]):
+    """set log level to warning"""
+    set_log_level(loggers, logging.WARNING)
+
+
+def set_log_level_error(loggers=["vaex"]):
+    """set log level to exception/error"""
+    set_log_level(loggers, logging.ERROR)
+
+
+def remove_handler():
+    """Disabled logging, remove default hander and add null handler"""
+    logging.getLogger('vaex').removeHandler(log_handler)
+    logging.getLogger('vaex').addHandler(logging.NullHandler())
+
+def reset():
+    '''Reset configuration of logging (i.e. remove the default handler)'''
+    logging.getLogger('vaex').removeHandler(log_handler)
+
+
+def _set_log_level(conf, level):
+    if conf:
+        if conf.startswith('vaex'):
+            set_log_level(conf.split(","), level=level)
+        else:
+            set_log_level(level=level)
+
+def setup():
+    """Setup logging based on the configuration in ``vaex.settings``
+
+    This function is automatically called when importing vaex. If settings are changed, call :func:`reset` and this function again
+    to re-apply the settings.
+    """
+    global log_handler
+
+    if vaex.settings.main.logging.setup:
+        logger.setLevel(logging.DEBUG)
+
+        # create console handler and accept all loglevels
+        if vaex.settings.main.logging.rich:
+            from rich.logging import RichHandler
+            log_handler = RichHandler()
+        else:
+            log_handler = logging.StreamHandler()
+
+            # create formatter
+            formatter = logging.Formatter('%(levelname)s:%(threadName)s:%(name)s:%(message)s')
+
+
+            # add formatter to console handler
+            log_handler.setFormatter(formatter)
+        log_handler.setLevel(logging.DEBUG)
+
+        # add console handler to logger
+        logger.addHandler(log_handler)
+
+    _set_log_level(vaex.settings.main.logging.error, logging.ERROR)
+    _set_log_level(vaex.settings.main.logging.warning, logging.WARNING)
+    _set_log_level(vaex.settings.main.logging.info, logging.INFO)
+    _set_log_level(vaex.settings.main.logging.debug, logging.DEBUG)
+    # VAEX_DEBUG behaves similar to VAEX_LOGGING_DEBUG, but has more effect
+    DEBUG_MODE = os.environ.get('VAEX_DEBUG', '')
+    if DEBUG_MODE:
+        _set_log_level(DEBUG_MODE, logging.DEBUG)
+
+setup()

packages/vaex-core/vaex/settings.py

@@ -124,6 +124,23 @@ class Config(ConfigDefault):
         env_prefix = 'vaex_data_'
 
 
+class Logging(BaseSettings):
+    """Configure logging for Vaex. By default Vaex sets up logging, which is useful when running a script. When Vaex is used in applications or services that already configure logging, set the environomental variables VAEX_LOGGING_SETUP to false.
+
+See the [API docs](api.html#module-vaex.logging) for more details.
+
+Note that settings `vaex.settings.main.logging.info` etc at runtime, has no direct effect, since logging is already configured. When needed, call `vaex.logging.reset()` and `vaex.logging.setup()` to reconfigure logging.
+    """
+    setup : bool = Field(True, title='Setup logging for Vaex at import time.')
+    rich : bool = Field(True, title='Use rich logger (colored fancy output).')
+    debug : str = Field('', title="Comma seperated list of loggers to set to the debug level (e.g. 'vaex.settings,vaex.cache'), or a '1' to set the root logger ('vaex')")
+    info : str = Field('', title="Comma seperated list of loggers to set to the info level (e.g. 'vaex.settings,vaex.cache'), or a '1' to set the root logger ('vaex')")
+    warning : str = Field('', title="Comma seperated list of loggers to set to the warning level (e.g. 'vaex.settings,vaex.cache'), or a '1' to set the root logger ('vaex')")
+    error : str = Field('', title="Comma seperated list of loggers to set to the error level (e.g. 'vaex.settings,vaex.cache'), or a '1' to set the root logger ('vaex')")
+    class Config(ConfigDefault):
+        env_prefix = 'vaex_logging_'
+
+
 class Settings(BaseSettings):
     """General settings for vaex"""
     aliases: Optional[dict] = Field(title='Aliases to be used for vaex.open', default_factory=dict)
@@ -146,6 +163,7 @@ class Settings(BaseSettings):
     fs: FileSystem = Field(FileSystem(), env='_VAEX_FS')
     memory_tracker = Field(MemoryTracker(), env='_VAEX_MEMORY_TRACKER')
     task_tracker = Field(TaskTracker(), env='_VAEX_TASK_TRACKER')
+    logging = Field(Logging(), env="_VAEX_LOGGING")
 
     if has_server:
         server: vaex.server.settings.Settings = vaex.server.settings.Settings()
@@ -256,6 +274,7 @@ def _to_md(cls, f=sys.stdout):
             Cache: 'cache',
             FileSystem: 'fs',
             Data: 'data',
+            Logging: 'main.logging',
         }[cls]
         pyvar = f'vaex.setting.{flat}.{pyname}'
         printf(f'Python setting `{pyvar}`')