diff --git a/doc/howto/setting-up-logging.rst b/doc/howto/setting-up-logging.rst index 8492c2729c..efd4ab216c 100644 --- a/doc/howto/setting-up-logging.rst +++ b/doc/howto/setting-up-logging.rst @@ -121,3 +121,102 @@ file. If installing NAV from the Debian packages provided by Sikt, log rotation through :program:`logrotate` is already provided for you (but you can change the rotation rules as you see fit). + + +Advanced logging configuration +============================== + +While a few simple use-cases for logging configuration are supported by +:file:`logging.conf`, much more advanced things can be achieved using the +alternative logging configuration file :file:`logging.yml`. Doing this on your +own, however, usually requires that you know your way around Python and have +extensive knowledge of how the standard Python logging framework works. + +:file:`logging.yml` is read and parsed as a Python dictionary, using +:func:`logging.config.dictConfig()`, right after :file:`logging.conf` is read +and parsed. This means that :file:`logging.yml` must adhere to the +configuration dictionary schema laid out in the Python docs. + +Be aware that by adding configuration to :file:`logging.yml`, you are altering +NAV's default logging configuration at a very low level, and you may also be +altering NAV's default behavior of storing logs in files. A :file:`logging.yml` +that replicates a default NAV setup may look something like this: + +.. code-block:: yaml + + version: 1 + loggers: + nav: + level: INFO + root: + handlers: [console] + + formatters: + default: + format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' + + handlers: + console: + class: logging.StreamHandler + formatter: default + +This replicates a setup that logs only **INFO**-level messages and above from +NAV to ``stderr``, using NAV's default log message format. Individual NAV +daemons will redirect their ``stderr`` streams to their respective log files as +they fork off background processes, so there is no need to redefine these. + +Leaving out the :class:`logging.StreamHandler` will still cause the log files +to be created, but they will be empty (save for any outpout to ``stderr`` that +did not come from the :mod:`logging` library). + +.. tip:: As with :file:`logging.conf`, processes can be directed to read a + bespoke :file:`logging.yml` file, but by setting the + :envvar:`NAV_LOGGING_YML` environment variable instead. + +Example: Directing logs to Falcon LogScale (Humio) +-------------------------------------------------- + +The following example shows how you can make all NAV programs ship their log +messages to a Falcon LogScale (previously known as Humio) ingestor using +something like the `humiologging `_ +library. Instead of shipping the file-based logs to LogScale and having them +parsed there, each log record can be shipped with structured attributes/tags. + +To achieve something like this, you need to first install the +:mod:`humiologging` library into your NAV installation's Python environment +(e.g. :code:`pip install humiologging`), and then create a :file:`logging.yml` +similar to this: + + +.. code-block:: yaml + + version: 1 + loggers: + nav: + level: DEBUG + root: + handlers: [humio, console] + + formatters: + default: + format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' + + handlers: + humio: + class: humiologging.handlers.HumioJSONHandler + level: DEBUG + humio_host: https://your-humio-ingest-addr-here + ingest_token: SECRET_TOKEN_THINGY + console: + class: logging.StreamHandler + formatter: default + + +This configuration attaches a :class:`HumioJSONHandler` to the ``root`` logger +and sets the global NAV log level to **DEBUG**. Unfortunately, as this +configuration manipulates the ``root`` logger, it removes the handler(s) that +NAV has by default installed on it, so if you want NAV to also keep logging to +files in addition to Humio, you need to replicate parts of NAV's default setup, +as mentioned in the previous section. Add an extra handler named ``console`` +that logs to a stream (``stderr`` by default), and specify a format for it. + diff --git a/python/nav/logs.py b/python/nav/logs.py index 89efe7e912..95d0ecec50 100644 --- a/python/nav/logs.py +++ b/python/nav/logs.py @@ -21,7 +21,10 @@ import sys import os import logging +import logging.config from itertools import chain +from typing import Optional +import yaml import configparser from nav.config import find_config_file, NAV_CONFIG @@ -31,6 +34,8 @@ ) LOGGING_CONF_VAR = 'NAV_LOGGING_CONF' LOGGING_CONF_FILE_DEFAULT = find_config_file('logging.conf') or '' +LOGGING_YAML_VAR = 'NAV_LOGGING_YAML' +LOGGING_YAML_FILE_DEFAULT = find_config_file('logging.yml') or '' _logger = logging.getLogger(__name__) @@ -39,6 +44,7 @@ def set_log_config(): """Set log levels and custom log files""" set_log_levels() _set_custom_log_file() + _read_dictconfig_from_yaml_file() def set_log_levels(): @@ -96,6 +102,18 @@ def _set_custom_log_file(): _logger.addHandler(filehandler) +def _read_dictconfig_from_yaml_file(): + """Reads legacy logging dictconfig from alternative yaml-based config file, if + possible. + + This allows for much more flexible logging configuration than NAV's standard logging + parameters. + """ + config = _get_logging_yaml() + if config: + logging.config.dictConfig(config) + + def _get_logging_conf(): """ Returns a ConfigParser with the logging configuration to use. @@ -119,6 +137,19 @@ def _get_logging_conf(): return config +def _get_logging_yaml() -> Optional[dict]: + """Returns a logging config dict from logging.yaml, if readable""" + filename = os.environ.get(LOGGING_YAML_VAR, LOGGING_YAML_FILE_DEFAULT) + try: + with open(filename, "rb") as yamlconf: + config = yaml.safe_load(yamlconf) + _logger.debug("Loaded logging config from %r", filename) + except OSError as error: + _logger.debug("Could not load yaml logging config: %r", error) + return None + return config + + def reset_log_levels(level=logging.WARNING): """Resets the log level of all loggers. diff --git a/tests/unittests/logs_test.py b/tests/unittests/logs_test.py new file mode 100644 index 0000000000..016954093f --- /dev/null +++ b/tests/unittests/logs_test.py @@ -0,0 +1,30 @@ +"""Tests for nav.logs module""" +import os +from unittest import mock + +import pytest + +from nav import logs + + +def test_get_logging_yaml(valid_logging_yaml): + """Tests the happy path. + + The failure path is covered implicitly by many other tests. + """ + with mock.patch.dict(os.environ, {"NAV_LOGGING_YAML": str(valid_logging_yaml)}): + config = logs._get_logging_yaml() + assert isinstance(config, dict) + + +@pytest.fixture +def valid_logging_yaml(tmp_path): + """Provides a minimally valid logging config file in YAML format""" + filename = tmp_path / "logging.yml" + with open(filename, "w") as yaml: + yaml.write( + """ + version: 1 + """ + ) + yield filename