From f333b9d46d0a165bffd61a0d2a7dbc07e8534e92 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 10 Aug 2023 15:03:59 +0200 Subject: [PATCH 1/4] Read logging config from logging.yml if available This adds the ability to override most of NAV's logging config with the more flexible dictConfig format, as read from an optional `logging.yml` file (which can also be overriden by environment variable, just like `logging.conf`) --- python/nav/logs.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/nav/logs.py b/python/nav/logs.py index 89efe7e912..95d0ecec50 100644 --- a/python/nav/logs.py +++ b/python/nav/logs.py @@ -21,7 +21,10 @@ import sys import os import logging +import logging.config from itertools import chain +from typing import Optional +import yaml import configparser from nav.config import find_config_file, NAV_CONFIG @@ -31,6 +34,8 @@ ) LOGGING_CONF_VAR = 'NAV_LOGGING_CONF' LOGGING_CONF_FILE_DEFAULT = find_config_file('logging.conf') or '' +LOGGING_YAML_VAR = 'NAV_LOGGING_YAML' +LOGGING_YAML_FILE_DEFAULT = find_config_file('logging.yml') or '' _logger = logging.getLogger(__name__) @@ -39,6 +44,7 @@ def set_log_config(): """Set log levels and custom log files""" set_log_levels() _set_custom_log_file() + _read_dictconfig_from_yaml_file() def set_log_levels(): @@ -96,6 +102,18 @@ def _set_custom_log_file(): _logger.addHandler(filehandler) +def _read_dictconfig_from_yaml_file(): + """Reads legacy logging dictconfig from alternative yaml-based config file, if + possible. + + This allows for much more flexible logging configuration than NAV's standard logging + parameters. + """ + config = _get_logging_yaml() + if config: + logging.config.dictConfig(config) + + def _get_logging_conf(): """ Returns a ConfigParser with the logging configuration to use. @@ -119,6 +137,19 @@ def _get_logging_conf(): return config +def _get_logging_yaml() -> Optional[dict]: + """Returns a logging config dict from logging.yaml, if readable""" + filename = os.environ.get(LOGGING_YAML_VAR, LOGGING_YAML_FILE_DEFAULT) + try: + with open(filename, "rb") as yamlconf: + config = yaml.safe_load(yamlconf) + _logger.debug("Loaded logging config from %r", filename) + except OSError as error: + _logger.debug("Could not load yaml logging config: %r", error) + return None + return config + + def reset_log_levels(level=logging.WARNING): """Resets the log level of all loggers. From fe723e86ee04c5d42ecfb4bb86885941b38950e2 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 11 Aug 2023 11:44:13 +0200 Subject: [PATCH 2/4] Unit test nav.logs._get_logging_yaml --- tests/unittests/logs_test.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/unittests/logs_test.py diff --git a/tests/unittests/logs_test.py b/tests/unittests/logs_test.py new file mode 100644 index 0000000000..016954093f --- /dev/null +++ b/tests/unittests/logs_test.py @@ -0,0 +1,30 @@ +"""Tests for nav.logs module""" +import os +from unittest import mock + +import pytest + +from nav import logs + + +def test_get_logging_yaml(valid_logging_yaml): + """Tests the happy path. + + The failure path is covered implicitly by many other tests. + """ + with mock.patch.dict(os.environ, {"NAV_LOGGING_YAML": str(valid_logging_yaml)}): + config = logs._get_logging_yaml() + assert isinstance(config, dict) + + +@pytest.fixture +def valid_logging_yaml(tmp_path): + """Provides a minimally valid logging config file in YAML format""" + filename = tmp_path / "logging.yml" + with open(filename, "w") as yaml: + yaml.write( + """ + version: 1 + """ + ) + yield filename From ad3b47eee7dbf4d4e35b98117ed9497678bda3d7 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 11 Aug 2023 14:17:43 +0200 Subject: [PATCH 3/4] Document logging.yml capabilites --- doc/howto/setting-up-logging.rst | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/doc/howto/setting-up-logging.rst b/doc/howto/setting-up-logging.rst index 8492c2729c..aa113f1ba4 100644 --- a/doc/howto/setting-up-logging.rst +++ b/doc/howto/setting-up-logging.rst @@ -121,3 +121,73 @@ file. If installing NAV from the Debian packages provided by Sikt, log rotation through :program:`logrotate` is already provided for you (but you can change the rotation rules as you see fit). + + +Advanced logging configuration +============================== + +While a few simple use-cases for logging configuration are supported by +:file:`logging.conf`, much more advanced things can be achieved using the +alternative logging configuration file :file:`logging.yml`. Doing this on your +own, however, usually requires that you know your way around Python and have +extensive knowledge of how the standard Python logging framework works. + +:file:`logging.yml` is read and parsed as a Python dictionary, using +:func:`logging.config.dictConfig()`, right after :file:`logging.conf` is read +and parsed. This means that :file:`logging.yml` must adhere to the +configuration dictionary schema laid out in the Python docs. + +This can allow you to do such things as sending all NAV log records to a Falcon +LogScale (previously known as Humio) ingestor using something like the +`humiologging `_ library. Instead of +shipping the file-based logs to LogScale and having them parsed there, each log +record can be shipped with structured attributes/tags. + +To achieve something like this, you need to first install the +:mod:`humiologging` library into your NAV installation's Python environment +(e.g. :code:`pip install humiologging`), and then create a :file:`logging.yml` +similar to this: + + +.. code-block:: yaml + + version: 1 + loggers: + nav: + level: DEBUG + root: + handlers: [humio, console] + + handlers: + humio: + class: humiologging.handlers.HumioJSONHandler + level: DEBUG + humio_host: https://your-humio-ingest-addr-here + ingest_token: SECRET_TOKEN_THINGY + console: + class: logging.StreamHandler + formatter: default + + + formatters: + default: + format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' + +This configuration attaches a :class:`HumioJSONHandler` to the ``root`` logger +and sets the global NAV log level to **DEBUG**. Unfortunately, as this +configuration manipulates the ``root`` logger, it removes the handler(s) that +NAV has by default installed on it, so if you want NAV to also keep logging to +files in addition to Humio, you need to add an extra handler that logs to a +stream (``stderr`` by default), and you need to specify a format for it. This +example just redefines the log line format NAV uses by default. + +As long as you add a :class:`logging.StreamHandler` to ``root``, you shouldn't +need to redefine which files the logs go to, as most NAV daemons achieve this +by redirecting their ``stderr`` to a file as they fork off a background +process. Leaving out the :class:`logging.StreamHandler` will still cause the +log files to be created, but they will be empty (save for any outpout to +``stderr`` that did not come from the :mod:`logging` library). + +.. tip:: As with :file:`logging.conf`, processes can be directed to read a + bespoke :file:`logging.yml` file, but by setting the + :envvar:`NAV_LOGGING_YML` environment variable instead. From 43cabfad1500a630d90f27a45c4877e63e84b2a8 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 7 Sep 2023 11:06:03 +0200 Subject: [PATCH 4/4] Add default logging config example As requested in review, this begins by documenting how to replicate NAV's default logging config, and proceeds to document Humio only after that. --- doc/howto/setting-up-logging.rst | 73 ++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/doc/howto/setting-up-logging.rst b/doc/howto/setting-up-logging.rst index aa113f1ba4..efd4ab216c 100644 --- a/doc/howto/setting-up-logging.rst +++ b/doc/howto/setting-up-logging.rst @@ -137,11 +137,50 @@ extensive knowledge of how the standard Python logging framework works. and parsed. This means that :file:`logging.yml` must adhere to the configuration dictionary schema laid out in the Python docs. -This can allow you to do such things as sending all NAV log records to a Falcon -LogScale (previously known as Humio) ingestor using something like the -`humiologging `_ library. Instead of -shipping the file-based logs to LogScale and having them parsed there, each log -record can be shipped with structured attributes/tags. +Be aware that by adding configuration to :file:`logging.yml`, you are altering +NAV's default logging configuration at a very low level, and you may also be +altering NAV's default behavior of storing logs in files. A :file:`logging.yml` +that replicates a default NAV setup may look something like this: + +.. code-block:: yaml + + version: 1 + loggers: + nav: + level: INFO + root: + handlers: [console] + + formatters: + default: + format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' + + handlers: + console: + class: logging.StreamHandler + formatter: default + +This replicates a setup that logs only **INFO**-level messages and above from +NAV to ``stderr``, using NAV's default log message format. Individual NAV +daemons will redirect their ``stderr`` streams to their respective log files as +they fork off background processes, so there is no need to redefine these. + +Leaving out the :class:`logging.StreamHandler` will still cause the log files +to be created, but they will be empty (save for any outpout to ``stderr`` that +did not come from the :mod:`logging` library). + +.. tip:: As with :file:`logging.conf`, processes can be directed to read a + bespoke :file:`logging.yml` file, but by setting the + :envvar:`NAV_LOGGING_YML` environment variable instead. + +Example: Directing logs to Falcon LogScale (Humio) +-------------------------------------------------- + +The following example shows how you can make all NAV programs ship their log +messages to a Falcon LogScale (previously known as Humio) ingestor using +something like the `humiologging `_ +library. Instead of shipping the file-based logs to LogScale and having them +parsed there, each log record can be shipped with structured attributes/tags. To achieve something like this, you need to first install the :mod:`humiologging` library into your NAV installation's Python environment @@ -158,6 +197,10 @@ similar to this: root: handlers: [humio, console] + formatters: + default: + format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' + handlers: humio: class: humiologging.handlers.HumioJSONHandler @@ -169,25 +212,11 @@ similar to this: formatter: default - formatters: - default: - format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s' - This configuration attaches a :class:`HumioJSONHandler` to the ``root`` logger and sets the global NAV log level to **DEBUG**. Unfortunately, as this configuration manipulates the ``root`` logger, it removes the handler(s) that NAV has by default installed on it, so if you want NAV to also keep logging to -files in addition to Humio, you need to add an extra handler that logs to a -stream (``stderr`` by default), and you need to specify a format for it. This -example just redefines the log line format NAV uses by default. +files in addition to Humio, you need to replicate parts of NAV's default setup, +as mentioned in the previous section. Add an extra handler named ``console`` +that logs to a stream (``stderr`` by default), and specify a format for it. -As long as you add a :class:`logging.StreamHandler` to ``root``, you shouldn't -need to redefine which files the logs go to, as most NAV daemons achieve this -by redirecting their ``stderr`` to a file as they fork off a background -process. Leaving out the :class:`logging.StreamHandler` will still cause the -log files to be created, but they will be empty (save for any outpout to -``stderr`` that did not come from the :mod:`logging` library). - -.. tip:: As with :file:`logging.conf`, processes can be directed to read a - bespoke :file:`logging.yml` file, but by setting the - :envvar:`NAV_LOGGING_YML` environment variable instead.