Read logging config from logging.yml if available

doc/howto/setting-up-logging.rst

@@ -121,3 +121,102 @@ file.
 If installing NAV from the Debian packages provided by Sikt, log rotation
 through :program:`logrotate` is already provided for you (but you can change
 the rotation rules as you see fit).
+
+
+Advanced logging configuration
+==============================
+
+While a few simple use-cases for logging configuration are supported by
+:file:`logging.conf`, much more advanced things can be achieved using the
+alternative logging configuration file :file:`logging.yml`.  Doing this on your
+own, however, usually requires that you know your way around Python and have
+extensive knowledge of how the standard Python logging framework works.
+
+:file:`logging.yml` is read and parsed as a Python dictionary, using
+:func:`logging.config.dictConfig()`, right after :file:`logging.conf` is read
+and parsed.  This means that :file:`logging.yml` must adhere to the
+configuration dictionary schema laid out in the Python docs.
+
+Be aware that by adding configuration to :file:`logging.yml`, you are altering
+NAV's default logging configuration at a very low level, and you may also be
+altering NAV's default behavior of storing logs in files. A :file:`logging.yml`
+that replicates a default NAV setup may look something like this:
+
+.. code-block:: yaml
+
+   version: 1
+   loggers:
+     nav:
+       level: INFO
+     root:
+       handlers: [console]
+
+   formatters:
+     default:
+       format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s'
+
+   handlers:
+     console:
+       class: logging.StreamHandler
+       formatter: default
+
+This replicates a setup that logs only **INFO**-level messages and above from
+NAV to ``stderr``, using NAV's default log message format.  Individual NAV
+daemons will redirect their ``stderr`` streams to their respective log files as
+they fork off background processes, so there is no need to redefine these.
+
+Leaving out the :class:`logging.StreamHandler` will still cause the log files
+to be created, but they will be empty (save for any outpout to ``stderr`` that
+did not come from the :mod:`logging` library).
+
+.. tip:: As with :file:`logging.conf`, processes can be directed to read a
+         bespoke :file:`logging.yml` file, but by setting the
+         :envvar:`NAV_LOGGING_YML` environment variable instead.
+
+Example: Directing logs to Falcon LogScale (Humio)
+--------------------------------------------------
+
+The following example shows how you can make all NAV programs ship their log
+messages to a Falcon LogScale (previously known as Humio) ingestor using
+something like the `humiologging <https://pypi.org/project/humiologging/>`_
+library.  Instead of shipping the file-based logs to LogScale and having them
+parsed there, each log record can be shipped with structured attributes/tags.
+
+To achieve something like this, you need to first install the
+:mod:`humiologging` library into your NAV installation's Python environment
+(e.g. :code:`pip install humiologging`), and then create a :file:`logging.yml`
+similar to this:
+
+
+.. code-block:: yaml
+
+   version: 1
+   loggers:
+     nav:
+       level: DEBUG
+     root:
+       handlers: [humio, console]
+
+   formatters:
+     default:
+       format: '%(asctime)s [%(levelname)s] [%(name)s] %(message)s'
+
+   handlers:
+     humio:
+       class: humiologging.handlers.HumioJSONHandler
+       level: DEBUG
+       humio_host: https://your-humio-ingest-addr-here
+       ingest_token: SECRET_TOKEN_THINGY
+     console:
+       class: logging.StreamHandler
+       formatter: default
+
+
+This configuration attaches a :class:`HumioJSONHandler` to the ``root`` logger
+and sets the global NAV log level to **DEBUG**.  Unfortunately, as this
+configuration manipulates the ``root`` logger, it removes the handler(s) that
+NAV has by default installed on it, so if you want NAV to also keep logging to
+files in addition to Humio, you need to replicate parts of NAV's default setup,
+as mentioned in the previous section.  Add an extra handler named ``console``
+that logs to a stream (``stderr`` by default), and specify a format for it.
+

python/nav/logs.py

@@ -21,7 +21,10 @@
 import sys
 import os
 import logging
+import logging.config
 from itertools import chain
+from typing import Optional
+import yaml
 
 import configparser
 from nav.config import find_config_file, NAV_CONFIG
@@ -31,6 +34,8 @@
 )
 LOGGING_CONF_VAR = 'NAV_LOGGING_CONF'
 LOGGING_CONF_FILE_DEFAULT = find_config_file('logging.conf') or ''
+LOGGING_YAML_VAR = 'NAV_LOGGING_YAML'
+LOGGING_YAML_FILE_DEFAULT = find_config_file('logging.yml') or ''
 
 _logger = logging.getLogger(__name__)
 
@@ -39,6 +44,7 @@ def set_log_config():
     """Set log levels and custom log files"""
     set_log_levels()
     _set_custom_log_file()
+    _read_dictconfig_from_yaml_file()
 
 
 def set_log_levels():
@@ -96,6 +102,18 @@ def _set_custom_log_file():
         _logger.addHandler(filehandler)
 
 
+def _read_dictconfig_from_yaml_file():
+    """Reads legacy logging dictconfig from alternative yaml-based config file, if
+    possible.
+
+    This allows for much more flexible logging configuration than NAV's standard logging
+    parameters.
+    """
+    config = _get_logging_yaml()
+    if config:
+        logging.config.dictConfig(config)
+
+
 def _get_logging_conf():
     """
     Returns a ConfigParser with the logging configuration to use.
@@ -119,6 +137,19 @@ def _get_logging_conf():
     return config
 
 
+def _get_logging_yaml() -> Optional[dict]:
+    """Returns a logging config dict from logging.yaml, if readable"""
+    filename = os.environ.get(LOGGING_YAML_VAR, LOGGING_YAML_FILE_DEFAULT)
+    try:
+        with open(filename, "rb") as yamlconf:
+            config = yaml.safe_load(yamlconf)
+            _logger.debug("Loaded logging config from %r", filename)
+    except OSError as error:
+        _logger.debug("Could not load yaml logging config: %r", error)
+        return None
+    return config
+
+
 def reset_log_levels(level=logging.WARNING):
     """Resets the log level of all loggers.
 

tests/unittests/logs_test.py

@@ -0,0 +1,30 @@
+"""Tests for nav.logs module"""
+import os
+from unittest import mock
+
+import pytest
+
+from nav import logs
+
+
+def test_get_logging_yaml(valid_logging_yaml):
+    """Tests the happy path.
+
+    The failure path is covered implicitly by many other tests.
+    """
+    with mock.patch.dict(os.environ, {"NAV_LOGGING_YAML": str(valid_logging_yaml)}):
+        config = logs._get_logging_yaml()
+        assert isinstance(config, dict)
+
+
+@pytest.fixture
+def valid_logging_yaml(tmp_path):
+    """Provides a minimally valid logging config file in YAML format"""
+    filename = tmp_path / "logging.yml"
+    with open(filename, "w") as yaml:
+        yaml.write(
+            """
+        version: 1
+        """
+        )
+    yield filename