Adding a logging module to ETOS library (#8)

Author: t-persson

src/etos_lib/lib/debug.py

@@ -18,6 +18,7 @@
 DEPRECATION WARNING: Some parameters which don't belong here will be removed.
 """
 import os
+from pathlib import Path
 from collections import deque
 
 
@@ -36,6 +37,16 @@ def default_secret_path(self):
         """Path to k8s secrets."""
         return os.getenv("ETOS_SECRET_PATH", "/etc/")
 
+    @property
+    def default_log_path(self):
+        """"Default log path."""
+        path = os.getenv("ETOS_LOG_PATH")
+        if path is None:
+            path = Path.home().joinpath("logs/log.json")
+        else:
+            path = Path(path)
+        return path
+
     @property
     def disable_sending_events(self):
         """Disable sending eiffel events."""

src/etos_lib/lib/logs.py

@@ -0,0 +1,16 @@
+# Copyright 2020 Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS logging module."""

src/etos_lib/logging/__init__.py

@@ -0,0 +1,30 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=generic,etos
+
+[formatters]
+keys=generic,etos
+
+[logger_root]
+level=DEBUG
+handlers=generic,etos
+
+[handler_generic]
+class=StreamHandler
+kwargs={"stream": sys.stdout}
+formatter=generic
+
+[handler_etos]
+class=FileHandler
+args=("%(logfilename)s",)
+formatter=etos
+
+[formatter_generic]
+format=[%(asctime)s][%(identifier)s] %(levelname)s: %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
+
+[formatter_etos]
+class=etos_lib.logging.formatter.EtosLogFormatter

src/etos_lib/logging/default_config.conf

@@ -0,0 +1,49 @@
+# Copyright 2020 Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS filter."""
+import logging
+
+
+class EtosFilter(logging.Filter):  # pylint:disable=too-few-public-methods
+    """Filter for adding extra application specific data to log messages."""
+
+    def __init__(self, application, version, environment):
+        """Initialize with a few ETOS application fields.
+
+        :param application: Name of application.
+        :type application: str
+        :param version: Version of application.
+        :type version: str
+        :param environment: In which environment is this executing.
+        :type environment: str
+        """
+        self.application = application
+        self.version = version
+        self.environment = environment
+        super().__init__()
+
+    def filter(self, record):
+        """Add contextual data to log record.
+
+        :param record: Log record to add data to.
+        :type record: :obj:`logging.LogRecord`
+        :return: True
+        :rtype: bool
+        """
+        record.application = self.application
+        record.version = self.version
+        record.environment = self.environment
+        return True

src/etos_lib/logging/filter.py

@@ -0,0 +1,101 @@
+# Copyright 2020 Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS log formatter."""
+import datetime
+import json
+import logging
+import sys
+import traceback
+
+
+# LogRecord fields that we exclude, typically because we offer
+# something better.
+_EXCLUDED_FIELDS = (
+    # Superfluous since we're saving the timestamp in '@timestamp'.
+    "asctime",
+    # The arguments to the format string can contain arbitrary objects
+    # that can't be serialized to JSON. We could pass them to str(),
+    # but that assumes that '%s' is used in the format string.
+    "args",
+    "msg",
+    # Exception info is saved in exception.{type,message,stacktrace}
+    # instead.
+    "exc_info",
+    "exc_text",
+    "stack_info",
+)
+
+
+class EtosLogFormatter(logging.Formatter):
+    """Logging formatter that produces a JSON object.
+
+    The resulting JSON object contains the logging.LogRecord fields
+    untouched. The following exception from this rule exist:
+
+        - A '@timestamp' key with the log record's UTC timestamp in ISO8601
+          format, digestible by Logstash.
+        - If an exception context exists (i.e. sys.exc_info() returns
+          something) the resulting JSON object will contain an 'exception'
+          key pointing to an object with the following keys:
+
+              - 'type': The fully-qualified type name (i.e. including
+                 package and module).
+              - 'message': The exception message.
+              - 'stacktrace': The formatted stacktrace as a multiline string
+                 (not ending in a newline character).
+    """
+
+    def format(self, record):
+        """Serialize LogRecord data as JSON.
+        Overrides the inherited behavior by ignoring any configured
+        format string and storing attributes of the passed LogRecord
+        object in a dictionary and serializing it to JSON. See class
+        docstring for details.
+        """
+        fields = {
+            k: v
+            for k, v in record.__dict__.items()
+            if k not in _EXCLUDED_FIELDS and not k.startswith("_")
+        }
+        fields["@timestamp"] = self.formatTime(record)
+        fields["message"] = record.getMessage()
+        exc_type, exc_message, stack = sys.exc_info()
+        if exc_type and exc_message and stack:
+            fields["exception"] = {
+                "type": "{}.{}".format(exc_type.__module__, exc_type.__name__),
+                "message": str(exc_message),
+                # format_tb() returns a list of newline-terminated strings.
+                # Don't include the final newline.
+                "stacktrace": "".join(traceback.format_tb(stack)).rstrip(),
+            }
+        # No need to append a newline character; that's up to the handler.
+        return json.dumps(fields)
+
+    def formatTime(self, record, datefmt=None):
+        """Formats the log record's timestamp in ISO8601 format, in UTC.
+        Overrides the inherited behavior by always returning ISO8601
+        strings without allowing custom date formats.
+        Raises:
+            NotImplementedError: If the `datefmt` parameter is not None.
+        """
+        if datefmt is not None:
+            raise NotImplementedError("Only default time format is supported.")
+        # Make Logstash's @timestamp parser happy by including a "T"
+        # between the date and the time. Append 'Z' to make it clear
+        # that the timestamp is UTC.
+        return datetime.datetime.utcfromtimestamp(record.created).strftime(
+            "%Y-%m-%dT%H:%M:%S.%fZ"
+        )

src/etos_lib/logging/formatter.py

@@ -0,0 +1,73 @@
+# Copyright 2020 Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS logger.
+
+Example::
+
+    from uuid import uuid4
+    from etos_lib.logging.logger import setup_logging, get_logger
+
+    setup_logging("myApp", "1.0.0", "production")
+    logger = get_logger(__name__, str(uuid4()))
+    logger.info("Hello!")
+    >>> [2020-12-16 10:35:00][cb7c8cd9-40a6-4ecc-8321-a1eae6beae35] INFO: Hello!
+
+"""
+from pathlib import Path
+import logging
+import logging.config
+from etos_lib.logging.filter import EtosFilter
+from etos_lib.lib.debug import Debug
+from etos_lib.lib.config import Config
+
+DEFAULT_CONFIG = Path(__file__).parent.joinpath("default_config.conf")
+DEFAULT_LOG_PATH = Debug().default_log_path
+DEFAULT_LOG_PATH.parent.mkdir(exist_ok=True, parents=True)
+
+
+def setup_logging(
+    application, version, environment, filename=DEFAULT_CONFIG, output=DEFAULT_LOG_PATH
+):
+    """Set up basic logging.
+
+    :param application: Name of application to setup logging for.
+    :type application: str
+    :param version: Version of application to setup logging for.
+    :type version: str
+    :param environment: Environment in which this application resides.
+    :type environment: str
+    :param filename: Filename of logging configuration.
+    :type filename: str
+    :param output: Output filename for logging to file.
+    :type output: str
+    """
+    Config().set("log_filter", EtosFilter(application, version, environment))
+    logging.config.fileConfig(filename, defaults={"logfilename": output})
+
+
+def get_logger(name, identifier):
+    """Get a logger adapter with attached identiifer.
+
+    :param name: Name of logger to get.
+    :type name: str
+    :param identifier: Unique identifier to attach to logger.
+    :type identifier: str
+    :return: LoggerAdapter instance.
+    :rtype: :obj:`logging.LoggerAdapter`
+    """
+    logger = logging.getLogger(name)
+    logger.addFilter(Config().get("log_filter"))
+    return logging.LoggerAdapter(logger, {"identifier": identifier})