Merge ed91b3c into a507c8f

dataengineeringutils3/logging.py

@@ -0,0 +1,49 @@
+import logging
+import io
+
+from typing import Tuple
+
+default_fmt = "%(asctime)s | %(funcName)s | %(levelname)s | %(context)s | %(message)s"
+default_date_fmt = "%Y-%m-%d %H:%M:%S"
+
+
+class ContextFilter(logging.Filter):
+    """
+    This is just overkill to apply a default context param to the log.
+    But it does mean I don't have to define extra everytime I wanna log.
+    So keeping it.
+    """
+
+    def filter(self, record):
+        if not getattr(record, "context", None):
+            record.context = "PROCESSING"
+        return True
+
+
+def get_logger(
+    fmt=default_fmt, datefmt=default_date_fmt
+) -> Tuple[logging.Logger, io.StringIO]:
+    """
+    returns a logger object and an io stream of the data that is logged
+    """
+
+    log = logging.getLogger("root")
+    log.setLevel(logging.DEBUG)
+
+    log_stringio = io.StringIO()
+    handler = logging.StreamHandler(log_stringio)
+
+    log_formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
+    handler.setFormatter(log_formatter)
+    log.addHandler(handler)
+
+    # Add console output
+    console = logging.StreamHandler()
+    console.setLevel(logging.INFO)
+    console.setFormatter(log_formatter)
+    log.addHandler(console)
+
+    cf = ContextFilter()
+    log.addFilter(cf)
+
+    return log, log_stringio

tests/test_logger.py

@@ -0,0 +1,54 @@
+import pytest
+import re
+
+from dataengineeringutils3.logging import get_logger
+
+
+@pytest.mark.parametrize("context_filter", (None, {"context": "A_DIFF_CONTEXT"}))
+def test_output(context_filter):
+    """
+    ensures the log ouput is as expected (including context filter)
+    """
+
+    # get the logger and the IO stream
+    logger, logger_io_stream = get_logger()
+
+    # log and retrieve a message
+    log_message = "a message!"
+    logger.info(log_message, extra=context_filter)
+    a = logger_io_stream.getvalue()
+
+    # get the required context
+    context = "PROCESSING" if not context_filter else list(context_filter.values())[0]
+
+    # ensure it matches the required pattern
+    regex = re.compile(
+        r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| "
+        f"test_output | INFO | {context} | {log_message}$"
+    )
+
+    assert regex.search(a)
+
+
+def test_diff_fmt():
+    """
+    makes sure different formats work correctly
+    """
+
+    # get the logger and the IO stream
+    logger, logger_io_stream = get_logger(
+        fmt="%(asctime)s | %(module)s %(table)s | %(levelname)s | %(message)s"
+    )
+
+    # log and retrieve a message
+    log_message = "a message!"
+    logger.info(log_message, extra={"table": "a_very_nice_table"})
+    a = logger_io_stream.getvalue()
+
+    # ensure it matches the required pattern
+    regex = re.compile(
+        r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| "
+        f"{__name__.split('.')[1]} a_very_nice_table | INFO | {log_message}$"
+    )
+
+    assert regex.search(a)