Merge 2357e46 into a507c8f

dataengineeringutils3/logging.py

@@ -0,0 +1,49 @@
+import logging
+import io
+
+from typing import Tuple
+
+default_fmt = "%(asctime)s | %(funcName)s | %(levelname)s | %(message)s"
+default_date_fmt = "%Y-%m-%d %H:%M:%S"
+
+
+def _make_fmt_json(fmt: str, sep: str):
+    fmt_list = fmt.split(sep)
+    fmt_val_list = [i.strip() for i in fmt_list]
+    fmt_key_list = [i[2:-2] for i in fmt_val_list]
+
+    json_str = ""
+    for key, val in zip(fmt_key_list, fmt_val_list):
+        json_str += f'"{key}" : "{val}", '
+    return "{" + json_str[:-2] + "}"
+
+
+def get_logger(
+    fmt: str = default_fmt,
+    datefmt: str = default_date_fmt,
+    output_format="human",
+    sep: str = "|",
+) -> Tuple[logging.Logger, io.StringIO]:
+    """
+    returns a logger object and an io stream of the data that is logged
+    """
+
+    fmt = _make_fmt_json(fmt, sep) if output_format == "json" else fmt
+
+    log = logging.getLogger("root")
+    log.setLevel(logging.DEBUG)
+
+    log_stringio = io.StringIO()
+    handler = logging.StreamHandler(log_stringio)
+
+    log_formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
+    handler.setFormatter(log_formatter)
+    log.addHandler(handler)
+
+    # Add console output
+    console = logging.StreamHandler()
+    console.setLevel(logging.INFO)
+    console.setFormatter(log_formatter)
+    log.addHandler(console)
+
+    return log, log_stringio

tests/test_logger.py

@@ -0,0 +1,67 @@
+import json
+import re
+
+from dataengineeringutils3.logging import get_logger
+
+
+def test_human_output():
+    """
+    ensures the log ouput is as expected (including context filter)
+    """
+
+    # get the logger and the IO stream
+    logger, logger_io_stream = get_logger()
+
+    # log and retrieve a message
+    log_message = "a message!"
+    logger.info(log_message)
+    a = logger_io_stream.getvalue()
+
+    # ensure it matches the required pattern
+    regex = re.compile(
+        r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| "
+        f"test_output | INFO | {log_message}$"
+    )
+
+    assert regex.search(a)
+
+
+def test_json_output():
+    """
+    tests the format of the output is valid json
+    """
+
+    # get the logger and the IO stream
+    logger, logger_io_stream = get_logger(output_format="json")
+
+    # log and retrieve a message
+    log_message = "a message!"
+    logger.info(log_message)
+    a = logger_io_stream.getvalue()
+
+    # assert that it can be json loaded
+    assert json.loads(a)
+
+
+def test_diff_fmt():
+    """
+    makes sure different formats work correctly
+    """
+
+    # get the logger and the IO stream
+    logger, logger_io_stream = get_logger(
+        fmt="%(asctime)s | %(module)s %(table)s | %(levelname)s | %(message)s"
+    )
+
+    # log and retrieve a message
+    log_message = "a message!"
+    logger.info(log_message, extra={"table": "a_very_nice_table"})
+    a = logger_io_stream.getvalue()
+
+    # ensure it matches the required pattern
+    regex = re.compile(
+        r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| "
+        f"{__name__.split('.')[1]} a_very_nice_table | INFO | {log_message}$"
+    )
+
+    assert regex.search(a)