better conditional logging support for syslog

src/quorum/__init__.py

@@ -99,7 +99,7 @@
 from .info import NAME, VERSION, AUTHOR, EMAIL, DESCRIPTION, LICENSE, KEYWORDS, URL,\
     COPYRIGHT
 from .jsonf import load_json
-from .log import MemoryHandler, ThreadFormatter, rotating_handler, smtp_handler,\
+from .log import MemoryHandler, BaseFormatter, ThreadFormatter, rotating_handler, smtp_handler,\
     in_signature, has_exception, debug, info, warning, error, critical
 from .mail import send_mail, send_mail_a
 from .meta import Ordered

src/quorum/base.py

@@ -43,6 +43,7 @@
 import time
 import flask
 import atexit
+import socket
 import logging
 import inspect
 import datetime
@@ -570,6 +571,20 @@ def start_log(
     # tries to retrieve some of the default configuration values
     # that are going to be used in the logger startup
     format = config.conf("LOGGING_FORMAT", None)
+    file_log = config.conf("FILE_LOG", False, cast = bool)
+    stream_log = config.conf("STREAM_LOG", True, cast = bool)
+    memory_log = config.conf("MEMORY_LOG", True, cast = bool)
+    syslog_host = config.conf("SYSLOG_HOST", None)
+    syslog_port = config.conf("SYSLOG_PORT", None, cast = int)
+    syslog_proto = config.conf("SYSLOG_PROTO", "udp")
+    syslog_kwargs = dict(socktype = socket.SOCK_STREAM) if\
+        syslog_proto in ("tcp",) else dict()
+    syslog_log = True if syslog_host else False
+
+    # tries to determine the default syslog port in case no port
+    # is defined and syslog logging is enabled
+    if not syslog_port and syslog_log:
+        syslog_port = log.SYSLOG_PORTS.get(syslog_proto)
 
     # "resolves" the proper logger file path taking into account
     # the currently defined operative system, should uses the system
@@ -599,13 +614,13 @@ def start_log(
 
     # creates both the stream and the memory based handlers that
     # are going to be used for the current logger
-    stream_handler = logging.StreamHandler()
-    memory_handler = log.MemoryHandler()
+    stream_handler = logging.StreamHandler() if stream_log else None
+    memory_handler = log.MemoryHandler() if memory_log else None
 
     try:
         # tries to create the file handler for the logger with the
         # resolve path (operation may fail for permissions)
-        file_handler = path and logging.FileHandler(path)
+        file_handler = path and file_log and logging.FileHandler(path)
     except:
         # in case there's an error creating the file handler for
         # the logger prints an error message indicating the problem
@@ -631,6 +646,25 @@ def start_log(
         handler.setFormatter(formatter)
         handler.setLevel(level)
 
+    # determines if the creation of the syslog handler is required and
+    # it that the case created it setting the appropriate formatter to it
+    syslog_handler = logging.handlers.SysLogHandler(
+        (syslog_host, syslog_port), **syslog_kwargs
+    ) if syslog_log else None if syslog_log else None
+
+    # in case the syslog handler has been created creates the appropriate
+    # formatter for it, sets the level and adds it to the logger
+    if syslog_handler:
+        syslog_formatter = log.BaseFormatter(
+            log.LOGGIGN_SYSLOG % "quorum",
+            datefmt = "%Y-%m-%dT%H:%M:%S.000000+00:00",
+            wrap = True
+        )
+        syslog_handler.setLevel(level)
+        syslog_handler.setFormatter(syslog_formatter)
+        logger.addHandler(syslog_handler)
+        app.handlers["syslog"] = syslog_handler
+
     # runs the extra logging step for the current state, meaning that
     # some more handlers may be created according to the logging config
     extra_logging(logger, level, formatter)

src/quorum/log.py

@@ -38,6 +38,8 @@
 """ The license for the module """
 
 import sys
+import json
+import socket
 import inspect
 import itertools
 import threading
@@ -66,6 +68,11 @@
 """ The extra logging attributes that are going to be applied
 to the format strings to obtain the final on the logging """
 
+LOGGIGN_SYSLOG = "1 %%(asctime)s %%(hostname)s %s %%(process)d %%(thread)d \
+[quorumSDID@0 tid=\"%%(thread)d\"] %%(json)s"
+""" The format to be used for the message sent using the syslog
+logger, should contain extra structured data """
+
 MAX_LENGTH = 10000
 """ The maximum amount of messages that are kept in
 memory until they are discarded, avoid a very large
@@ -98,6 +105,10 @@
 """ Map defining a series of alias that may be used latter
 for proper debug level resolution """
 
+SYSLOG_PORTS = dict(tcp = 601, udp = 514)
+""" Dictionary that maps the multiple transport protocol
+used by syslog with the appropriate default ports """
+
 LOGGING_FORMAT = LOGGING_FORMAT % LOGGING_EXTRA
 LOGGING_FORMAT_TID = LOGGING_FORMAT_TID % LOGGING_EXTRA
 
@@ -189,28 +200,73 @@ def get_latest(self, count = None, level = None):
         slice = itertools.islice(messages, 0, count)
         return list(slice)
 
-class ThreadFormatter(logging.Formatter):
+class BaseFormatter(logging.Formatter):
+    """
+    The base Quorum logging formatted used to add some extra
+    functionality on top of Python's base formatting infra-structure.
+
+    Most of its usage focus on empowering the base record object
+    with some extra values.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._wrap = kwargs.pop("wrap", False)
+        logging.Formatter.__init__(self, *args, **kwargs)
+
+    @classmethod
+    def _wrap_record(cls, record):
+        if hasattr(record, "_wrapped"): return
+        record.hostname = socket.gethostname()
+        record.json = json.dumps(
+            dict(
+                message = str(record.msg),
+                hostname = record.hostname,
+                lineno = record.lineno,
+                module = record.module,
+                callable = record.funcName,
+                level = record.levelname,
+                thread = record.thread,
+                process = record.process,
+                logger = record.name
+            )
+        )
+        record._wrapped = True
+
+    def format(self, record):
+        # runs the wrapping operation on the record so that more
+        # information becomes available in it (as expected)
+        if self._wrap: self.__class__._wrap_record(record)
+
+        # runs the basic format operation on the record so that
+        # it gets properly formatted into a plain string
+        return logging.Formatter.format(self, record)
+
+class ThreadFormatter(BaseFormatter):
     """
     Custom formatter class that changing the default format
     behavior so that the thread identifier is printed when
     the threading printing the log records is not the main one.
     """
 
     def __init__(self, *args, **kwargs):
-        logging.Formatter.__init__(self, *args, **kwargs)
-        self._tidfmt = logging.Formatter(self._fmt)
+        BaseFormatter.__init__(self, *args, **kwargs)
+        self._tidfmt = BaseFormatter(*args, **kwargs)
 
     def format(self, record):
+        # runs the wrapping operation on the record so that more
+        # information becomes available in it (as expected)
+        if self._wrap: self.__class__._wrap_record(record)
+
         # retrieves the reference to the current thread and verifies
         # if it represent the current process main thread, then selects
         # the appropriate formating string taking that into account
         current = threading.current_thread()
         is_main = current.name == "MainThread"
         if not is_main: return self._tidfmt.format(record)
-        return logging.Formatter.format(self, record)
+        return BaseFormatter.format(self, record)
 
-    def set_tid(self, value):
-        self._tidfmt = logging.Formatter(value)
+    def set_tid(self, value, *args, **kwargs):
+        self._tidfmt = logging.Formatter(value, *args, **kwargs)
 
 def rotating_handler(
     path = "quorum.log",