logutil: move logging bits to a new logutil module in ohmu_common_py

pglookout/cluster_monitor.py

@@ -8,7 +8,8 @@
 See the file `LICENSE` for details.
 """
 
-from .common import get_iso_timestamp, parse_iso_datetime, set_syslog_handler
+from . import logutil
+from .common import get_iso_timestamp, parse_iso_datetime
 from .pgutil import mask_connection_info
 from concurrent.futures import as_completed, ThreadPoolExecutor
 from email.utils import parsedate
@@ -65,9 +66,11 @@ def __init__(self, config, cluster_state, observer_state, create_alert_file, tri
         self.trigger_check_queue = trigger_check_queue
         self.session = requests.Session()
         if self.config.get("syslog"):
-            self.syslog_handler = set_syslog_handler(self.config.get("syslog_address", "/dev/log"),
-                                                     self.config.get("syslog_facility", "local2"),
-                                                     self.log)
+            self.syslog_handler = logutil.set_syslog_handler(
+                address=self.config.get("syslog_address", "/dev/log"),
+                facility=self.config.get("syslog_facility", "local2"),
+                logger=self.log,
+            )
         self.log.debug("Initialized ClusterMonitor with: %r", cluster_state)
 
     def _connect_to_db(self, instance, dsn):

pglookout/common.py

@@ -5,12 +5,8 @@
 See LICENSE for details
 """
 import datetime
-import logging
 import re
 
-LOG_FORMAT = "%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s"
-LOG_FORMAT_SYSLOG = '%(name)s %(levelname)s: %(message)s'
-
 
 def convert_xlog_location_to_offset(xlog_location):
     log_id, offset = xlog_location.split("/")
@@ -40,11 +36,3 @@ def get_iso_timestamp(fetch_time=None):
     elif fetch_time.tzinfo:
         fetch_time = fetch_time.replace(tzinfo=None) - datetime.timedelta(seconds=fetch_time.utcoffset().seconds)
     return fetch_time.isoformat() + "Z"
-
-
-def set_syslog_handler(syslog_address, syslog_facility, logger):
-    syslog_handler = logging.handlers.SysLogHandler(address=syslog_address, facility=syslog_facility)
-    logger.addHandler(syslog_handler)
-    formatter = logging.Formatter(LOG_FORMAT_SYSLOG)
-    syslog_handler.setFormatter(formatter)
-    return syslog_handler

pglookout/logutil.py

@@ -0,0 +1,47 @@
+# Copied from https://github.com/ohmu/ohmu_common_py ohmu_common_py/logutil.py version 0.0.1-0-unknown-fa54b44
+"""
+pglookout - logging formats and utility functions
+
+Copyright (c) 2015 Ohmu Ltd
+See LICENSE for details
+"""
+
+import logging
+import logging.handlers
+import os
+
+try:
+    from systemd import daemon  # pylint: disable=no-name-in-module
+except ImportError:
+    daemon = None
+
+
+LOG_FORMAT = "%(asctime)s\t%(name)s\t%(threadName)s\t%(levelname)s\t%(message)s"
+LOG_FORMAT_SHORT = "%(levelname)s\t%(message)s"
+LOG_FORMAT_SYSLOG = "%(name)s %(threadName)s %(levelname)s: %(message)s"
+
+
+def set_syslog_handler(address, facility, logger):
+    syslog_handler = logging.handlers.SysLogHandler(address=address, facility=facility)
+    logger.addHandler(syslog_handler)
+    formatter = logging.Formatter(LOG_FORMAT_SYSLOG)
+    syslog_handler.setFormatter(formatter)
+    return syslog_handler
+
+
+def configure_logging(level=logging.DEBUG, short_log=False):
+    # Are we running under systemd?
+    if os.getenv("NOTIFY_SOCKET"):
+        logging.basicConfig(level=level, format=LOG_FORMAT_SYSLOG)
+        if not daemon:
+            print(
+                "WARNING: Running under systemd but python-systemd not available, "
+                "systemd won't see our notifications"
+            )
+    else:
+        logging.basicConfig(level=level, format=LOG_FORMAT_SHORT if short_log else LOG_FORMAT)
+
+
+def notify_systemd(status):
+    if daemon:
+        daemon.notify(status)

pglookout/pglookout.py

@@ -9,11 +9,9 @@
 """
 
 from __future__ import print_function
-from . import statsd, version
+from . import logutil, statsd, version
 from .cluster_monitor import ClusterMonitor
-from .common import (
-    convert_xlog_location_to_offset, parse_iso_datetime, get_iso_timestamp,
-    set_syslog_handler, LOG_FORMAT, LOG_FORMAT_SYSLOG)
+from .common import convert_xlog_location_to_offset, parse_iso_datetime, get_iso_timestamp
 from .pgutil import (
     create_connection_string, get_connection_info, get_connection_info_from_config_line)
 from .webserver import WebServer
@@ -36,11 +34,6 @@
 except ImportError:
     from Queue import Queue  # pylint: disable=import-error
 
-try:
-    from systemd import daemon  # pylint: disable=import-error
-except:
-    daemon = None
-
 
 class PgLookout(object):
     def __init__(self, config_path):
@@ -89,9 +82,7 @@ def __init__(self, config_path):
         self.cluster_monitor.log.setLevel(self.log_level)
         self.webserver = WebServer(self.config, self.cluster_state)
 
-        if daemon:  # If we can import systemd we always notify it
-            daemon.notify("READY=1")
-            self.log.info("Sent startup notification to systemd that pglookout is READY")
+        logutil.notify_systemd("READY=1")
         self.log.info("PGLookout initialized, local hostname: %r, own_db: %r, cwd: %r",
                       socket.gethostname(), self.own_db, os.getcwd())
 
@@ -132,9 +123,11 @@ def load_config(self, _signal=None, _frame=None):
             self.cluster_monitor.config = copy.deepcopy(self.config)
 
         if self.config.get("syslog") and not self.syslog_handler:
-            self.syslog_handler = set_syslog_handler(self.config.get("syslog_address", "/dev/log"),
-                                                     self.config.get("syslog_facility", "local2"),
-                                                     logging.getLogger())
+            self.syslog_handler = logutil.set_syslog_handler(
+                address=self.config.get("syslog_address", "/dev/log"),
+                facility=self.config.get("syslog_facility", "local2"),
+                logger=logging.getLogger(),
+            )
         self.own_db = self.config.get("own_db")
 
         log_level_name = self.config.get("log_level", "DEBUG")
@@ -609,16 +602,7 @@ def main(args=None):
         print("pglookout: {!r} doesn't exist".format(arg.config))
         return 1
 
-    # Are we running under systemd?
-    if os.getenv("NOTIFY_SOCKET"):
-        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT_SYSLOG)
-        if not daemon:
-            print(
-                "WARNING: Running under systemd but python-systemd not available, "
-                "systemd won't see our notifications"
-            )
-    else:
-        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
+    logutil.configure_logging()
 
     pglookout = PgLookout(arg.config)
     pglookout.run()

test/conftest.py

@@ -0,0 +1,11 @@
+"""
+pglookout - test configuration
+
+Copyright (c) 2016 Ohmu Ltd
+See LICENSE for details
+"""
+
+from pglookout import logutil
+
+
+logutil.configure_logging()

test/test_cluster_monitor.py

@@ -5,7 +5,6 @@
 See LICENSE for details
 """
 
-from pglookout.common import LOG_FORMAT
 from pglookout.cluster_monitor import ClusterMonitor
 from psycopg2.extensions import POLL_OK
 from datetime import datetime, timedelta
@@ -17,9 +16,6 @@
     from Queue import Queue  # pylint: disable=import-error
     from mock import MagicMock, Mock, patch  # pylint: disable=import-error
 
-import logging
-logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
-
 
 def test_replication_lag():
     # pylint: disable=protected-access