From b691ef134765292efb4313a714f891d5bf4e57a2 Mon Sep 17 00:00:00 2001 From: Zhi-Qiang You Date: Thu, 7 Feb 2019 11:34:19 -0500 Subject: [PATCH 1/7] Add syslog handler in logging.py * Support syslog handler in Logging and Performance Logging --- reframe/core/logging.py | 6 ++++++ unittests/test_logging.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index b36843c78f..20e71d575c 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -197,6 +197,10 @@ def _create_filelog_handler(handler_config): return MultiFileHandler(filename_patt, mode='a+' if append else 'w+') +def _create_syslog_handler(handler_config): + return logging.handlers.SysLogHandler(address = '/dev/log') + + def _create_stream_handler(handler_config): stream = handler_config.get('name', 'stdout') if stream == 'stdout': @@ -258,6 +262,8 @@ def _extract_handlers(handlers_list): hdlr = _create_file_handler(handler_config) elif handler_type == 'filelog': hdlr = _create_filelog_handler(handler_config) + elif handler_type == 'syslog': + hdlr = _create_syslog_handler(handler_config) elif handler_type == 'stream': hdlr = _create_stream_handler(handler_config) elif handler_type == 'graylog': diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 513856caa8..505ed609a1 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -296,11 +296,12 @@ def test_multiple_handlers(self): 'level': 'INFO', 'handlers': [ {'type': 'stream', 'name': 'stderr'}, - {'type': 'file', 'name': self.logfile} + {'type': 'file', 'name': self.logfile}, + {'type': 'syslog'} ], } rlog.configure_logging(self.logging_config) - self.assertEqual(len(rlog.getlogger().logger.handlers), 2) + self.assertEqual(len(rlog.getlogger().logger.handlers), 3) def test_file_handler_timestamp(self): self.logging_config['handlers'][0]['timestamp'] = '%F' From f98a8a99780e9ee8824b53f35204c17cfcace9ae Mon Sep 17 00:00:00 2001 From: Zhi-Qiang You Date: Fri, 15 Feb 2019 15:10:59 -0500 Subject: [PATCH 2/7] Improve syslog handler in logging * Support facility and socket type attributes * The attribute address is now required --- reframe/core/logging.py | 18 +++++++++++++++++- unittests/test_logging.py | 22 +++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 20e71d575c..854df4acd9 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -198,7 +198,23 @@ def _create_filelog_handler(handler_config): def _create_syslog_handler(handler_config): - return logging.handlers.SysLogHandler(address = '/dev/log') + from socket import SOCK_DGRAM, SOCK_STREAM + address = handler_config.get('address', None) + facility = handler_config.get('facility', logging.handlers.SysLogHandler.LOG_USER) + socket = handler_config.get('socket', 'udp') + if address is None: + raise ConfigError('syslog handler: no address specified') + + if socket == 'udp': + socktype = SOCK_DGRAM + elif socket == 'tcp': + socktype = SOCK_STREAM + else: + raise ConfigError('syslog handler: unsupported socket type %s' % socket) + + return logging.handlers.SysLogHandler(address=address, + facility=facility, + socktype=socktype) def _create_stream_handler(handler_config): diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 505ed609a1..0cb3643a9d 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -297,7 +297,7 @@ def test_multiple_handlers(self): 'handlers': [ {'type': 'stream', 'name': 'stderr'}, {'type': 'file', 'name': self.logfile}, - {'type': 'syslog'} + {'type': 'syslog', 'address': '/dev/log'} ], } rlog.configure_logging(self.logging_config) @@ -331,6 +331,26 @@ def test_stream_handler_unknown_stream(self): self.assertRaises(ConfigError, rlog.configure_logging, self.logging_config) + def test_syslog_handler_syntax_no_address(self): + self.logging_config = { + 'level': 'INFO', + 'handlers': [ + {'type': 'syslog'} + ], + } + self.assertRaises(ConfigError, rlog.configure_logging, + self.logging_config) + + def test_syslog_handler_unknown_socket(self): + self.logging_config = { + 'level': 'INFO', + 'handlers': [ + {'type': 'syslog', 'socket': 'foo'}, + ], + } + self.assertRaises(ConfigError, rlog.configure_logging, + self.logging_config) + def test_global_noconfig(self): # This is to test the case when no configuration is set, but since the # order the unit tests are invoked is arbitrary, we emulate the From b727fc5c84fc078563a6fb34facd5867c89d39e9 Mon Sep 17 00:00:00 2001 From: Zhi-Qiang You Date: Thu, 28 Feb 2019 11:11:25 -0500 Subject: [PATCH 3/7] Improve configuration handling in syslog handler * Chagne the attribute `socket` to `socktype` * Make the attribute `facility` string-based --- reframe/core/logging.py | 25 +++++++++++++++---------- unittests/test_logging.py | 18 +++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 854df4acd9..63eca40f49 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -8,6 +8,7 @@ import shutil import sys import warnings +import socket from datetime import datetime import reframe @@ -198,23 +199,27 @@ def _create_filelog_handler(handler_config): def _create_syslog_handler(handler_config): - from socket import SOCK_DGRAM, SOCK_STREAM address = handler_config.get('address', None) - facility = handler_config.get('facility', logging.handlers.SysLogHandler.LOG_USER) - socket = handler_config.get('socket', 'udp') if address is None: raise ConfigError('syslog handler: no address specified') - if socket == 'udp': - socktype = SOCK_DGRAM - elif socket == 'tcp': - socktype = SOCK_STREAM + facility = handler_config.get('facility', 'user') + try: + facility_type = logging.handlers.SysLogHandler.facility_names[facility] + except KeyError: + raise ConfigError('syslog handler: unknown facility: %s' % facility) from None + + socktype = handler_config.get('socktype', 'udp') + if socktype == 'udp': + socket_type = socket.SOCK_DGRAM + elif socktype == 'tcp': + socket_type = socket.SOCK_STREAM else: - raise ConfigError('syslog handler: unsupported socket type %s' % socket) + raise ConfigError('syslog handler: unsupported socket type: %s' % socktype) return logging.handlers.SysLogHandler(address=address, - facility=facility, - socktype=socktype) + facility=facility_type, + socktype=socket_type) def _create_stream_handler(handler_config): diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 0cb3643a9d..2ded61e711 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -334,19 +334,23 @@ def test_stream_handler_unknown_stream(self): def test_syslog_handler_syntax_no_address(self): self.logging_config = { 'level': 'INFO', - 'handlers': [ - {'type': 'syslog'} - ], + 'handlers': [{'type': 'syslog'}] } self.assertRaises(ConfigError, rlog.configure_logging, self.logging_config) - def test_syslog_handler_unknown_socket(self): + def test_syslog_handler_unknown_facility(self): self.logging_config = { 'level': 'INFO', - 'handlers': [ - {'type': 'syslog', 'socket': 'foo'}, - ], + 'handlers': [{'type': 'syslog', 'facility': 'foo'}] + } + self.assertRaises(ConfigError, rlog.configure_logging, + self.logging_config) + + def test_syslog_handler_unknown_socktype(self): + self.logging_config = { + 'level': 'INFO', + 'handlers': [{'type': 'syslog', 'socktype': 'foo'}] } self.assertRaises(ConfigError, rlog.configure_logging, self.logging_config) From 1dcff86bd1b9656da1fa3be0a9ca0a0cd2827d9e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Mar 2019 10:38:52 +0100 Subject: [PATCH 4/7] Fix coding style --- reframe/core/logging.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 63eca40f49..d2511c268e 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -207,7 +207,8 @@ def _create_syslog_handler(handler_config): try: facility_type = logging.handlers.SysLogHandler.facility_names[facility] except KeyError: - raise ConfigError('syslog handler: unknown facility: %s' % facility) from None + raise ConfigError('syslog handler: ' + 'unknown facility: %s' % facility) from None socktype = handler_config.get('socktype', 'udp') if socktype == 'udp': @@ -215,11 +216,9 @@ def _create_syslog_handler(handler_config): elif socktype == 'tcp': socket_type = socket.SOCK_STREAM else: - raise ConfigError('syslog handler: unsupported socket type: %s' % socktype) + raise ConfigError('syslog handler: unknown socket type: %s' % socktype) - return logging.handlers.SysLogHandler(address=address, - facility=facility_type, - socktype=socket_type) + return logging.handlers.SysLogHandler(address, facility_type, socket_type) def _create_stream_handler(handler_config): From eabf49f3d967ad53e83fef0ce3e1ac12c38e8973 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Mar 2019 10:43:14 +0100 Subject: [PATCH 5/7] Fix trailing whitespace error --- reframe/core/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index d2511c268e..03d4305b64 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -199,7 +199,7 @@ def _create_filelog_handler(handler_config): def _create_syslog_handler(handler_config): - address = handler_config.get('address', None) + address = handler_config.get('address', None) if address is None: raise ConfigError('syslog handler: no address specified') From b4114072f1169e5a1accde33c65b96a1035a5934 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Mar 2019 11:26:48 +0100 Subject: [PATCH 6/7] Document syslog handler --- docs/running.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 0bc6f92252..3c9882f4c0 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -650,10 +650,14 @@ Common Log Handler Attributes All handlers accept the following set of attributes (keys) in their configuration: * ``type``: (required) the type of the handler. - There are two types of handlers used for standard logging in ReFrame + There are several types of handlers used for logging in ReFrame. + Some of them are only relevant for performance logging: 1. ``file``: a handler that writes log records in file. 2. ``stream``: a handler that writes log records in a file stream. + 3. ``syslog``: a handler that sends log records to Unix syslog. + 4. ``filelog``: a handler for writing performance logs (relevant only for `performance logging <#performance-logging>`__). + 5. ``graylog``: a handler for sending performance logs to a Graylog server (relevant only for `performance logging <#performance-logging>`__). * ``level``: (default: ``DEBUG``) The lowest level of log records that this handler can process. @@ -709,6 +713,25 @@ In addition to the common log handler attributes, file log handlers accept the f Available values: ``stdout`` for standard output and ``stderr`` for standard error. +Syslog log handler +"""""""""""""""""" + +In addition to the common log handler attributes, file log handlers accept the following: + +* ``socktype``: The type of socket where the handler will send log records to. There are two socket types: + + 1. ``udp``: (default) This opens a UDP datagram socket. + 2. ``tcp``: This opens a TCP stream socket. + +* ``facility``: (default: ``user``) The Syslog facility to send records to. + The list of supported facilities can be found `here `__. +* ``address``: (required) The address where the handler will connect to. + This can either be of the form ``:`` or simply a path that refers to a Unix domain socket. + + +.. note:: + .. versionadded:: 2.17 + Performance Logging ^^^^^^^^^^^^^^^^^^^ From 2f0f0c9ae40122831269d20859b9c46e79b3de7e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Mar 2019 11:58:48 +0100 Subject: [PATCH 7/7] Handler properly the `host:port` syntax in syslog Also - Add Syslog handler unit test --- reframe/core/logging.py | 8 ++++++++ unittests/test_logging.py | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 03d4305b64..171ad8714f 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -203,6 +203,14 @@ def _create_syslog_handler(handler_config): if address is None: raise ConfigError('syslog handler: no address specified') + # Check if address is in `host:port` format + try: + host, port = address.split(':', maxsplit=1) + except ValueError: + pass + else: + address = (host, port) + facility = handler_config.get('facility', 'user') try: facility_type = logging.handlers.SysLogHandler.facility_names[facility] diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 2ded61e711..11ff3f470f 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -331,7 +331,23 @@ def test_stream_handler_unknown_stream(self): self.assertRaises(ConfigError, rlog.configure_logging, self.logging_config) - def test_syslog_handler_syntax_no_address(self): + def test_syslog_handler(self): + import platform + + if platform.system() == 'Linux': + addr = '/dev/log' + elif platform.system() == 'Darwin': + addr = '/dev/run/syslog' + else: + self.skipTest() + + self.logging_config = { + 'level': 'INFO', + 'handlers': [{'type': 'syslog', 'address': addr}] + } + rlog.getlogger().info('foo') + + def test_syslog_handler_no_address(self): self.logging_config = { 'level': 'INFO', 'handlers': [{'type': 'syslog'}]