From e579c75769872c557b32d0d15d9730c01d435ad5 Mon Sep 17 00:00:00 2001 From: davepallot Date: Wed, 7 Jul 2021 14:06:10 +0800 Subject: [PATCH 1/2] LIU-65: Generate individual log files per session. --- daliuge-engine/dlg/manager/cmdline.py | 6 +++-- daliuge-engine/dlg/manager/node_manager.py | 7 ++++-- daliuge-engine/dlg/manager/session.py | 28 +++++++++++++++++++++- daliuge-engine/test/manager/test_dim.py | 1 + daliuge-engine/test/test_session.py | 1 + 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/daliuge-engine/dlg/manager/cmdline.py b/daliuge-engine/dlg/manager/cmdline.py index 87d95b870..2f6053e71 100644 --- a/daliuge-engine/dlg/manager/cmdline.py +++ b/daliuge-engine/dlg/manager/cmdline.py @@ -264,7 +264,8 @@ def dlgNM(parser, args): 'host': options.host, 'error_listener': options.errorListener, 'event_listeners': list(filter(None, options.event_listeners.split(":"))), - 'max_threads': options.max_threads} + 'max_threads': options.max_threads, + 'logdir': options.logdir} options.dmAcronym = 'NM' options.restType = NMRestServer @@ -290,7 +291,8 @@ def dlgCompositeManager(parser, args, dmType, acronym, dmPort, dmRestServer): # Add DIM-specific options options.dmType = dmType options.dmArgs = ([s for s in options.nodes.split(',') if s],) - options.dmKwargs = {'pkeyPath': options.pkeyPath, 'dmCheckTimeout': options.dmCheckTimeout} + options.dmKwargs = {'pkeyPath': options.pkeyPath, + 'dmCheckTimeout': options.dmCheckTimeout} options.dmAcronym = acronym options.restType = dmRestServer diff --git a/daliuge-engine/dlg/manager/node_manager.py b/daliuge-engine/dlg/manager/node_manager.py index 81a6a26d8..51818cbd1 100644 --- a/daliuge-engine/dlg/manager/node_manager.py +++ b/daliuge-engine/dlg/manager/node_manager.py @@ -109,10 +109,12 @@ def __init__(self, dlgPath=None, error_listener=None, event_listeners=[], - max_threads = 0): + max_threads=0, + logdir=utils.getDlgLogsDir()): self._dlm = DataLifecycleManager() if useDLM else None self._sessions = {} + self.logdir = logdir # dlgPath contains code added by the user with possible # DROP applications @@ -472,6 +474,7 @@ class RpcMixIn(rpc.RPCClient, rpc.RPCServer): pass class NodeManager(EventMixIn, RpcMixIn, NodeManagerBase): def __init__(self, useDLM=True, dlgPath=None, error_listener=None, event_listeners=[], max_threads=0, + logdir=utils.getDlgLogsDir(), host=None, rpc_port=constants.NODE_DEFAULT_RPC_PORT, events_port=constants.NODE_DEFAULT_EVENTS_PORT): # We "just know" that our RpcMixIn will have a create_context static @@ -480,7 +483,7 @@ def __init__(self, useDLM=True, dlgPath=None, error_listener=None, event_listene host = host or '127.0.0.1' EventMixIn.__init__(self, host, events_port) RpcMixIn.__init__(self, host, rpc_port) - NodeManagerBase.__init__(self, useDLM, dlgPath, error_listener, event_listeners, max_threads) + NodeManagerBase.__init__(self, useDLM, dlgPath, error_listener, event_listeners, max_threads, logdir) def shutdown(self): super(NodeManager, self).shutdown() diff --git a/daliuge-engine/dlg/manager/session.py b/daliuge-engine/dlg/manager/session.py index 99904f1b2..8514b1f69 100644 --- a/daliuge-engine/dlg/manager/session.py +++ b/daliuge-engine/dlg/manager/session.py @@ -27,6 +27,7 @@ import inspect import logging import threading +import time from . import constants from .. import droputils @@ -97,6 +98,30 @@ def __init__(self, sessionId, nm=None): self._nm = nm self._dropsubs = {} + class SessionFilter(logging.Filter): + def __init__(self, sessionId): + self.sessionId = sessionId + + def filter(self, record): + if hasattr(record, 'session_id'): + return record.session_id == self.sessionId + return False + + fmt = '%(asctime)-15s [%(levelname)5.5s] [%(threadName)15.15s] ' + fmt += '[%(session_id)10.10s] [%(drop_uid)10.10s] ' + fmt += '%(name)s#%(funcName)s:%(lineno)s %(message)s' + fmt = logging.Formatter(fmt) + fmt.converter = time.gmtime + + logdir = utils.getDlgLogsDir() + if self._nm is not None: + logdir = self._nm.logdir + logfile = '%s/dlg_%s' % (logdir, self.sessionId,) + self.file_handler = logging.FileHandler(logfile) + self.file_handler.setFormatter(fmt) + self.file_handler.addFilter(SessionFilter(self.sessionId)) + logging.root.addHandler(self.file_handler) + @property def sessionId(self): return self._sessionId @@ -389,7 +414,8 @@ def getGraph(self): return dict(self._graph) def destroy(self): - pass + self.file_handler.close() + logging.root.removeHandler(self.file_handler) __del__ = destroy diff --git a/daliuge-engine/test/manager/test_dim.py b/daliuge-engine/test/manager/test_dim.py index a17d3142a..1f512e7a8 100644 --- a/daliuge-engine/test/manager/test_dim.py +++ b/daliuge-engine/test/manager/test_dim.py @@ -27,6 +27,7 @@ import pkg_resources +from dlg import runtime from dlg import droputils from dlg import utils from dlg.common import tool, Categories diff --git a/daliuge-engine/test/test_session.py b/daliuge-engine/test/test_session.py index de54363d7..01c892d89 100644 --- a/daliuge-engine/test/test_session.py +++ b/daliuge-engine/test/test_session.py @@ -21,6 +21,7 @@ # import unittest +from dlg import runtime from dlg.ddap_protocol import DROPLinkType, DROPStates, AppDROPStates from dlg.manager.session import Session, SessionStates from dlg.exceptions import InvalidGraphException From 2f6fe917825213c59a64847b5ededd3c5d319c7e Mon Sep 17 00:00:00 2001 From: davepallot Date: Wed, 7 Jul 2021 14:49:54 +0800 Subject: [PATCH 2/2] LIU-65: Make recommended changes. --- daliuge-engine/dlg/manager/session.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/daliuge-engine/dlg/manager/session.py b/daliuge-engine/dlg/manager/session.py index 8514b1f69..a5730b0b0 100644 --- a/daliuge-engine/dlg/manager/session.py +++ b/daliuge-engine/dlg/manager/session.py @@ -103,12 +103,10 @@ def __init__(self, sessionId): self.sessionId = sessionId def filter(self, record): - if hasattr(record, 'session_id'): - return record.session_id == self.sessionId - return False + return getattr(record, 'session_id', None) == self.sessionId fmt = '%(asctime)-15s [%(levelname)5.5s] [%(threadName)15.15s] ' - fmt += '[%(session_id)10.10s] [%(drop_uid)10.10s] ' + fmt += '[%(drop_uid)10.10s] ' fmt += '%(name)s#%(funcName)s:%(lineno)s %(message)s' fmt = logging.Formatter(fmt) fmt.converter = time.gmtime