Skip to content

Commit

Permalink
Log strings now lazily evaluated
Browse files Browse the repository at this point in the history
  • Loading branch information
@pritchardn
pritchardn committed May 25, 2022
1 parent 318cf2b commit 14931a9
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 38 deletions.
9 changes: 4 additions & 5 deletions daliuge-common/dlg/clients.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
from . import constants
from .restutils import RestClient


logger = logging.getLogger(__name__)
compress = os.environ.get("DALIUGE_COMPRESSED_JSON", True)

Expand Down Expand Up @@ -137,7 +136,7 @@ def sessions(self):
"Successfully read %d sessions from %s:%s",
len(sessions),
self.host,
self.port,
self.port
)
return sessions

Expand Down Expand Up @@ -223,7 +222,7 @@ class NodeManagerClient(BaseDROPManagerClient):
"""

def __init__(
self, host="localhost", port=constants.NODE_DEFAULT_REST_PORT, timeout=10
self, host="localhost", port=constants.NODE_DEFAULT_REST_PORT, timeout=10
):
super(NodeManagerClient, self).__init__(host=host, port=port, timeout=timeout)

Expand Down Expand Up @@ -259,7 +258,7 @@ class DataIslandManagerClient(CompositeManagerClient):
"""

def __init__(
self, host="localhost", port=constants.ISLAND_DEFAULT_REST_PORT, timeout=10
self, host="localhost", port=constants.ISLAND_DEFAULT_REST_PORT, timeout=10
):
super(DataIslandManagerClient, self).__init__(
host=host, port=port, timeout=timeout
Expand All @@ -272,7 +271,7 @@ class MasterManagerClient(CompositeManagerClient):
"""

def __init__(
self, host="localhost", port=constants.MASTER_DEFAULT_REST_PORT, timeout=10
self, host="localhost", port=constants.MASTER_DEFAULT_REST_PORT, timeout=10
):
super(MasterManagerClient, self).__init__(host=host, port=port, timeout=timeout)

Expand Down
6 changes: 3 additions & 3 deletions daliuge-engine/dlg/manager/composite_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ def addGraphSpec(self, sessionId, graphSpec):
# belong to the same host, so we can submit that graph into the individual
# DMs. For this we need to make sure that our graph has a the correct
# attribute set
logger.info(f"Separating graph using partition attribute {self._partitionAttr}")
logger.info("Separating graph using partition attribute %s", self._partitionAttr)
perPartition = collections.defaultdict(list)
if "rmode" in graphSpec[-1]:
init_pg_repro_data(graphSpec)
Expand Down Expand Up @@ -386,7 +386,7 @@ def addGraphSpec(self, sessionId, graphSpec):
# At each partition the relationships between DROPs should be local at the
# moment of submitting the graph; thus we record the inter-partition
# relationships separately and remove them from the original graph spec
logger.info(f"Graph splitted into {perPartition.keys()}")
logger.info("Graph split into %r", perPartition.keys())
inter_partition_rels = []
for dropSpecs in perPartition.values():
inter_partition_rels += graph_loader.removeUnmetRelationships(dropSpecs)
Expand Down Expand Up @@ -452,7 +452,7 @@ def deploySession(self, sessionId, completedDrops=[]):
)
logger.info("Delivered node subscription list to node managers")
logger.debug(
"Number of subscriptions: %s" % len(self._drop_rels[sessionId].items())
"Number of subscriptions: %s", len(self._drop_rels[sessionId].items())
)

logger.info("Deploying Session %s in all hosts", sessionId)
Expand Down
36 changes: 18 additions & 18 deletions daliuge-engine/dlg/manager/proc_daemon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,15 +180,15 @@ def startNM(self):
tool = get_tool()
args = ["--host", "0.0.0.0"]
args += self._verbosity_as_cmdline()
logger.info("Starting Node Drop Manager with args: %s" % (" ".join(args)))
logger.info("Starting Node Drop Manager with args: %s", (" ".join(args)))
self._nm_proc = tool.start_process("nm", args)
logger.info("Started Node Drop Manager with PID %d" % (self._nm_proc.pid))
logger.info("Started Node Drop Manager with PID %d", self._nm_proc.pid)

# Registering the new NodeManager via zeroconf so it gets discovered
# by the Master Manager
if self._zeroconf:
addrs = utils.get_local_ip_addr()
logger.info("Registering this NM with zeroconf: %s" % addrs)
logger.info("Registering this NM with zeroconf: %s", addrs)
self._nm_info = utils.register_service(
self._zeroconf,
"NodeManager",
Expand All @@ -205,11 +205,11 @@ def startDIM(self, nodes):
if nodes:
args += ["--nodes", ",".join(nodes)]
logger.info(
"Starting Data Island Drop Manager with args: %s" % (" ".join(args))
"Starting Data Island Drop Manager with args: %s", (" ".join(args))
)
self._dim_proc = tool.start_process("dim", args)
logger.info(
"Started Data Island Drop Manager with PID %d" % (self._dim_proc.pid)
"Started Data Island Drop Manager with PID %d", self._dim_proc.pid
)

# Registering the new DIM via zeroconf so it gets discovered
Expand All @@ -230,9 +230,9 @@ def startMM(self):
tool = get_tool()
args = ["--host", "0.0.0.0"]
args += self._verbosity_as_cmdline()
logger.info("Starting Master Drop Manager with args: %s" % (" ".join(args)))
logger.info("Starting Master Drop Manager with args: %s", (" ".join(args)))
self._mm_proc = tool.start_process("mm", args)
logger.info("Started Master Drop Manager with PID %d" % (self._mm_proc.pid))
logger.info("Started Master Drop Manager with PID %d", self._mm_proc.pid)

# Also subscribe to zeroconf events coming from NodeManagers and feed
# the Master Manager with the new hosts we find
Expand All @@ -246,14 +246,14 @@ def nm_callback(zeroconf, service_type, name, state_change):
port = info.port
nm_assigner.add_nm(name, server, port)
logger.info(
"Found a new Node Manager on %s:%d, will add it to the MM"
% (server, port)
"Found a new Node Manager on %s:%d, will add it to the MM",
(server, port)
)
elif state_change is zc.ServiceStateChange.Removed:
server, port = nm_assigner.NMs[name]
logger.info(
"Node Manager on %s:%d disappeared, removing it from the MM"
% (server, port)
"Node Manager on %s:%d disappeared, removing it from the MM",
(server, port)
)

# Don't bother to remove it if we're shutting down. This way
Expand All @@ -269,14 +269,14 @@ def dim_callback(zeroconf, service_type, name, state_change):
port = info.port
nm_assigner.add_dim(name, server, port)
logger.info(
"Found a new Data Island Manager on %s:%d, will add it to the MM"
% (server, port)
"Found a new Data Island Manager on %s:%d, will add it to the MM",
(server, port)
)
elif state_change is zc.ServiceStateChange.Removed:
server, port = nm_assigner.DIMs[name]
logger.info(
"Data Island Manager on %s:%d disappeared, removing it from the MM"
% (server, port)
"Data Island Manager on %s:%d disappeared, removing it from the MM",
(server, port)
)

# Don't bother to remove it if we're shutting down. This way
Expand Down Expand Up @@ -319,7 +319,7 @@ def _rest_stop_manager(self, proc, stop_method):
def _rest_get_manager_info(self, proc):
if proc:
bottle.response.content_type = "application/json"
logger.info("Sending response: %s" % json.dumps({"pid": proc.pid}))
logger.info("Sending response: %s", json.dumps({"pid": proc.pid}))
return json.dumps({"pid": proc.pid})
else:
return json.dumps({"pid": None})
Expand All @@ -337,7 +337,7 @@ def rest_getMgrs(self):
if mgrs["node"]:
mgrs["node"] = self._nm_proc.pid

logger.info("Sending response: %s" % json.dumps(mgrs))
logger.info("Sending response: %s", json.dumps(mgrs))
return json.dumps(mgrs)

def rest_startNM(self):
Expand Down Expand Up @@ -437,7 +437,7 @@ def handle_signal(signalNo, stack_frame):
global terminating
if terminating:
return
logger.info("Received signal %d, will stop the daemon now" % (signalNo,))
logger.info("Received signal %d, will stop the daemon now", signalNo)
terminating = True
daemon.stop(10)

Expand Down
24 changes: 12 additions & 12 deletions daliuge-engine/dlg/manager/rest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ def _stop_manager(self):
self.dm.shutdown()
self.stop()
logger.info(
"Thanks for using our %s, come back again :-)"
% (self.dm.__class__.__name__)
"Thanks for using our %s, come back again :-)",
self.dm.__class__.__name__
)

@daliuge_aware
Expand Down Expand Up @@ -400,7 +400,7 @@ def linkGraphParts(self, sessionId):

@daliuge_aware
def add_node_subscriptions(self, sessionId):
logger.debug(f"NM REST call: add_subscriptions {bottle.request.json}")
logger.debug("NM REST call: add_subscriptions %s", bottle.request.json)
if bottle.request.content_type != "application/json":
bottle.response.status = 415
return
Expand Down Expand Up @@ -471,12 +471,12 @@ def getAllCMNodes(self):

@daliuge_aware
def addCMNode(self, node):
logger.debug("Adding node %s" % node)
logger.debug("Adding node %s", node)
self.dm.add_node(node)

@daliuge_aware
def removeCMNode(self, node):
logger.debug("Removing node %s" % node)
logger.debug("Removing node %s", node)
self.dm.remove_node(node)

@daliuge_aware
Expand Down Expand Up @@ -606,12 +606,12 @@ def getDIMs(self):

@daliuge_aware
def addDIM(self, dim):
logger.debug("Adding DIM %s" % dim)
logger.debug("Adding DIM %s", dim)
self.dm.addDmHost(dim)

@daliuge_aware
def removeDIM(self, dim):
logger.debug("Removing dim %s" % dim)
logger.debug("Removing dim %s", dim)
self.dm.removeDmHost(dim)

@daliuge_aware
Expand All @@ -621,21 +621,21 @@ def getNMs(self):
@daliuge_aware
def startNM(self, host):
port = constants.DAEMON_DEFAULT_REST_PORT
logger.debug("Sending NM start request to %s:%s" % (host, port))
logger.debug("Sending NM start request to %s:%s", (host, port))
with RestClient(host=host, port=port, timeout=10) as c:
return json.loads(c._POST("/managers/node/start").read())

@daliuge_aware
def stopNM(self, host):
port = constants.DAEMON_DEFAULT_REST_PORT
logger.debug("Sending NM stop request to %s:%s" % (host, port))
logger.debug("Sending NM stop request to %s:%s", (host, port))
with RestClient(host=host, port=port, timeout=10) as c:
return json.loads(c._POST("/managers/node/stop").read())

@daliuge_aware
def addNM(self, host, node):
port = constants.ISLAND_DEFAULT_REST_PORT
logger.debug("Adding NM %s to DIM %s" % (node, host))
logger.debug("Adding NM %s to DIM %s", (node, host))
with RestClient(host=host, port=port, timeout=10, url_prefix="/api") as c:
return json.loads(
c._POST(
Expand All @@ -646,14 +646,14 @@ def addNM(self, host, node):
@daliuge_aware
def removeNM(self, host, node):
port = constants.ISLAND_DEFAULT_REST_PORT
logger.debug("Removing NM %s from DIM %s" % (node, host))
logger.debug("Removing NM %s from DIM %s", (node, host))
with RestClient(host=host, port=port, timeout=10, url_prefix="/api") as c:
return json.loads(c._DELETE("/nodes/%s" % (node,)).read())

@daliuge_aware
def getNMInfo(self, host):
port = constants.DAEMON_DEFAULT_REST_PORT
logger.debug("Sending request %s:%s/managers/node" % (host, port))
logger.debug("Sending request %s:%s/managers/node", (host, port))
with RestClient(host=host, port=port, timeout=10) as c:
return json.loads(c._GET("/managers/node").read())

Expand Down

0 comments on commit 14931a9

Please sign in to comment.