Skip to content

Commit

Permalink
Convert ctrl_pool code to lsst.log
Browse files Browse the repository at this point in the history
Use the default logger from lsst.log to log messages for parallel.py
Remove pickling code and copyreg for pex_logging.Log
Make pickleLog use the isst.log.Log type
Instead of writing to "jobs.log", write to job name + '.log'
  • Loading branch information
pgee2000 committed Dec 15, 2016
1 parent f8db4f1 commit a6f84a4
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 19 deletions.
29 changes: 13 additions & 16 deletions python/lsst/ctrl/pool/log.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,33 @@
import copyreg

import lsst.log as lsstLog
import lsst.pex.logging as pexLog
from lsst.utils import getPackageDir



def pickleLog(log):
"""Pickle a log
Drop the log on the floor; we'll get a new default.
Assumes that we're always just using the default log,
and not modifying it.
Assumes that we're always just using the lsst.log default.
"""
return pexLog.getDefaultLog, tuple()


def pickleLsstLog(log):
"""Pickle a lsst.log.Log"""
return lsstLog.Log, tuple()

copyreg.pickle(pexLog.Log, pickleLog)
copyreg.pickle(pexLog.ScreenLog, pickleLog)
copyreg.pickle(lsstLog.Log, pickleLsstLog)
copyreg.pickle(lsstLog.Log, pickleLog)


def jobLog(job):
"""Add a job-specific log destination"""
if job is None or job == "None":
return
machine = os.uname()[1].split(".")[0]
pexLog.getDefaultLog().addDestination(job + ".%s.%d" % (machine, os.getpid()))
packageDir = getPackageDir("ctrl_pool")
lsstLog.configure(os.path.join(packageDir, "config/log4cxx.properties"))
# Load the properties file and try to modify the File destination
fin = open(os.path.join(packageDir, "config/log4cxx.properties"), 'r')
lines = fin.readlines()
fin.close()
for i in range(len(lines)):
index = lines[i].find("File=")
if index > 0:
lines[i] = lines[i][0: index + 5] + job + ".log\n"
# this would produce a log for every machine, pid combination
#lines[i] = lines[i][0: index + 5] + job + ".%s.%d" % (machine, os.getpid()) + ".log\n"
lsstLog.configure_prop(''.join(lines))
lsstLog.MDC("PID", os.getpid())
7 changes: 4 additions & 3 deletions python/lsst/ctrl/pool/parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import contextlib
from lsst.pipe.base import CmdLineTask, TaskRunner
from .pool import startPool, Pool, NODE, abortOnError, setBatchType
from . import log # register pickle functions for pex_logging
from . import log # register pickle functions for log

__all__ = ["Batch", "PbsBatch", "SlurmBatch", "SmpBatch", "BATCH_TYPES", "BatchArgumentParser",
"BatchCmdLineTask", "BatchPoolTask", ]
Expand Down Expand Up @@ -63,8 +63,9 @@ def processStats():

def printProcessStats():
"""Print the process statistics to the log"""
from lsst.pex.logging import getDefaultLog
getDefaultLog().info("Process stats for %s: %s" % (NODE, processStats()))
from lsst.log import Log
log = Log.getDefaultLogger()
log.info("Process stats for %s: %s" % (NODE, processStats()))


class Batch(object):
Expand Down

0 comments on commit a6f84a4

Please sign in to comment.