DM7183: Port to Python 3

.gitignore

@@ -8,6 +8,8 @@ config.log
 *.pyc
 *_wrap.cc
 *Lib.py
+_build.*
+doc/xml
 doc/html
 doc/*.tag
 doc/*.inc

bin.src/Logger.py

@@ -1,44 +1,50 @@
 #!/usr/bin/env python
 
-# 
+#
 # LSST Data Management System
 # Copyright 2008, 2009, 2010 LSST Corporation.
-# 
+#
 # This product includes software developed by the
 # LSST Project (http://www.lsst.org/).
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
-# You should have received a copy of the LSST License Statement and 
-# the GNU General Public License along with this program.  If not, 
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
+from __future__ import print_function
+from builtins import object
 import argparse
 import os
 import sys
 import lsst.ctrl.events as events
+import lsst.log as log
+import lsst.utils
 
 from lsst.daf.base import PropertySet
 from lsst.ctrl.orca.db.DatabaseLogger import DatabaseLogger
 from lsst.daf.persistence import DbAuth
 from lsst.pex.policy import Policy
 
+
 class EventChecker(object):
+
     def __init__(self, broker, runid):
         self.runid = runid
 
         # create an event receiver
         self.receiver = events.EventReceiver(broker, events.LogEvent.LOGGING_TOPIC, "RUNID = '%s'" % runid)
-        
+
         # create an event transmitter
         self.transmitter = events.EventTransmitter(broker, "LoggerStatus")
 
@@ -68,27 +74,25 @@ def __init__(self, broker, host, port, runid, database):
         self.highwatermark = 10000
 
         self.database = database
-        
+
         #
         # get database authorization info
         #
         home = os.getenv("HOME")
         pol = Policy(home+"/.lsst/db-auth.paf")
-        
+
         dbAuth = DbAuth()
         dbAuth.setPolicy(pol)
-        
-        user = dbAuth.username(host,port)
-        password = dbAuth.password(host,port)
-        
+
+        user = dbAuth.username(host, port)
+        password = dbAuth.password(host, port)
+
         #
         # create the logger for the database and connect to it
         #
         self.dbLogger = DatabaseLogger(host, int(port))
-        
+
         self.dbLogger.connect(user, password, self.database)
-
-
 
     def execute(self):
         # set to the name of the file to write to
@@ -99,18 +103,18 @@ def execute(self):
 
         # initialize the message counter
         cnt = 0
-        
+
         #
         # main loop - attempt to get messages until either no messages are retrieved, or until
         # the highwater mark is reached.  If either of these happens, insert all current events
         # in the incoming message list into the database
         #
         while True:
             event = self.receiver.receiveEvent(50)
-            if event != None:
+            if event is not None:
                 propSet = event.getPropertySet()
                 log = propSet.get("LOGGER")
-                if log == None:
+                if log is None:
                     continue
                 if self.finalMessageReceived(propSet):
                     self.dbLogger.disconnect()
@@ -121,12 +125,12 @@ def execute(self):
                     self.dbLogger.insertRecords("%s.Logs" % self.database, msgs, tmpFilename)
                     cnt = 0
                     msgs = []
-            elif event == None:
+            elif event is None:
                 if len(msgs) > 0:
                     self.dbLogger.insertRecords("%s.Logs" % self.database, msgs, tmpFilename)
                     cnt = 0
                     msgs = []
-                    
+
 
 class Monitor(EventChecker):
 
@@ -137,17 +141,16 @@ def __init__(self, broker, runid):
     def execute(self):
         while True:
             event = self.receiver.receiveEvent(50)
-            if event == None:
+            if event is None:
                 continue
             propSet = event.getPropertySet()
             log = propSet.get("LOGGER")
-            if log == None:
+            if log is None:
                 continue
             if self.finalMessageReceived(propSet):
                 sys.exit(10)
 
 
-
 if __name__ == "__main__":
     basename = os.path.basename(sys.argv[0])
     parser = argparse.ArgumentParser(basename, "monitor logging information and completion of pipeline")
@@ -161,10 +164,14 @@ def execute(self):
 
     args = parser.parse_args()
 
+    package = lsst.utils.getPackageDir("ctrl_orca")
+    configPath = os.path.join(package, "etc", "log4j.properties")
+    log.configure(configPath)
+
     num_args = len([x for x in (args.host, args.port, args.database) if x is not None])
 
     if num_args == 1 or num_args == 2:
-        print "if used, --host --port and --database must be given together"
+        print("if used, --host --port and --database must be given together")
         sys.exit(10)
 
     if num_args == 3:
@@ -173,4 +180,3 @@ def execute(self):
     else:
         mon = Monitor(args.broker, args.runid)
         mon.execute()
-

bin.src/ProvenanceRecorder.py

@@ -1,32 +1,39 @@
 #!/usr/bin/env python
 
-# 
+#
 # LSST Data Management System
 # Copyright 2008, 2009, 2010 LSST Corporation.
-# 
+#
 # This product includes software developed by the
 # LSST Project (http://www.lsst.org/).
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
-# You should have received a copy of the LSST License Statement and 
-# the GNU General Public License along with this program.  If not, 
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
-import os, os.path, getopt, sets, sys
+import os
+import os.path
+import getopt
+import sets
+import sys
 import lsst.pex.policy as pol
+from builtins import object
 from lsst.ctrl.orca.NamedClassFactory import NamedClassFactory
 
+
 class ProvenanceRecorder(object):
+
     def __init__(self, provenance, repository):
         self.policySet = sets.Set()
         self.provenance = provenance
@@ -35,7 +42,7 @@ def __init__(self, provenance, repository):
 
     def record(self, name):
         # prov object init-ed here
-        filename = os.path.join(self.repository,name) 
+        filename = os.path.join(self.repository, name)
         provenance.recordPolicy(filename)
         policyNamesSet = self._extractPolicies(filename)
         for p in policyNamesSet:
@@ -66,26 +73,26 @@ def _extract(self, field, repos, policy, pipelinePolicySet):
         if policy.getValueType(field) == pol.Policy.FILE:
             filename = policy.getFile(field).getPath()
             filename = os.path.join(repos, filename)
-            if (filename in self.policySet) == False:
+            if not (filename in self.policySet):
                 self.policySet.add(filename)
-            if (filename in pipelinePolicySet) == False:
+            if not (filename in pipelinePolicySet):
                 pipelinePolicySet.add(filename)
             newPolicy = pol.Policy.createPolicy(filename, False)
             self._extractChildPolicies(repos, newPolicy, pipelinePolicySet)
 
 if __name__ == "__main__":
-    arguments = "--type=<type> --runid=<runid> --user=<user>, --dbrun=<dbrun> --dbglobal=<dbglobal> --filename=<file> --repos=<repos>"
-    options, xarguments = getopt.getopt(sys.argv[1:], "h", ["type=", "runid=", "user=", "dbrun=", "dbglobal=", "filename=", "repos="])
-
+    arguments = ("--type=<type> --runid=<runid> --user=<user>, --dbrun=<dbrun> --dbglobal=<dbglobal> ",
+                 "--filename=<file> --repos=<repos>")
+    options, xarguments = getopt.getopt(
+        sys.argv[1:], "h", ["type=", "runid=", "user=", "dbrun=", "dbglobal=", "filename=", "repos="])
 
     dict = {}
-    for a,o in options:
+    for a, o in options:
         dict[a.lstrip('-')] = o
 
     classFactory = NamedClassFactory()
     provClass = classFactory.createClass(dict["type"])
     provenance = provClass(dict)
 
-
-    recorder = ProvenanceRecorder(provenance,dict["repos"])
+    recorder = ProvenanceRecorder(provenance, dict["repos"])
     recorder.record(dict["filename"])

bin.src/filewaiter.py

@@ -1,72 +1,76 @@
 #!/usr/bin/env python
 
-# 
+#
 # LSST Data Management System
 # Copyright 2008, 2009, 2010 LSST Corporation.
-# 
+#
 # This product includes software developed by the
 # LSST Project (http://www.lsst.org/).
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
-# You should have received a copy of the LSST License Statement and 
-# the GNU General Public License along with this program.  If not, 
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
-import os, os.path, sys, time
+from __future__ import print_function
+import os
+import os.path
+import sys
+import time
 import optparse
 
 # filewaiter.py - wait for creation of files
 if __name__ == "__main__":
 
-
     usage = """usage: %prog [-f|-l] filenames.txt"""
 
     parser = optparse.OptionParser(usage)
     # TODO: handle "--dryrun"
-    parser.add_option("-f", "--first", action="store_const", const=1, dest="bFirst", help="wait for first file in list")
-    parser.add_option("-l", "--list", action="store_const", const=1, dest="bList", help="wait for all the files in the list")
+    parser.add_option("-f", "--first", action="store_const", const=1,
+                      dest="bFirst", help="wait for first file in list")
+    parser.add_option("-l", "--list", action="store_const", const=1,
+                      dest="bList", help="wait for all the files in the list")
 
     parser.opts = {}
     parser.args = []
 
     # parse and check command line arguments
     (parser.opts, parser.args) = parser.parse_args()
     if len(parser.args) < 1:
-        print usage
+        print(usage)
         raise RuntimeError("Missing args: pipelinePolicyFile runId")
 
     filename = parser.args[0]
 
-    bFirst = parser.opts.bFirst 
+    bFirst = parser.opts.bFirst
     bList = parser.opts.bList
-    
+
     f = open(filename, 'r')
 
     lines = f.readlines()
 
-    list = []
+    fileList = []
     for line in lines:
         list.append(line.split('\n')[0])
 
-
-    if bFirst == True:
-        item = list[0]
-        while os.path.exists(item) == False:
+    if bFirst:
+        item = fileList[0]
+        while not os.path.exists(item):
             time.sleep(1)
         sys.exit(0)
 
-    while len(list) > 0:
-        newlist = [ item for item in list if (os.path.exists(item) == False)]
-        list = newlist
+    while len(fileList) > 0:
+        newlist = [item for item2 in fileList if (os.path.exists(item) is False)]
+        fileList = newlist
         time.sleep(1)
     sys.exit(0)