DM-7245: Port to Python 3

examples/argumentParser.py

@@ -30,41 +30,43 @@
 ./argumentParser.py $OBS_TEST_DIR/data/input --id filter=g --show data
 ./argumentParser.py $OBS_TEST_DIR/data/input --id filter=g --config oneFloat=1.5 --show config
 """
+from __future__ import print_function
 import lsst.pex.config as pexConfig
 import lsst.pipe.base as pipeBase
 
+
 class ExampleConfig(pexConfig.Config):
     """Config for argument parser example
     """
     oneInt = pexConfig.Field(
-        dtype = int,
-        doc = "Example integer value",
-        default = 1,
+        dtype=int,
+        doc="Example integer value",
+        default=1,
     )
     oneFloat = pexConfig.Field(
-        dtype = float,
-        doc = "Example float value",
-        default = 3.14159265358979,
+        dtype=float,
+        doc="Example float value",
+        default=3.14159265358979,
     )
     oneStr = pexConfig.Field(
-        dtype = str,
-        doc = "Example string value",
-        default = "default value",
+        dtype=str,
+        doc="Example string value",
+        default="default value",
     )
     intList = pexConfig.ListField(
-        dtype = int,
-        doc = "example list of integers",
-        default = [-1, 0, 1],
+        dtype=int,
+        doc="example list of integers",
+        default=[-1, 0, 1],
     )
     floatList = pexConfig.ListField(
-        dtype = float,
-        doc = "example list of floats",
-        default = [-2.7, 0, 3.7e42],
+        dtype=float,
+        doc="example list of floats",
+        default=[-2.7, 0, 3.7e42],
     )
     strList = pexConfig.ListField(
-        dtype = str,
-        doc = "example list of strings",
-        default = ["a", "bb", "ccc"],
+        dtype=str,
+        doc="example list of strings",
+        default=["a", "bb", "ccc"],
     )
 
 parser = pipeBase.ArgumentParser(name="argumentParser")
@@ -73,4 +75,4 @@ class ExampleConfig(pexConfig.Config):
 parsedCmd = parser.parse_args(config=config)
 pcDict = parsedCmd.__dict__
 for key in sorted(pcDict):
-        print "parsedCmd.%s=%r" % (key, pcDict[key])
+    print("parsedCmd.%s=%r" % (key, pcDict[key]))

python/lsst/__init__.py

@@ -1,2 +1,3 @@
-import pkgutil, lsstimport
+import pkgutil
+import lsstimport
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/pipe/__init__.py

@@ -1,2 +1,3 @@
-import pkgutil, lsstimport
+import pkgutil
+import lsstimport
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/pipe/base/cmdLineTask.py

@@ -1,4 +1,6 @@
 from __future__ import absolute_import, division
+from builtins import str
+from builtins import object
 #
 # LSST Data Management System
 # Copyright 2008-2015 AURA/LSST.
@@ -36,6 +38,7 @@
 
 __all__ = ["CmdLineTask", "TaskRunner", "ButlerInitializedTaskRunner"]
 
+
 def _poolFunctionWrapper(function, arg):
     """Wrapper around function to catch exceptions that don't inherit from Exception
 
@@ -45,14 +48,15 @@ def _poolFunctionWrapper(function, arg):
     try:
         return function(arg)
     except Exception:
-        raise # No worries
+        raise  # No worries
     except:
         # Need to wrap the exception with something multiprocessing will recognise
         cls, exc, tb = sys.exc_info()
         log = getDefaultLog()
         log.warn("Unhandled exception %s (%s):\n%s" % (cls.__name__, exc, traceback.format_exc()))
         raise Exception("Unhandled exception: %s (%s)" % (cls.__name__, exc))
 
+
 def _runPool(pool, timeout, function, iterable):
     """Wrapper around pool.map_async, to handle timeout
 
@@ -64,6 +68,7 @@ def _runPool(pool, timeout, function, iterable):
     """
     return pool.map_async(functools.partial(_poolFunctionWrapper, function), iterable).get(timeout)
 
+
 @contextlib.contextmanager
 def profile(filename, log=None):
     """!Context manager for profiling with cProfile
@@ -128,7 +133,8 @@ class TaskRunner(object):
     [1] http://bugs.python.org/issue8296
     [2] http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool)
     """
-    TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
+    TIMEOUT = 9999  # Default timeout (sec) for multiprocessing
+
     def __init__(self, TaskClass, parsedCmd, doReturnResults=False):
         """!Construct a TaskRunner
 
@@ -199,10 +205,10 @@ def run(self, parsedCmd):
             if len(targetList) > 0:
                 with profile(profileName, log):
                     # Run the task using self.__call__
-                    resultList = mapFunc(self, targetList)
+                    resultList = list(mapFunc(self, targetList))
             else:
                 log.warn("Not running the task because there is no data to process; "
-                    "you may preview data using \"--show data\"")
+                         "you may preview data using \"--show data\"")
 
         if pool is not None:
             pool.close()
@@ -297,7 +303,7 @@ def precall(self, parsedCmd):
         else:
             try:
                 self._precallImpl(task, parsedCmd)
-            except Exception, e:
+            except Exception as e:
                 task.log.fatal("Failed in task initialization: %s" % e)
                 if not isinstance(e, TaskError):
                     traceback.print_exc(file=sys.stderr)
@@ -332,13 +338,13 @@ def __call__(self, args):
         elif isinstance(dataRef, (list, tuple)):
             self.log.addLabel(str([ref.dataId for ref in dataRef if hasattr(ref, "dataId")]))
         task = self.makeTask(args=args)
-        result = None # in case the task fails
+        result = None  # in case the task fails
         if self.doRaise:
             result = task.run(dataRef, **kwargs)
         else:
             try:
                 result = task.run(dataRef, **kwargs)
-            except Exception, e:
+            except Exception as e:
                 # don't use a try block as we need to preserve the original exception
                 if hasattr(dataRef, "dataId"):
                     task.log.fatal("Failed on dataId=%s: %s" % (dataRef.dataId, e))
@@ -354,15 +360,17 @@ def __call__(self, args):
 
         if self.doReturnResults:
             return Struct(
-                dataRef = dataRef,
-                metadata = task.metadata,
-                result = result,
+                dataRef=dataRef,
+                metadata=task.metadata,
+                result=result,
             )
 
+
 class ButlerInitializedTaskRunner(TaskRunner):
     """!A TaskRunner for CmdLineTasks that require a 'butler' keyword argument to be passed to
     their constructor.
     """
+
     def makeTask(self, parsedCmd=None, args=None):
         """!A variant of the base version that passes a butler argument to the task's constructor
 
@@ -381,6 +389,7 @@ def makeTask(self, parsedCmd=None, args=None):
             raise RuntimeError("parsedCmd or args must be specified")
         return self.TaskClass(config=self.config, log=self.log, butler=butler)
 
+
 class CmdLineTask(Task):
     """!Base class for command-line tasks: tasks that may be executed from the command line
 
@@ -462,10 +471,10 @@ def parseAndRun(cls, args=None, config=None, log=None, doReturnResults=False):
         taskRunner = cls.RunnerClass(TaskClass=cls, parsedCmd=parsedCmd, doReturnResults=doReturnResults)
         resultList = taskRunner.run(parsedCmd)
         return Struct(
-            argumentParser = argumentParser,
-            parsedCmd = parsedCmd,
-            taskRunner = taskRunner,
-            resultList = resultList,
+            argumentParser=argumentParser,
+            parsedCmd=parsedCmd,
+            taskRunner=taskRunner,
+            resultList=resultList,
         )
 
     @classmethod
@@ -484,7 +493,7 @@ def _makeArgumentParser(cls):
         """
         parser = ArgumentParser(name=cls._DefaultName)
         parser.add_id_argument(name="--id", datasetType="raw",
-            help="data IDs, e.g. --id visit=12345 ccd=1,2^0,3")
+                               help="data IDs, e.g. --id visit=12345 ccd=1,2^0,3")
         return parser
 
     def writeConfig(self, butler, clobber=False, doBackup=True):
@@ -512,8 +521,8 @@ def writeConfig(self, butler, clobber=False, doBackup=True):
             output = lambda msg: self.log.fatal("Comparing configuration: " + msg)
             if not self.config.compare(oldConfig, shortcut=False, output=output):
                 raise TaskError(
-                    ("Config does not match existing task config %r on disk; tasks configurations " + \
-                    "must be consistent within the same output repo (override with --clobber-config)") % \
+                    ("Config does not match existing task config %r on disk; tasks configurations " +
+                     "must be consistent within the same output repo (override with --clobber-config)") %
                     (configName,))
         else:
             butler.put(self.config, configName)
@@ -532,16 +541,16 @@ def writeSchemas(self, butler, clobber=False, doBackup=True):
         then some schemas may have been saved successfully and others may not, and there is no easy way to
         tell which is which.
         """
-        for dataset, catalog in self.getAllSchemaCatalogs().iteritems():
+        for dataset, catalog in self.getAllSchemaCatalogs().items():
             schemaDataset = dataset + "_schema"
             if clobber:
                 butler.put(catalog, schemaDataset, doBackup=doBackup)
             elif butler.datasetExists(schemaDataset):
                 oldSchema = butler.get(schemaDataset, immediate=True).getSchema()
                 if not oldSchema.compare(catalog.getSchema(), afwTable.Schema.IDENTICAL):
                     raise TaskError(
-                        ("New schema does not match schema %r on disk; schemas must be " + \
-                        " consistent within the same output repo (override with --clobber-config)") % \
+                        ("New schema does not match schema %r on disk; schemas must be " +
+                         " consistent within the same output repo (override with --clobber-config)") %
                         (dataset,))
             else:
                 butler.put(catalog, schemaDataset)
@@ -556,7 +565,7 @@ def writeMetadata(self, dataRef):
             metadataName = self._getMetadataName()
             if metadataName is not None:
                 dataRef.put(self.getFullMetadata(), metadataName)
-        except Exception, e:
+        except Exception as e:
             self.log.warn("Could not persist metadata for dataId=%s: %s" % (dataRef.dataId, e,))
 
     def writePackageVersions(self, butler, clobber=False, doBackup=True, dataset="packages"):

python/lsst/pipe/base/struct.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import, division
+from builtins import object
 #
 # LSST Data Management System
 # Copyright 2008, 2009, 2010, 2011 LSST Corporation.
@@ -22,6 +23,7 @@
 #
 __all__ = ["Struct"]
 
+
 class Struct(object):
     """!A struct to which you can add any fields
 
@@ -41,6 +43,7 @@ class Struct(object):
     all the safety advantages of Struct. In addition, named tuples are clumsy to define and Structs
     are much more mutable (e.g. one can trivially combine Structs and add additional fields).
     """
+
     def __init__(self, **keyArgs):
         """!Create a Struct with the specified field names and values
 
@@ -55,7 +58,7 @@ def __init__(self, **keyArgs):
         @param[in] **keyArgs    keyword arguments specifying name=value pairs
         """
         object.__init__(self)
-        for name, val in keyArgs.iteritems():
+        for name, val in keyArgs.items():
             self.__safeAdd(name, val)
 
     def __safeAdd(self, name, val):
@@ -106,5 +109,5 @@ def __len__(self):
         return len(self.__dict__)
 
     def __repr__(self):
-        itemList = ["%s=%r" % (name, val) for name, val in self.getDict().iteritems()]
+        itemList = ["%s=%r" % (name, val) for name, val in self.getDict().items()]
         return "%s(%s)" % (self.__class__.__name__, "; ".join(itemList))

python/lsst/pipe/base/task.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import, division
+from builtins import object
 #
 # LSST Data Management System
 # Copyright 2008-2016 AURA/LSST.
@@ -30,6 +31,7 @@
 
 __all__ = ["Task", "TaskError"]
 
+
 class TaskError(Exception):
     """!Use to report errors for which a traceback is not useful.
 
@@ -39,6 +41,7 @@ class TaskError(Exception):
     """
     pass
 
+
 class Task(object):
     """!Base class for data processing tasks
 
@@ -76,6 +79,7 @@ class Task(object):
     Tasks intended to be run from the command line should be subclasses of \ref cmdLineTask.CmdLineTask
     "CmdLineTask", not Task.
     """
+
     def __init__(self, config=None, name=None, parentTask=None, log=None):
         """!Create a Task
 
@@ -127,7 +131,7 @@ def __init__(self, config=None, name=None, parentTask=None, log=None):
 
     def emptyMetadata(self):
         """!Empty (clear) the metadata for this Task and all sub-Tasks."""
-        for subtask in self._taskDict.itervalues():
+        for subtask in self._taskDict.values():
             subtask.metadata = dafBase.PropertyList()
 
     def getSchemaCatalogs(self):
@@ -162,7 +166,7 @@ def getAllSchemaCatalogs(self):
         Task.getSchemaCatalogs, not this method.
         """
         schemaDict = self.getSchemaCatalogs()
-        for subtask in self._taskDict.itervalues():
+        for subtask in self._taskDict.values():
             schemaDict.update(subtask.getSchemaCatalogs())
         return schemaDict
 
@@ -180,7 +184,7 @@ def getFullMetadata(self):
             for the top-level task and all subtasks, sub-subtasks, etc.
         """
         fullMetadata = dafBase.PropertySet()
-        for fullName, task in self.getTaskDict().iteritems():
+        for fullName, task in self.getTaskDict().items():
             fullMetadata.set(fullName.replace(".", ":"), task.metadata)
         return fullMetadata
 
@@ -228,7 +232,7 @@ def makeSubtask(self, name, **keyArgs):
         setattr(self, name, subtask)
 
     @contextlib.contextmanager
-    def timer(self, name, logLevel = pexLog.Log.DEBUG):
+    def timer(self, name, logLevel=pexLog.Log.DEBUG):
         """!Context manager to log performance data for an arbitrary block of code
 
         @param[in] name         name of code being timed;
@@ -243,11 +247,11 @@ def timer(self, name, logLevel = pexLog.Log.DEBUG):
 
         See timer.logInfo for the information logged
         """
-        logInfo(obj = self, prefix = name + "Start", logLevel = logLevel)
+        logInfo(obj=self, prefix=name + "Start", logLevel=logLevel)
         try:
             yield
         finally:
-            logInfo(obj = self, prefix = name + "End",   logLevel = logLevel)
+            logInfo(obj=self, prefix=name + "End", logLevel=logLevel)
 
     @classmethod
     def makeField(cls, doc):