Merge master to tickets/2303; resolve conflicts in favour of tickets/…

…2303.

SConstruct

@@ -1,7 +1,3 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
-
-scripts.BasicSConstruct(
-    packageName="pipe_base",
-    versionString=r"$HeadURL",
-)
+scripts.BasicSConstruct("pipe_base")

doc/main.dox

@@ -10,18 +10,6 @@ The basic design is as follows:
   This is done by calling the parseAndRun method of the task, which parses command-line arguments
   and executes the parsed command. parseAndRun also persists the configuration used for the task,
   and metadata generated by the task (including timing data).
-
-- When a pipeline task is run, two camera-specific configuration overrides are loaded, if found;
-  first one for the telescope then one for the camera. These files may replace set configuration
-  parameters or even replace subtasks with camera-specific variants (e.g. for instrument
-  signature removal). The configuration override files are, in order:
-  - <em>obs_path</em>/config/<em>task_name</em>.py
-  - <em>obs_path</em>/config/<em>camera_name</em>/<em>task_name</em>.py
-  .
-  where the path elements are:
-  - <em>task_name</em>: the name of the pipeline task, e.g. "processCcd"
-  - <em>camera_name</em>: the name of the camera, e.g. "lsstSim"
-  - <em>obs_path</em>: the path to the obs package for the camera, e.g. "obs_lsstSim"
 
 - Pipeline tasks rarely, if ever, perform multitasking (though they may well use GPUs).
   For multitasking, run multiple instances of the task.
@@ -35,15 +23,66 @@ The basic design is as follows:
   this allows named access to returned data, which provides safer evolution
   than relying on the order of returned values.
 
-- Tasks are typically put into the pipe_tasks package, in the python directory.
+- Some useful general tasks are found in the pipe_tasks package, in the python directory.
   Tasks meant to be run from the command line (subclasses of CmdLineTask) also have
   a short script in the bin directory.
+  Package-specific tasks belong in the relevant package, e.g. ip_isr has an IsrTask.
 
 pipe_base is written purely in Python. Contents include:
-@li lsst.pipe.base.CmdLineTask: base class for pipeline tasks
-@li lsst.pipe.base.Task: base class for subtasks.
-@li lsst.pipe.base.Struct: object returned by the run method of a task.
-@li lsst.pipe.base.ArgumentParser: command line parser for pipeline tasks.
-@li lsst.pipe.base.timeMethod: decorator to log performance information about a Task method.
+- lsst.pipe.base.CmdLineTask: base class for pipeline tasks that can be run from the command line.
+- lsst.pipe.base.Task: base class for subtasks that are not meant to be run from the command line.
+- lsst.pipe.base.Struct: object returned by the run method of a task.
+- lsst.pipe.base.ArgumentParser: command line parser for pipeline tasks.
+- lsst.pipe.base.timeMethod: decorator to log performance information about a Task method.
+
+\section pipeBase_argumentParser Argument Parser Hints
+
+To shorten input, calib and output paths see \ref pipeBase_environmentVariables.
+
+For long or repetetive command lines you may wish to put some arguments into one or more files.
+The format is the same as the command line itself, except that if you specify the data on multiple lines
+do not use \\ to indicate line continuation. Specify such files using @@path.
+
+\subsection pipeBase_argumentParser_configOverride Overriding Config
+
+The argument parser automatically loads specific config override files based on
+the camera name and its obs package. See \ref pipeBase_configOverrideFiles.
+
+In addition, you can specify config override files on the command line using --configfile
+and override some (but not all) config parameters by specifying values on the command line using --config.
+
+Examples:
+-  --config str1=foo str2="fancier string" int1=5 intList=2,4,-87 float1=1.53 floatList=3.14,-5.6e7
+-  --configfile %config.py # where %config.py contains root.strList = "first string", "second string"
+
+There are important limitations on using --config; use a config override file to get around these issues:
+- You cannot specify values for items in registries
+- You cannot specify values for lists of strings
+- You cannot specify a subset of list; you must specify all values at once
+
+\section pipeBase_environmentVariables Environment Variables
+
+The command parser uses environment variables PIPE_INPUT_ROOT, PIPE_CALIB_ROOT, and PIPE_OUTPUT_ROOT,
+if available, to make it easier to specify the input, calib and output data repositories.
+Each environment variable is used as a root directory for relative paths and ignored for absolute paths.
+The default value for each of these environment variables is the current working directory. For example:
+- mytask cameraname foo # use $PIPE_INPUT_ROOT/foo as the input repository (or ./foo if $PIPE_INPUT_ROOT is undefined)
+- mytask cameraname . # use $PIPE_INPUT_ROOT (= $PIPE_INPUT_ROOT/.) as the input repository
+- mytask cameraname /a/b # use /a/b as the input repository ($PIPE_INPUT_ROOT is ignored for absolute paths)
+
+\section pipeBase_configOverrideFiles Automatically Loaded Config Override Files
+
+When a pipeline task is run, two camera-specific configuration overrides are loaded, if found;
+first one for the obs package then one for the camera. (There are two because in some cases
+an obs package may contain data for multiple cameras). These files may override configuration
+parameters or even replace subtasks with camera-specific variants (e.g. for instrument
+signature removal). The configuration override files are, in order:
+- <em>obs_path</em>/config/<em>task_name</em>.py
+- <em>obs_path</em>/config/<em>camera_name</em>/<em>task_name</em>.py
+
+where the path elements are:
+- <em>task_name</em>: the name of the pipeline task, e.g. "processCcd"
+- <em>camera_name</em>: the name of the camera, e.g. "lsstSim"
+- <em>obs_path</em>: the path to the obs package for the camera, e.g. "obs_lsstSim"
 */
 }}}

examples/argumentParser.py

@@ -52,6 +52,16 @@ class ExampleConfig(pexConfig.Config):
         doc = "example list of integers",
         default = [-1, 0, 1],
     )
+    floatList = pexConfig.ListField(
+        dtype = float,
+        doc = "example list of floats",
+        default = [-2.7, 0, 3.7e42],
+    )
+    strList = pexConfig.ListField(
+        dtype = str,
+        doc = "example list of strings",
+        default = ["a", "bb", "ccc"],
+    )
 
 parser = pipeBase.ArgumentParser(name="argumentParser")
 config = ExampleConfig()

python/lsst/pipe/base/argumentParser.py

@@ -111,13 +111,16 @@ def __init__(self,
             help="display final configuration and/or data IDs to stdout? If exit, then don't process data.")
         self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use")
 
-    def parse_args(self, config, args=None, log=None):
+    def parse_args(self, config, args=None, log=None, override=None):
         """Parse arguments for a pipeline task
 
-        @params config: config for the task being run
-        @params args: argument list; if None use sys.argv[1:]
-        @params log: log (instance pex_logging Log); if None use the default log
-        
+        @param config: config for the task being run
+        @param args: argument list; if None use sys.argv[1:]
+        @param log: log (instance pex_logging Log); if None use the default log
+        @param override: a config override callable, to be applied after camera-specific overrides
+            files but before any command-line config overrides.  It should take the root config
+            object as its only argument.
+
         @return namespace: a struct containing many useful fields including:
         - config: the supplied config with all overrides applied, validated and frozen
         - butler: a butler for the data
@@ -149,6 +152,8 @@ def parse_args(self, config, args=None, log=None):
         self.handleCamera(namespace)
 
         self._applyInitialOverrides(namespace)
+        if override is not None:
+            override(namespace.config)
 
         namespace.dataIdList = []
 
@@ -159,26 +164,7 @@ def parse_args(self, config, args=None, log=None):
         if "config" in namespace.show:
             namespace.config.saveToStream(sys.stdout, "config")
 
-        def fixPath(defName, path):
-            """Apply environment variable as default root, if present, and abspath
-            
-            @param defName: name of environment variable containing default root path;
-                if the environment variable does not exist then the path is relative
-                to the current working directory
-            @param path: path relative to default root path
-            @return abspath: path that has been expanded, or None if the environment variable does not exist
-                and path is None
-            """
-            defRoot = os.environ.get(defName)
-            if defRoot is None:
-                if path is None:
-                    return None
-                return os.path.abspath(path)
-            return os.path.abspath(os.path.join(defRoot, path or ""))
-
-        namespace.input = fixPath(DEFAULT_INPUT_NAME, namespace.input)
-        namespace.calib = fixPath(DEFAULT_CALIB_NAME, namespace.calib)
-        namespace.output = fixPath(DEFAULT_OUTPUT_NAME, namespace.output)
+        self._fixPaths(namespace)
 
         if not os.path.isdir(namespace.input):
             self.error("Error: input=%r not found" % (namespace.input,))
@@ -252,6 +238,36 @@ def fixPath(defName, path):
 
         return namespace
 
+
+    def _fixPaths(self, namespace):
+        """
+        Adjust paths using environment variables or custom options.
+        
+        This has been refactored into a separate method to allow subclasses to
+        override before the mapper is created.
+        """
+        def fixPath(defName, path):
+            """Apply environment variable as default root, if present, and abspath
+            
+            @param defName: name of environment variable containing default root path;
+                if the environment variable does not exist then the path is relative
+                to the current working directory
+            @param path: path relative to default root path
+            @return abspath: path that has been expanded, or None if the environment variable does not exist
+                and path is None
+            """
+            defRoot = os.environ.get(defName)
+            if defRoot is None:
+                if path is None:
+                    return None
+                return os.path.abspath(path)
+            return os.path.abspath(os.path.join(defRoot, path or ""))
+
+        namespace.input = fixPath(DEFAULT_INPUT_NAME, namespace.input)
+        namespace.calib = fixPath(DEFAULT_CALIB_NAME, namespace.calib)
+        namespace.output = fixPath(DEFAULT_OUTPUT_NAME, namespace.output)
+
+
     def _makeDataRefList(self, namespace):
         """Make namespace.dataRefList from namespace.dataIdList
         
@@ -329,6 +345,9 @@ def _getMapper(self, namespace):
             elif lowCamera == "suprimecam":
                 obsPkg = "obs_subaru"
                 from lsst.obs.suprimecam.suprimecamMapper import SuprimecamMapper as Mapper
+            elif lowCamera == "suprimecam-mit":
+                obsPkg = "obs_subaru"
+                from lsst.obs.suprimecam.suprimecamMapper import SuprimecamMapperMit as Mapper
             elif lowCamera == "sst":
                 obsPkg = "obs_sst"
                 from lsst.obs.sst.sstMapper import SstMapper as Mapper

python/lsst/pipe/base/cmdLineTask.py

@@ -34,14 +34,27 @@ class CmdLineTask(Task):
     Subclasses must specify the following attribute:
     _DefaultName: default name used for this task
     """
+
+    @classmethod
+    def applyOverrides(cls, config):
+        """A hook to allow a task to change the values of its config *after* the camera-specific
+        overrides are loaded but before any command-line overrides are applied.  This is invoked
+        only when parseAndRun is used; other ways of constructing a config will not apply these
+        overrides.
+
+        This is necessary in some cases because the camera-specific overrides may retarget subtasks,
+        wiping out changes made in ConfigClass.setDefaults.  See LSST ticket #2282 for more discussion.
+        """
+        pass
+
     @classmethod
     def parseAndRun(cls, args=None, config=None, log=None):
         """Parse an argument list and run the command
 
         @param args: list of command-line arguments; if None use sys.argv
         @param config: config for task (instance of pex_config Config); if None use cls.ConfigClass()
         @param log: log (instance of pex_logging Log); if None use the default log
-        
+
         @return a Struct containing:
         - argumentParser: the argument parser
         - parsedCmd: the parsed command returned by argumentParser.parse_args
@@ -51,7 +64,7 @@ def parseAndRun(cls, args=None, config=None, log=None):
         argumentParser = cls._makeArgumentParser()
         if config is None:
             config = cls.ConfigClass()
-        parsedCmd = argumentParser.parse_args(config=config, args=args, log=log)
+        parsedCmd = argumentParser.parse_args(config=config, args=args, log=log, override=cls.applyOverrides)
         task = cls(name=cls._DefaultName, config=parsedCmd.config, log=parsedCmd.log)
 
         useMP = useMultiProcessing(parsedCmd)

python/lsst/pipe/base/task.py

@@ -221,8 +221,9 @@ def display(self, name, exposure=None, sources=(), matches=None,
 first ctype is used for the first SourceSet); the matches are interpreted as an extra pair of SourceSets
 but the sizes are doubled
         """
-        if not self._display or not self._display.has_key(name) or self._display < 0 or \
-               self._display in (False, None) or self._display[name] in (False, None):
+        if not self._display or not self._display.has_key(name) or \
+                self._display < 0 or self._display in (False, None) or \
+                self._display[name] < 0 or self._display[name] in (False, None):
             return
 
         if isinstance(self._display, int):
@@ -249,6 +250,7 @@ def display(self, name, exposure=None, sources=(), matches=None,
             sources = [sources]
 
         with ds9.Buffering():
+            i = 0
             for i, ss in enumerate(sources):
                 ctype = ctypes[i%len(ctypes)]
                 ptype = ptypes[i%len(ptypes)]

python/lsst/pipe/base/timer.py

@@ -111,8 +111,9 @@ def run(self, ...): # or any other instance method you want to time
     Writes various measures of time and possibly memory usage to the task's metadata;
     all items are prefixed with the function name.
     
-    @warning This decorator only works with instance methods of Task (or any class with a metadata attribute
-      that supports add(name, value)).
+    @warning This decorator only works with instance methods of Task, or any class with these attributes:
+        * metadata: a daf_data PropertyList (or other object with add(name, value) method)
+        * log: a pex_logging Log
     """
     @functools.wraps(func)
     def wrapper(self, *args, **keyArgs):

ups/pipe_base.build

@@ -1,5 +1,2 @@
-svn co @SVNROOT@/DMS/@PRODUCT@/tags/@VERSION@ @PRODUCT@-@VERSION@ &&
-cd @PRODUCT@-@VERSION@ &&
-setup -r . &&
-scons opt=3 &&
-scons opt=3 install
+@LSST BUILD@ &&
+build_lsst @PRODUCT@ @VERSION@ @REPOVERSION@