Merge branch 'tickets/DM-26613' into master

python/lsst/pipe/tasks/cli/cmd/__init__.py

@@ -0,0 +1,25 @@
+# This file is part of pipe_tasks.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ["make_discrete_skymap",]
+
+
+from .commands import make_discrete_skymap

python/lsst/pipe/tasks/cli/cmd/commands.py

@@ -0,0 +1,51 @@
+# This file is part of obs_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import click
+
+from lsst.daf.butler.cli.opt import (repo_argument, config_file_option, options_file_option)
+from lsst.daf.butler.cli.utils import (cli_handle_exception, split_commas, typeStrAcceptsMultiple)
+from lsst.obs.base.cli.opt import instrument_option
+from ... import script
+
+
+@click.command(short_help="Define a discrete skymap from calibrated exposures.")
+@repo_argument(required=True)
+@config_file_option(help="Path to a pex_config override to be included after the Instrument config overrides"
+                         "are applied.")
+@options_file_option()
+@click.option("--collections",
+              help=("The collections to be searched (in order) when reading datasets. "
+                    "This includes the seed skymap if --append is specified."),
+              multiple=True,
+              callback=split_commas,
+              metavar=typeStrAcceptsMultiple,
+              required=True)
+@click.option("--out-collection",
+              help=("The collection to write the skymap to."),
+              type=str, default="skymaps", show_default=True)
+@click.option("--skymap-id",
+              help=("The identifier of the skymap to write."),
+              type=str, default="discrete", show_default=True)
+@instrument_option(required=True)
+def make_discrete_skymap(*args, **kwargs):
+    """Define a discrete skymap from calibrated exposures in the butler registry."""
+    cli_handle_exception(script.makeDiscreteSkyMap, *args, **kwargs)

python/lsst/pipe/tasks/cli/resources.yaml

@@ -0,0 +1,4 @@
+cmd:
+  import: lsst.pipe.tasks.cli.cmd
+  commands:
+    - make-discrete-skymap

python/lsst/pipe/tasks/makeDiscreteSkyMap.py

@@ -25,7 +25,6 @@
 
 import lsst.geom as geom
 import lsst.afw.image as afwImage
-import lsst.afw.geom as afwGeom
 import lsst.pex.config as pexConfig
 import lsst.pipe.base as pipeBase
 from lsst.skymap import DiscreteSkyMap, BaseSkyMap
@@ -127,25 +126,61 @@ class MakeDiscreteSkyMapTask(pipeBase.CmdLineTask):
     RunnerClass = MakeDiscreteSkyMapRunner
 
     def __init__(self, **kwargs):
-        pipeBase.CmdLineTask.__init__(self, **kwargs)
+        super().__init__(**kwargs)
 
-    @pipeBase.timeMethod
     def runDataRef(self, butler, dataRefList):
-        """!Make a skymap from the bounds of the given set of calexps.
-
-        @param[in]  butler        data butler used to save the SkyMap
-        @param[in]  dataRefList   dataRefs of calexps used to determine the size and pointing of the SkyMap
-        @return     a pipeBase Struct containing:
-                    - skyMap: the constructed SkyMap
+        """Make a skymap from the bounds of the given set of calexps using the butler.
+
+        Parameters
+        ----------
+        butler : `lsst.daf.persistence.Butler`
+           Gen2 data butler used to save the SkyMap
+        dataRefList : iterable
+           A list of Gen2 data refs of calexps used to determin the size and pointing of the SkyMap
+        Returns
+        -------
+        struct : `lsst.pipe.base.Struct`
+           The returned struct has one attribute, ``skyMap``, which holds the returned SkyMap
         """
-        self.log.info("Extracting bounding boxes of %d images" % len(dataRefList))
-        points = []
+        wcs_md_tuple_list = []
+        oldSkyMap = None
+        datasetName = self.config.coaddName + "Coadd_skyMap"
         for dataRef in dataRefList:
             if not dataRef.datasetExists("calexp"):
                 self.log.warn("CalExp for %s does not exist: ignoring" % (dataRef.dataId,))
                 continue
-            md = dataRef.get("calexp_md", immediate=True)
-            wcs = afwGeom.makeSkyWcs(md)
+            wcs_md_tuple_list.append((dataRef.get("calexp_wcs", immediate=True),
+                                      dataRef.get("calexp_md", immediate=True)))
+        if self.config.doAppend and butler.datasetExists(datasetName):
+            oldSkyMap = butler.get(datasetName, immediate=True)
+            if not isinstance(oldSkyMap.config, DiscreteSkyMap.ConfigClass):
+                raise TypeError("Cannot append to existing non-discrete skymap")
+            compareLog = []
+            if not self.config.skyMap.compare(oldSkyMap.config, output=compareLog.append):
+                raise ValueError("Cannot append to existing skymap - configurations differ:", *compareLog)
+        result = self.run(wcs_md_tuple_list, oldSkyMap)
+        if self.config.doWrite:
+            butler.put(result.skyMap, datasetName)
+        return result
+
+    @pipeBase.timeMethod
+    def run(self, wcs_md_tuple_list, oldSkyMap=None):
+        """Make a SkyMap from the bounds of the given set of calexp metadata.
+
+        Parameters
+        ----------
+        wcs_md_tuple_list : iterable
+           A list of tuples with each element expected to be a (Wcs, PropertySet) pair
+        oldSkyMap : `lsst.skymap.DiscreteSkyMap`, option
+           The SkyMap to extend if appending
+        Returns
+        -------
+        struct : `lsst.pipe.base.Struct
+           The returned struct has one attribute, ``skyMap``, which holds the returned SkyMap
+        """
+        self.log.info("Extracting bounding boxes of %d images" % len(wcs_md_tuple_list))
+        points = []
+        for wcs, md in wcs_md_tuple_list:
             # nb: don't need to worry about xy0 because Exposure saves Wcs with CRPIX shifted by (-x0, -y0).
             boxI = afwImage.bboxFromMetadata(md)
             boxD = geom.Box2D(boxI)
@@ -161,16 +196,8 @@ def runDataRef(self, butler, dataRefList):
             )
         circle = polygon.getBoundingCircle()
 
-        datasetName = self.config.coaddName + "Coadd_skyMap"
-
         skyMapConfig = DiscreteSkyMap.ConfigClass()
-        if self.config.doAppend and butler.datasetExists(datasetName):
-            oldSkyMap = butler.get(datasetName, immediate=True)
-            if not isinstance(oldSkyMap.config, DiscreteSkyMap.ConfigClass):
-                raise TypeError("Cannot append to existing non-discrete skymap")
-            compareLog = []
-            if not self.config.skyMap.compare(oldSkyMap.config, output=compareLog.append):
-                raise ValueError("Cannot append to existing skymap - configurations differ:", *compareLog)
+        if oldSkyMap:
             skyMapConfig.raList.extend(oldSkyMap.config.raList)
             skyMapConfig.decList.extend(oldSkyMap.config.decList)
             skyMapConfig.radiusList.extend(oldSkyMap.config.radiusList)
@@ -196,8 +223,6 @@ def runDataRef(self, butler, dataRefList):
             self.log.info("tract %s has corners %s (RA, Dec deg) and %s x %s patches" %
                           (tractInfo.getId(), ", ".join(posStrList),
                            tractInfo.getNumPatches()[0], tractInfo.getNumPatches()[1]))
-        if self.config.doWrite:
-            butler.put(skyMap, datasetName)
         return pipeBase.Struct(
             skyMap=skyMap
         )

python/lsst/pipe/tasks/script/__init__.py

@@ -0,0 +1,22 @@
+# This file is part of pipe_tasks.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from .makeDiscreteSkyMap import makeDiscreteSkyMap

python/lsst/pipe/tasks/script/makeDiscreteSkyMap.py

@@ -0,0 +1,86 @@
+# This file is part of obs_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from lsst.daf.butler import Butler, DatasetType
+from lsst.pipe.tasks.makeDiscreteSkyMap import MakeDiscreteSkyMapTask, MakeDiscreteSkyMapConfig
+from lsst.obs.base.utils import getInstrument
+
+
+def makeDiscreteSkyMap(repo, config_file, collections, instrument,
+                       out_collection='skymaps', skymap_id='discrete'):
+    """Implements the command line interface `butler make-discrete-skymap` subcommand,
+    should only be called by command line tools and unit test code that tests
+    this function.
+
+    Constructs a skymap from calibrated exposure in the butler repository
+
+    Parameters
+    ----------
+    repo : `str`
+        URI to the location to read the repo.
+    config_file : `str` or `None`
+        Path to a config file that contains overrides to the skymap config.
+    collections : `list` [`str`]
+        An expression specifying the collections to be searched (in order) when
+        reading datasets, and optionally dataset type restrictions on them.
+        At least one collection must be specified.  This is the collection
+        with the calibrated exposures.
+    instrument : `str`
+        The name or fully-qualified class name of an instrument.
+    out_collection : `str`, optional
+        The name of the collection to save the skymap to.  Default is 'skymaps'.
+    skymap_id : `str`, optional
+        The identifier of the skymap to save.  Default is 'discrete'.
+    """
+    butler = Butler(repo, collections=collections, writeable=True, run=out_collection)
+    instr = getInstrument(instrument, butler.registry)
+    config = MakeDiscreteSkyMapConfig()
+    instr.applyConfigOverrides(MakeDiscreteSkyMapTask._DefaultName, config)
+
+    if config_file is not None:
+        config.load(config_file)
+    skymap_name = config.coaddName + "Coadd_skyMap"
+    oldSkyMap = None
+    if config.doAppend:
+        if out_collection in collections:
+            raise ValueError(f"Cannot overwrite dataset.  If appending, specify an output "
+                             f"collection not in the input collections.")
+        dataId = {'skymap': skymap_id}
+        try:
+            oldSkyMap = butler.get(skymap_name, collections=collections, dataId=dataId)
+        except LookupError as e:
+            msg = (f"Could not find seed skymap for {skymap_name} with dataId {dataId} "
+                   f"in collections {collections} but doAppend is {config.doAppend}.  Aborting...")
+            raise LookupError(msg, *e.args[1:])
+
+    datasets = butler.registry.queryDatasets('calexp', collections=collections)
+    wcs_md_tuple_list = [(butler.getDirect('calexp.metadata', ref), butler.getDirect('calexp.wcs', ref))
+                         for ref in datasets]
+    task = MakeDiscreteSkyMapTask(config=config)
+    result = task.run(wcs_md_tuple_list, oldSkyMap)
+    skymap_dataset_type = DatasetType(skymap_name, dimensions=["skymap", ],
+                                      universe=butler.registry.dimensions,
+                                      storageClass="SkyMap")
+    butler.registry.registerDatasetType(skymap_dataset_type)
+    if config.doAppend:
+        # By definition if appending the dataset has already been registered
+        result.skyMap.register(skymap_id, butler.registry)
+    butler.put(result.skyMap, skymap_name, dataId={'skymap': skymap_id})

tests/SConscript

@@ -1,5 +1,10 @@
 # -*- python -*-
-from lsst.sconsUtils import scripts
+import os
+
+from lsst.sconsUtils import env, scripts
 
 scripts.BasicSConscript.tests(pySingles=['nopytest_test_coadds.py'],
                               pyList=[])
+
+if "DAF_BUTLER_PLUGINS" in os.environ:
+    env["ENV"]["DAF_BUTLER_PLUGINS"] = os.environ["DAF_BUTLER_PLUGINS"]

tests/test_cliCmdMakeDiscreteSkymap.py

@@ -0,0 +1,85 @@
+# This file is part of pipe_tasks.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Unit tests for daf_butler CLI make-discrete-skymap command.
+"""
+
+import unittest
+
+from lsst.daf.butler.tests import CliCmdTestBase
+from lsst.pipe.tasks.cli.cmd import make_discrete_skymap
+
+
+class DefineMakeDiscreteSkymap(CliCmdTestBase, unittest.TestCase):
+
+    @staticmethod
+    def defaultExpected():
+        return dict(config_file=None,
+                    collections=())
+
+    @staticmethod
+    def command():
+        return make_discrete_skymap
+
+    def test_repoBasic(self):
+        """Test the most basic required arguments."""
+        self.run_test(["make-discrete-skymap",
+                       "--collections", "foo/bar,baz",
+                       "--instrument", "a.b.c", "here"],
+                      self.makeExpected(repo="here",
+                                        collections=("foo/bar", "baz"),
+                                        out_collection="skymaps",
+                                        skymap_id="discrete",
+                                        instrument="a.b.c"))
+
+    def test_all(self):
+        """Test all the arguments."""
+        self.run_test(["make-discrete-skymap",
+                       "--instrument", "a.b.c",
+                       "--collections", "foo/bar,baz",
+                       "--config-file", "/path/to/config",
+                       "--collections", "boz",
+                       "--out-collection", "biz",
+                       "--skymap-id", "wiz",
+                       "here"],
+                      self.makeExpected(repo="here",
+                                        instrument="a.b.c",
+                                        config_file="/path/to/config",
+                                        out_collection="biz",
+                                        skymap_id="wiz",
+                                        # The list of collections must be in
+                                        # exactly the same order as it is
+                                        # passed in the list of arguments to
+                                        # run_test.
+                                        collections=("foo/bar", "baz", "boz")))
+
+    def test_missing(self):
+        """test a missing argument"""
+        self.run_missing(["make-discrete-skymap", "--collections", "foo/bar,baz", "--instrument", "a.b.c"],
+                         'Missing argument "REPO"')
+        self.run_missing(["make-discrete-skymap", "--collections", "foo/bar,baz", "here"],
+                         'Missing option "-i" / "--instrument"')
+        self.run_missing(["make-discrete-skymap", "--instrument", "a.b.c", "here"],
+                         'Error: Missing option "--collections".')
+
+
+if __name__ == "__main__":
+    unittest.main()