Merge pull request #202 from lsst/tickets/DM-22655

DM-22655: Genericize gen2to3.py to be useable with any gen2 repo

.gitignore

@@ -13,6 +13,7 @@ config.log
 .cache
 .pytest_cache
 .coverage
+bin/
 doc/html
 doc/*.tag
 doc/*.inc

bin.src/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.shebang()

bin.src/convert_gen2_repo_to_gen3.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+# This file is part of obs_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Convert a gen2 butler repo to gen 3. See
+`lsst.obs.base.ConvertRepoConfig` for most of the config options.
+"""
+
+from lsst.obs.base.script import convertGen2RepoToGen3
+
+
+if __name__ == "__main__":
+    convertGen2RepoToGen3.main()

doc/lsst.obs.base/index.rst

@@ -18,6 +18,15 @@ Contributing
 ``lsst.obs.base`` is developed at https://github.com/lsst/obs_base.
 You can find Jira issues for this module under the `obs_base <https://jira.lsstcorp.org/issues/?jql=project%20%3D%20DM%20AND%20component%20%3D%20obs_base>`_ component.
 
+.. _lsst.obs.base-scripts:
+
+Command Line Scripts
+====================
+
+.. autoprogram:: lsst.obs.base.script.convertGen2RepoToGen3:build_argparser()
+   :prog: convert_gen2_repo_to_gen3.py
+   :groups:
+
 .. _lsst.obs.base-pyapi:
 
 Python API reference

python/lsst/obs/base/gen2to3/calibRepoConverter.py

@@ -38,16 +38,6 @@
     from ..cameraMapper import CameraMapper
     from ..mapping import Mapping as CameraMapperMapping  # disambiguate from collections.abc.Mapping
 
-CURATED_CALIBRATION_DATASET_TYPES = (
-    "defects",
-    "camera",
-    "transmission_sensor",
-    "transmission_filter",
-    "transmission_optics",
-    "transmission_atmosphere",
-    "bfKernel"
-)
-
 
 class CalibRepoConverter(RepoConverter):
     """A specialization of `RepoConverter` for calibration repositories.
@@ -65,11 +55,11 @@ class CalibRepoConverter(RepoConverter):
     def __init__(self, *, mapper: CameraMapper, **kwds):
         super().__init__(**kwds)
         self.mapper = mapper
-        self._datasetTypes = []
+        self._datasetTypes = set()
 
     def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
         # Docstring inherited from RepoConverter.
-        return datasetTypeName in CURATED_CALIBRATION_DATASET_TYPES
+        return datasetTypeName in self.task.config.curatedCalibrations
 
     def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
         # Docstring inherited from RepoConverter.
@@ -86,7 +76,7 @@ def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[s
             instrument=self.task.instrument.getName(),
             universe=self.task.registry.dimensions,
         )
-        self._datasetTypes.append(target.datasetType)
+        self._datasetTypes.add(target.datasetType)
         return target
 
     def insertDimensionData(self):
@@ -105,17 +95,29 @@ def insertDimensionData(self):
                 fields.append(self.task.config.ccdKey)
             else:
                 fields.append(f"NULL AS {self.task.config.ccdKey}")
-            if "physical_filter" in datasetType.dimensions.names:
+            if ("physical_filter" in datasetType.dimensions.names
+                    or "abstract_filter" in datasetType.dimensions.names):
                 fields.append("filter")
             else:
                 fields.append("NULL AS filter")
             query = f"SELECT DISTINCT {', '.join(fields)} FROM {datasetType.name};"
             try:
                 results = db.execute(query)
-            except sqlite3.OperationalError:
-                self.task.log.warn("Could not extract calibration ranges for %s in %s.",
-                                   datasetType.name, self.root)
-                continue
+            except sqlite3.OperationalError as e:
+                if (self.mapper.mappings[datasetType.name].tables is None
+                        or len(self.mapper.mappings[datasetType.name].tables) == 0):
+                    self.task.log.warn("Could not extract calibration ranges for %s in %s: %r",
+                                       datasetType.name, self.root, e)
+                    continue
+                # Try using one of the alternate table names in the mapper (e.g. cpBias->bias for DECam).
+                name = self.mapper.mappings[datasetType.name].tables[0]
+                query = f"SELECT DISTINCT {', '.join(fields)} FROM {name};"
+                try:
+                    results = db.execute(query)
+                except sqlite3.OperationalError as e:
+                    self.task.log.warn("Could not extract calibration ranges for %s in %s: %r",
+                                       datasetType.name, self.root, e)
+                    continue
             for row in results:
                 label = makeCalibrationLabel(datasetType.name, row["calibDate"],
                                              ccd=row[self.task.config.ccdKey], filter=row["filter"])

python/lsst/obs/base/gen2to3/convertRepo.py

@@ -129,6 +129,10 @@ class ConvertRepoConfig(Config):
         keytype=str,
         itemtype=str,
         default={
+            "bias": "ExposureF",
+            "dark": "ExposureF",
+            "flat": "ExposureF",
+            "defects": "Defects",
             "BaseSkyMap": "SkyMap",
             "BaseCatalog": "Catalog",
             "BackgroundList": "Background",
@@ -189,6 +193,18 @@ class ConvertRepoConfig(Config):
         dtype=bool,
         default=False,
     )
+    curatedCalibrations = ListField(
+        "Dataset types that are handled by `Instrument.writeCuratedCalibrations()` "
+        "and thus should not be converted using the standard calibration "
+        "conversion system.",
+        dtype=str,
+        default=["camera",
+                 "transmission_sensor",
+                 "transmission_filter",
+                 "transmission_optics",
+                 "transmission_atmosphere",
+                 "bfKernel"]
+    )
 
     @property
     def transfer(self):
@@ -245,6 +261,7 @@ class ConvertRepoTask(Task):
     _DefaultName = "convertRepo"
 
     def __init__(self, config=None, *, butler3: Butler3, **kwds):
+        config.validate()  # Not a CmdlineTask nor PipelineTask, so have to validate the config here.
         super().__init__(config, **kwds)
         self.butler3 = butler3
         self.registry = self.butler3.registry
@@ -259,11 +276,14 @@ def __init__(self, config=None, *, butler3: Butler3, **kwds):
         self._configuredSkyMapsByName = {}
         for name, config in self.config.skyMaps.items():
             instance = config.skyMap.apply()
-            struct = ConfiguredSkyMap(name=name, sha1=instance.getSha1(), instance=instance)
-            self._configuredSkyMapsBySha1[struct.sha1] = struct
-            self._configuredSkyMapsByName[struct.name] = struct
+            self._populateSkyMapDicts(name, instance)
         self._usedSkyPix = set()
 
+    def _populateSkyMapDicts(self, name, instance):
+        struct = ConfiguredSkyMap(name=name, sha1=instance.getSha1(), instance=instance)
+        self._configuredSkyMapsBySha1[struct.sha1] = struct
+        self._configuredSkyMapsByName[struct.name] = struct
+
     def isDatasetTypeIncluded(self, datasetTypeName: str):
         """Return `True` if configuration indicates that the given dataset type
         should be converted.
@@ -288,7 +308,7 @@ def isDatasetTypeIncluded(self, datasetTypeName: str):
                         for pattern in self.config.datasetIgnorePatterns)
         )
 
-    def useSkyMap(self, skyMap: BaseSkyMap) -> str:
+    def useSkyMap(self, skyMap: BaseSkyMap, skyMapName: str) -> str:
         """Indicate that a repository uses the given SkyMap.
 
         This method is intended to be called primarily by the
@@ -299,17 +319,27 @@ def useSkyMap(self, skyMap: BaseSkyMap) -> str:
         skyMap : `lsst.skymap.BaseSkyMap`
             SkyMap instance being used, typically retrieved from a Gen2
             data repository.
+        skyMapName : `str`
+            The name of the gen2 skymap, for error reporting.
 
         Returns
         -------
         name : `str`
             The name of the skymap in Gen3 data IDs.
+
+        Raises
+        ------
+            LookupError
+                Raised if the specified skymap cannot be found.
         """
         sha1 = skyMap.getSha1()
+        if sha1 not in self._configuredSkyMapsBySha1:
+            self._populateSkyMapDicts(skyMapName, skyMap)
         try:
             struct = self._configuredSkyMapsBySha1[sha1]
         except KeyError as err:
-            raise LookupError(f"SkyMap with sha1={sha1} not included in configuration.") from err
+            msg = f"SkyMap '{skyMapName}' with sha1={sha1} not included in configuration."
+            raise LookupError(msg) from err
         struct.used = True
         return struct.name