Merge pull request #128 from lsst/tickets/DM-22655

DM-22655: Genericize gen2to3.py to be useable with any gen2 repo

config/convertRepo.py

@@ -0,0 +1,23 @@
+# Config overrides for converting gen2 to gen3 repos.
+
+from lsst.obs.base.gen2to3 import Translator, AbstractToPhysicalFilterKeyHandler
+import lsst.obs.decam
+
+# Use the specialized Decam ingest task to handle multi-HDU FITS files.
+config.raws.retarget(lsst.obs.decam.DecamRawIngestTask)
+
+# DECam names its detectors differently in the registry
+config.ccdKey = "ccdnum"
+
+# Community Pipeline-produced calibrations are all Images.
+config.storageClasses['cpBias'] = "ExposureF"
+config.storageClasses['cpFlat'] = "ExposureF"
+config.storageClasses['cpIllumcor'] = "ExposureF"
+
+# DECam calibRegistry entries are abstract_filters, but we need physical_filter
+# in the gen3 registry.
+Translator.addRule(AbstractToPhysicalFilterKeyHandler(lsst.obs.decam.DarkEnergyCamera.filterDefinitions),
+                   instrument=lsst.obs.decam.DarkEnergyCamera.getName(),
+                   gen2keys=("filter",),
+                   consume=("filter",),
+                   datasetTypeName="cpFlat")

policy/DecamMapper.yaml

@@ -71,7 +71,7 @@ exposures:
   deepCoadd_psfMatchedWarp:
     template: deepCoadd/%(filter)s/%(tract)d/%(patch)stempExp/psfMatched-v%(visit)d-f%(filter)s.fits
   dcrCoadd_directWarp:
-    template: dcrCoadd/%(filter)s/%(tract)d/%(patch)stempExp/v%(visit)d-f%(filter)s%(subfilter)s.fits
+    template: dcrCoadd/%(filter)s/%(tract)d/%(patch)stempExp/v%(visit)d-f%(filter)s%(subfilter)d.fits
   deepDiff_differenceExp:
     template: deepDiff/v%(visit)d/diffexp-%(ccdnum)02d.fits
   deepDiff_matchedExp:
@@ -261,7 +261,7 @@ datasets:
   deep_safeClipAssembleCoadd_metadata:
     template: deepCoadd/%(filter)s/%(tract)d/%(patch)s/assembleCoaddMD.yaml
   dcrCoadd_measMatchFull:
-    template: dcrCoadd-results/%(filter)s%(subfilter)s/%(tract)d/%(patch)s/srcMatchFull-%(filter)s%(subfilter)s-%(tract)d-%(patch)s.fits
+    template: dcrCoadd-results/%(filter)s%(subfilter)d/%(tract)d/%(patch)s/srcMatchFull-%(filter)s%(subfilter)d-%(tract)d-%(patch)s.fits
   detectCoaddSources_config:
     template: config/detectCoadd.py
   detectCoaddSources_metadata:
@@ -287,7 +287,7 @@ datasets:
   deepDiff_kernelSrc:
     template: deepDiff/v%(visit)d/kernelSrc-%(ccdnum)02d.fits
   dcrCoadd_measMatch:
-    template: dcrCoadd-results/%(filter)s%(subfilter)s/%(tract)d/%(patch)s/srcMatch-%(filter)s%(subfilter)s-%(tract)d-%(patch)s.fits
+    template: dcrCoadd-results/%(filter)s%(subfilter)d/%(tract)d/%(patch)s/srcMatch-%(filter)s%(subfilter)d-%(tract)d-%(patch)s.fits
   dcrCoadd_forced_src:
     template: dcrCoadd-results/%(filter)s/%(tract)d/%(patch)s/forced-%(filter)s-%(tract)d-%(patch)s.fits
   dcrCoadd_forced_metadata:

python/lsst/obs/decam/decamFilters.py

@@ -49,6 +49,6 @@
     FilterDefinition(physical_filter="N964 DECam c0008 9645.0 94.0",
                      abstract_filter="N964",
                      lambdaEff=964),
-    FilterDefinition(physical_filter="solid",
+    FilterDefinition(physical_filter="solid plate 0.0 0.0",
                      afw_name='SOLID', lambdaEff=0)
 )

python/lsst/obs/decam/rawFormatter.py

@@ -31,7 +31,7 @@
 
 from . import DarkEnergyCamera
 
-__all__ = ("DarkEnergyCameraRawFormatter",)
+__all__ = ("DarkEnergyCameraRawFormatter", "DarkEnergyCameraCPCalibFormatter")
 
 
 # The mapping of detector id to HDU in raw files for "most" DECam data.
@@ -133,3 +133,22 @@ def readMetadata(self):
     def readImage(self):
         index, metadata = self._determineHDU(self.dataId['detector'])
         return lsst.afw.image.ImageI(self.fileDescriptor.location.path, index)
+
+
+class DarkEnergyCameraCPCalibFormatter(DarkEnergyCameraRawFormatter):
+    """DECam Community Pipeline calibrations (bias, dark, flat, fringe) are
+    multi-extension FITS files with detector=index+1.
+    """
+
+    def _determineHDU(self, detectorId):
+        """The HDU to read is the same as the detector number."""
+        filename = self.fileDescriptor.location.path
+        metadata = lsst.afw.image.readMetadata(filename, detectorId)
+        if metadata['CCDNUM'] != detectorId:
+            msg = f"Found CCDNUM={metadata['CCDNUM']} instead of {detectorId} in {filename} HDU={detectorId}."
+            raise ValueError(msg)
+        return detectorId, metadata
+
+    def readImage(self):
+        index, metadata = self._determineHDU(self.dataId['detector'])
+        return lsst.afw.image.ImageF(self.fileDescriptor.location.path, index)

setup.cfg

@@ -6,6 +6,7 @@ exclude =
     __init__.py
     config/*
     tests/.tests/*
+    tests/config/*
     decam/camGeom/camera.py
 
 

tests/config/convert2to3Config.py

@@ -0,0 +1,4 @@
+# Config overrides for `test_convert2to3.py`
+# The decam conversion tests use `ap_verify_ci_hits2015`, which has two refcats.
+
+config.refCats = ['gaia', 'panstarrs']

tests/test_convert2to3.py

@@ -0,0 +1,80 @@
+# This file is part of obs_decam.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Unit tests for DECam gen2 to gen3 conversion.
+"""
+
+import itertools
+import os
+import unittest
+
+import lsst.utils.tests
+from lsst.obs.base.gen2to3 import convertTests
+import lsst.obs.decam
+
+# testDataPackage = "testdata_decam"
+testDataPackage = "ap_verify_ci_hits2015"
+try:
+    testDataDirectory = lsst.utils.getPackageDir(testDataPackage)
+except LookupError:
+    testDataDirectory = None
+
+
+@unittest.skipIf(testDataDirectory is None, f"{testDataPackage} not setup")
+class DecamConvertGen2To3TestCase(convertTests.ConvertGen2To3TestCase,
+                                  lsst.utils.tests.TestCase):
+    def setUp(self):
+        # self.gen2root = os.path.join(testDataDirectory, 'rawData')
+        # self.gen2calib = os.path.join(testDataDirectory, 'rawData/cpCalib')
+        # NOTE: this test will produce a bunch fo "Skipping ingestion for" warnings,
+        # due to the bias/flat calibRegistry only having the same ccds as the
+        # raws, even though the files themselves have more.
+        self.gen2root = os.path.join(testDataDirectory, 'gen2-output/ingested')
+        self.gen2calib = os.path.join(testDataDirectory, 'gen2-output/calibingested')
+        self.instrumentName = "DECam"
+        self.instrumentClass = "lsst.obs.decam.DarkEnergyCamera"
+        self.config = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                   "config", "convert2to3Config.py")
+        self.args = ("--calibFilterType", "abstract_filter")
+        self.collections = set(['skymaps'])
+
+        detectors = (5, 10, 56, 60)
+        dates = ('2015-02-18', '2015-03-13')
+        self.biasName = 'cpBias'
+        self.biases = [{'detector': detector, 'calibration_label': f"gen2/cpBias_{date}_{detector:03}",
+                        'instrument': 'DECam'}
+                       for detector, date in itertools.product(detectors, dates)]
+        self.flatName = 'cpFlat'
+        self.flats = [{'detector': detector,
+                       'calibration_label': f"gen2/cpFlat_{date}_{detector:03}_g",
+                       'instrument': 'DECam', 'physical_filter': 'g DECam SDSS c0001 4720.0 1520.0'}
+                      for detector, date in itertools.product(detectors, dates)]
+        self.refcats = ['gaia', 'panstarrs']
+        super().setUp()
+
+
+def setup_module(module):
+    lsst.utils.tests.init()
+
+
+if __name__ == "__main__":
+    lsst.utils.tests.init()
+    unittest.main()

tests/test_formatter.py

@@ -38,12 +38,18 @@
 except LookupError:
     testDataDirectory = None
 
+try:
+    hitsDirectory = lsst.utils.getPackageDir("ap_verify_ci_hits2015")
+except LookupError:
+    hitsDirectory = None
+
 
 @unittest.skipIf(testDataDirectory is None, "testdata_decam must be set up")
 class DarkEnergyCameraRawFormatterTestCase(lsst.utils.tests.TestCase):
     def setUp(self):
-        self.filename = os.path.join(testDataDirectory, 'rawData/2013-09-01/z/decam0229388.fits.fz')
-        location = lsst.daf.butler.Location(testDataDirectory, 'rawData/2013-09-01/z/decam0229388.fits.fz')
+        path = 'rawData/2013-09-01/z/decam0229388.fits.fz'
+        self.filename = os.path.join(testDataDirectory, path)
+        location = lsst.daf.butler.Location(testDataDirectory, path)
         self.fileDescriptor = lsst.daf.butler.FileDescriptor(location, None)
 
     def check_readMetadata(self, dataId, expected):
@@ -112,6 +118,41 @@ def test_readMetadata_full_file(self):
             self.assertEqual(shuffled_metadata, full_metadata)
 
 
+@unittest.skipIf(hitsDirectory is None, "ap_verify_ci_hits2015 must be set up")
+class DarkEnergyCameraCPCalibFormatterTestCase(lsst.utils.tests.TestCase):
+    """DECam Community Pipeline calibrations have one detector per HDU.
+
+    Note: this test uses the ap_verify_ci_hits2015 dataset, which has complete
+    calibrations, whereas the testdata_decam package has only one detector in
+    its bias and flat files.
+    """
+    def setUp(self):
+        path = 'calib/c4d_150218_191721_zci_v1.fits.fz'
+        self.biasFile = os.path.join(hitsDirectory, path)
+        location = lsst.daf.butler.Location(hitsDirectory, path)
+        self.biasDescriptor = lsst.daf.butler.FileDescriptor(location, None)
+
+        path = 'calib/c4d_150218_200522_fci_g_v1.fits.fz'
+        self.flatFile = os.path.join(hitsDirectory, path)
+        location = lsst.daf.butler.Location(hitsDirectory, path)
+        self.flatDescriptor = lsst.daf.butler.FileDescriptor(location, None)
+
+    def check_readMetadata(self, dataId, expected, fileDescriptor):
+        formatter = lsst.obs.decam.DarkEnergyCameraCPCalibFormatter(fileDescriptor, dataId)
+        metadata = formatter.readMetadata()
+        self.assertEqual(metadata['CCDNUM'], dataId['detector'], dataId)
+        self.assertEqual(metadata, expected, msg=dataId)
+
+    def test_readMetadata(self):
+        for i in range(1, 63):
+            dataId = {'detector': i}
+            expected = lsst.afw.image.readMetadata(self.biasFile, i)
+            self.check_readMetadata(dataId, expected, self.biasDescriptor)
+
+            expected = lsst.afw.image.readMetadata(self.flatFile, i)
+            self.check_readMetadata(dataId, expected, self.flatDescriptor)
+
+
 def setup_module(module):
     lsst.utils.tests.init()
 

tests/test_instrument.py

@@ -39,7 +39,7 @@ def setUp(self):
                             "z DECam SDSS c0004 9260.0 1520.0",
                             "i DECam SDSS c0003 7835.0 1470.0",
                             "u DECam c0006 3500.0 1000.0",
-                            "solid"}
+                            "solid plate 0.0 0.0"}
         self.data = InstrumentTestData(name="DECam",
                                        nDetectors=62,
                                        firstDetectorName="S29",