Merge pull request #74 from lsst/tickets/DM-21860

DM-21860: Add Gen3 support to obs_cfht

bin.src/megacamCalibRegistry.py

@@ -63,7 +63,8 @@
 
 
 def fixString(s):
-    """Work around apparent pyfits bug: it's not terminating strings at the NULL"""
+    """Work around apparent pyfits bug: it's not terminating strings at the
+    NULL"""
     c = s.find('\0')
     return s if c == -1 else s[:c]
 
@@ -106,8 +107,8 @@ def convertUnixTime(time):
         imageType = imageTypes[imageType]
         filterName = filterNames[filtNum]
 
-        # Remove multiple versions, since our own software doesn't yet handle selection
-        # among multiple versions
+        # Remove multiple versions, since our own software doesn't yet handle
+        # selection among multiple versions
         cur = conn.cursor()
         cmd = "DELETE FROM " + imageType + \
             " WHERE FILTER = ? AND validStart = ? AND validEnd = ? AND version <= ?"

bin/genDefects.py

@@ -11,11 +11,13 @@
 
 def makeBBList(mask, ccd):
 
-    # Create a bounding box corresponding to the useful part of the CCD (exclude overscan regions)
+    # Create a bounding box corresponding to the useful part of the CCD
+    # (exclude overscan regions)
     bb = geom.Box2I(geom.Point2I(32, 0), geom.Point2I(2079, 4611))
     # Read mask file provided by Elixir team
     im = afwImage.ImageF('%s.fits['%mask + str(ccd+1) + ']', bbox=bb, origin=afwImage.ImageOrigin.PARENT)
-    # Pixel values in mask files are 1 for background and 0 for bad pixels - Need to inverse this
+    # Pixel values in mask files are 1 for background and 0 for bad pixels
+    # - Need to inverse this
     im *= -1.
     im += 1.
     im *= 10.

megacam/camera/camera.py

@@ -2230,4 +2230,4 @@
 config.radialCoeffs = None
 
 # Name of this camera
-config.name = 'CFHT MegaCam'
+config.name = 'MegaPrime'

python/lsst/obs/cfht/__init__.py

@@ -21,3 +21,5 @@
 #
 from .version import *
 from .megacamMapper import *
+from ._instrument import MegaPrime
+from .ingest import MegaPrimeRawIngestTask

python/lsst/obs/cfht/_instrument.py

@@ -0,0 +1,111 @@
+# This file is part of obs_cfht.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Butler instrument description for the CFHT MegaCam camera.
+"""
+
+__all__ = ("MegaPrime",)
+
+import os
+from functools import lru_cache
+
+from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
+from lsst.obs.base import Instrument
+from lsst.obs.base.gen2to3 import TranslatorFactory, AbstractToPhysicalFilterKeyHandler
+from .cfhtFilters import MEGAPRIME_FILTER_DEFINITIONS
+
+from lsst.daf.butler.core.utils import getFullTypeName
+from lsst.utils import getPackageDir
+
+
+class MegaPrime(Instrument):
+    filterDefinitions = MEGAPRIME_FILTER_DEFINITIONS
+    policyName = "megacam"
+    obsDataPackage = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        packageDir = getPackageDir("obs_cfht")
+        self.configPaths = [os.path.join(packageDir, "config")]
+
+    @classmethod
+    def getName(cls):
+        return "MegaPrime"
+
+    def getCamera(self):
+        path = os.path.join(getPackageDir("obs_cfht"), self.policyName, "camera")
+        return self._getCameraFromPath(path)
+
+    @staticmethod
+    @lru_cache()
+    def _getCameraFromPath(path):
+        """Return the camera geometry given solely the path to the location
+        of that definition."""
+        config = CameraConfig()
+        config.load(os.path.join(path, "camera.py"))
+        return makeCameraFromPath(
+            cameraConfig=config,
+            ampInfoPath=path,
+            shortNameFunc=lambda name: name.replace(" ", "_"),
+        )
+
+    def register(self, registry):
+        camera = self.getCamera()
+        obsMax = 2**31
+        registry.insertDimensionData(
+            "instrument",
+            {"name": self.getName(), "detector_max": 36, "visit_max": obsMax, "exposure_max": obsMax,
+             "class_name": getFullTypeName(self),
+             }
+        )
+
+        for detector in camera:
+            registry.insertDimensionData(
+                "detector",
+                {
+                    "instrument": self.getName(),
+                    "id": detector.getId(),
+                    "full_name": detector.getName(),
+                    "name_in_raft": detector.getName(),
+                    "raft": None,  # MegaPrime does not have rafts
+                    "purpose": str(detector.getType()).split(".")[-1],
+                }
+            )
+
+        self._registerFilters(registry)
+
+    def getRawFormatter(self, dataId):
+        # local import to prevent circular dependency
+        from .rawFormatter import MegaPrimeRawFormatter
+        return MegaPrimeRawFormatter
+
+    def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
+        # Docstring inherited from lsst.obs.base.Instrument.
+        factory = TranslatorFactory()
+        factory.addGenericInstrumentRules(self.getName(), calibFilterType="abstract_filter")
+
+        # calibRegistry entries are abstract_filters, but we need
+        # physical_filter in the gen3 registry.
+        factory.addRule(AbstractToPhysicalFilterKeyHandler(self.filterDefinitions),
+                        instrument=self.getName(),
+                        gen2keys=("filter",),
+                        consume=("filter",))
+        return factory

python/lsst/obs/cfht/cfhtFilters.py

@@ -0,0 +1,89 @@
+# This file is part of obs_cfht.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ("MEGAPRIME_FILTER_DEFINITIONS",)
+
+from lsst.obs.base import FilterDefinition, FilterDefinitionCollection
+
+# Filter specification comes from
+# https://www.cfht.hawaii.edu/Instruments/Filters/megaprimenew.html
+
+# With current afwFilter singleton we can not define abstract filters
+# properly since we are only allowed one u alias.
+MEGAPRIME_FILTER_DEFINITIONS = FilterDefinitionCollection(
+    FilterDefinition(physical_filter="u.MP9301",
+                     abstract_filter="u",
+                     lambdaEff=374, lambdaMin=336, lambdaMax=412),
+    FilterDefinition(physical_filter="u.MP9302",
+                     abstract_filter="u2",
+                     alias={"u2"},
+                     lambdaEff=354, lambdaMin=310, lambdaMax=397),
+    FilterDefinition(physical_filter="u.MP9303",
+                     abstract_filter="u3",
+                     lambdaEff=395, lambdaMin=390, lambdaMax=400),
+    FilterDefinition(physical_filter="g.MP9401",
+                     abstract_filter="g",
+                     lambdaEff=487, lambdaMin=414, lambdaMax=560),
+    FilterDefinition(physical_filter="g.MP9402",
+                     abstract_filter="g2",
+                     alias={"g2"},
+                     lambdaEff=472, lambdaMin=396, lambdaMax=548),
+    FilterDefinition(physical_filter="g.MP9501",
+                     abstract_filter="g3",
+                     lambdaEff=501, lambdaMin=495, lambdaMax=506),
+    FilterDefinition(physical_filter="g.MP9502",
+                     abstract_filter="g4",
+                     lambdaEff=511, lambdaMin=506, lambdaMax=516),
+    FilterDefinition(physical_filter="r.MP9601",
+                     abstract_filter="r",
+                     lambdaEff=628, lambdaMin=567, lambdaMax=689),
+    FilterDefinition(physical_filter="r.MP9602",
+                     abstract_filter="r2",
+                     alias={"r2"},
+                     lambdaEff=640, lambdaMin=566, lambdaMax=714),
+    FilterDefinition(physical_filter="r.MP9603",
+                     abstract_filter="r3",
+                     lambdaEff=659, lambdaMin=654, lambdaMax=664),
+    FilterDefinition(physical_filter="r.MP9604",
+                     abstract_filter="r4",
+                     lambdaEff=672, lambdaMin=666, lambdaMax=677),
+    FilterDefinition(physical_filter="r.MP9605",
+                     abstract_filter="r5",
+                     lambdaEff=611, lambdaMin=400, lambdaMax=821),
+    FilterDefinition(physical_filter="i.MP9701",
+                     abstract_filter="i",
+                     lambdaEff=778, lambdaMin=702, lambdaMax=853,),
+    FilterDefinition(physical_filter="i.MP9702",
+                     abstract_filter="i2",
+                     alias={"i2"},
+                     lambdaEff=764, lambdaMin=684, lambdaMax=845),
+    FilterDefinition(physical_filter="i.MP9703",
+                     abstract_filter="i3",
+                     alias={"i3"},
+                     lambdaEff=776, lambdaMin=696, lambdaMax=857,),
+    FilterDefinition(physical_filter="z.MP9801",
+                     abstract_filter="z",
+                     lambdaEff=1170, lambdaMin=827, lambdaMax=1514),
+    FilterDefinition(physical_filter="z.MP9901",
+                     abstract_filter="z2",
+                     alias={"z2"},
+                     lambdaEff=926, lambdaMin=849, lambdaMax=1002),
+)

python/lsst/obs/cfht/cfhtIsrTask.py

@@ -46,7 +46,7 @@ def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, def
             Exposure of flatfield.
         defects : `list`
             list of detects
-        fringes : `lsst.afw.image.exposure` or `list` of `lsst.afw.image.exposure`
+        fringes : `lsst.afw.image.Exposure` or list `lsst.afw.image.Exposure`
             exposure of fringe frame or list of fringe exposure
         bfKernel : None
             kernel used for brighter-fatter correction; currently unsupported

python/lsst/obs/cfht/ingest.py

@@ -20,12 +20,17 @@
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
-__all__ = ["MegacamParseTask"]
+__all__ = ["MegacamParseTask", "MegaPrimeRawIngestTask"]
 
 import re
 
+import lsst.obs.base
+from lsst.obs.base.ingest import RawFileData
+from astro_metadata_translator import fix_header
+
 from lsst.pipe.tasks.ingest import ParseTask
 import lsst.pex.exceptions
+from ._instrument import MegaPrime
 
 filters = {'u.MP9301': 'u',
            'u.MP9302': 'u2',
@@ -41,6 +46,34 @@
            }
 
 
+class MegaPrimeRawIngestTask(lsst.obs.base.RawIngestTask):
+    """Task for ingesting raw MegaPrime multi-extension FITS data into Gen3.
+    """
+    def extractMetadata(self, filename: str) -> RawFileData:
+        datasets = []
+        fitsData = lsst.afw.fits.Fits(filename, "r")
+
+        # NOTE: The primary header (HDU=0) does not contain detector data.
+        for i in range(1, fitsData.countHdus()):
+            fitsData.setHdu(i)
+            header = fitsData.readMetadata()
+            if not header["EXTNAME"].startswith("ccd"):
+                continue
+            fix_header(header)
+            datasets.append(self._calculate_dataset_info(header, filename))
+
+        # The data model currently assumes that whilst multiple datasets
+        # can be associated with a single file, they must all share the
+        # same formatter.
+        instrument = MegaPrime()
+        FormatterClass = instrument.getRawFormatter(datasets[0].dataId)
+
+        self.log.info(f"Found images for {len(datasets)} detectors in {filename}")
+        return RawFileData(datasets=datasets, filename=filename,
+                           FormatterClass=FormatterClass,
+                           instrumentClass=type(instrument))
+
+
 class MegacamParseTask(ParseTask):
 
     def translate_ccd(self, md):

python/lsst/obs/cfht/megacamMapper.py

@@ -34,11 +34,13 @@
 from lsst.daf.persistence import Policy
 from lsst.obs.base import CameraMapper, exposureFromImage
 from .makeMegacamRawVisitInfo import MakeMegacamRawVisitInfo
+from ._instrument import MegaPrime
 
 
 class MegacamMapper(CameraMapper):
     """Camera Mapper for CFHT MegaCam."""
     packageName = "obs_cfht"
+    _gen3instrument = MegaPrime
 
     MakeRawVisitInfoClass = MakeMegacamRawVisitInfo
 
@@ -48,8 +50,9 @@ def __init__(self, **kwargs):
         repositoryDir = os.path.dirname(policyFile)
         super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs)
 
-        # Defect registry and root. Defects are stored with the camera and the registry is loaded from the
-        # camera package, which is on the local filesystem.
+        # Defect registry and root. Defects are stored with the camera and the
+        #  registry is loaded from the camera package, which is on the local
+        # filesystem.
         self.defectRegistry = None
         if 'defects' in policy:
             self.defectPath = os.path.join(repositoryDir, policy['defects'])
@@ -96,16 +99,16 @@ def __init__(self, **kwargs):
             self.mappings[name].keyDict.update(keys)
 
         #
-        # The number of bits allocated for fields in object IDs, appropriate for
-        # the default-configured Rings skymap.
+        # The number of bits allocated for fields in object IDs, appropriate
+        # for the default-configured Rings skymap.
         #
 
         MegacamMapper._nbit_tract = 16
         MegacamMapper._nbit_patch = 5
         MegacamMapper._nbit_filter = 6
 
-        MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch +
-                                       MegacamMapper._nbit_filter)
+        MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch
+                                       + MegacamMapper._nbit_filter)
 
         if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter:
             raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %