DM-15189: Initial implementation of Gen3 raw-data ingest

python/lsst/obs/subaru/gen3/__init__.py

@@ -0,0 +1,24 @@
+# This file is part of obs_subaru.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import hsc  # noqa
+
+# suprimecam is rarely used; not automatically imported by default.

python/lsst/obs/subaru/gen3/hsc/__init__.py

@@ -0,0 +1,25 @@
+# This file is part of obs_subaru.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .instrument import *  # noqa
+from .gen2convert import *  # noqa
+from .ingest import *  # noqa
+from .rawFormatter import *  # noqa

python/lsst/obs/subaru/gen3.py → python/lsst/obs/subaru/gen3/hsc/gen2convert.py

@@ -19,63 +19,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Gen3 Butler specializations for Hyper Suprime-Cam.
+"""Gen2->Gen3 Data repo conversion specializations for Hyper Suprime-Cam
 """
 
-import re
-
-from lsst.daf.butler.instrument import Instrument
 from lsst.daf.butler.gen2convert import Translator, KeyHandler, ConstantKeyHandler, CopyKeyHandler
 
-__all__ = ("HyperSuprimeCam",)
-
-
-# Regular expression that matches HSC PhysicalFilter names (the same as Gen2
-# filternames), with a group that can be lowercased to yield the
-# associated AbstractFilter.
-FILTER_REGEX = re.compile(r"HSC\-([GRIZY])2?")
+from .instrument import FILTER_REGEX
 
 
-class HyperSuprimeCam(Instrument):
-    """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
-
-    The current implementation simply retrieves the information it needs
-    from a Gen2 HscMapper instance (the only constructor argument).  This
-    will obviously need to be changed before Gen2 is retired, but it avoids
-    duplication for now.
-    """
-
-    camera = "HSC"
-
-    def __init__(self, mapper):
-        self.sensors = [
-            dict(
-                sensor=sensor.getId(),
-                name=sensor.getName(),
-                # getType() returns a pybind11-wrapped enum, which
-                # unfortunately has no way to extract the name of just
-                # the value (it's always prefixed by the enum type name).
-                purpose=str(sensor.getType()).split(".")[-1],
-                # The most useful grouping of sensors in HSC is by their
-                # orientation w.r.t. the focal plane, so that's what
-                # we put in the 'group' field.
-                group="NQUARTER{:d}".format(sensor.getOrientation().getNQuarter() % 4)
-            )
-            for sensor in mapper.camera
-        ]
-        self.physicalFilters = []
-        for name in mapper.filters:
-            # We use one of grizy for the abstract filter, when appropriate,
-            # which we identify as when the physical filter starts with
-            # "HSC-[GRIZY]".  Note that this means that e.g. "HSC-I" and
-            # "HSC-I2" are both mapped to abstract filter "i".
-            m = FILTER_REGEX.match(name)
-            self.physicalFilters.append(
-                dict(
-                    physical_filter=name,
-                    abstract_filter=m.group(1).lower() if m is not None else None
-                )
-            )
+__all__ = ()
 
 
 class HscAbstractFilterKeyHandler(KeyHandler):

python/lsst/obs/subaru/gen3/hsc/ingest.py

@@ -0,0 +1,97 @@
+# This file is part of obs_subaru.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Gen3 Butler specializations for Hyper Suprime-Cam.
+"""
+
+import re
+
+from ....hsc.makeHscRawVisitInfo import MakeHscRawVisitInfo
+from ..ingest import SubaruRawIngestTask
+from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
+
+__all__ = ("HyperSuprimeCamRawIngestTask", )
+
+
+class HyperSuprimeCamRawIngestTask(SubaruRawIngestTask):
+    """Hyper Suprime-Cam Gen3 raw data ingest specialization.
+    """
+
+    DAY0 = 55927  # Zero point for  2012-01-01  51544 -> 2000-01-01
+
+    @staticmethod
+    def getExposureId(header):
+        expId = header.getScalar("EXP-ID").strip()
+        m = re.search(r"^HSCE(\d{8})$", expId)  # 2016-06-14 and new scheme
+        if m:
+            return int(m.group(1))
+
+        # Fallback to old scheme
+        m = re.search(r"^HSC([A-Z])(\d{6})00$", expId)
+        if not m:
+            raise RuntimeError("Unable to interpret EXP-ID: %s" % expId)
+        letter, exposure = m.groups()
+        exposure = int(exposure)
+        if exposure == 0:
+            # Don't believe it
+            frameId = header.getScalar("FRAMEID").strip()
+            m = re.search(r"^HSC([A-Z])(\d{6})\d{2}$", frameId)
+            if not m:
+                raise RuntimeError("Unable to interpret FRAMEID: %s" % frameId)
+            letter, exposure = m.groups()
+            exposure = int(exposure)
+            if exposure % 2:  # Odd?
+                exposure -= 1
+        return exposure + 1000000*(ord(letter) - ord("A"))
+
+    # CCD index mapping for commissioning run 2
+    CCD_MAP_COMMISSIONING_2 = {112: 106,
+                               107: 105,
+                               113: 107,
+                               115: 109,
+                               108: 110,
+                               114: 108,
+                               }
+
+    @classmethod
+    def getSensorId(cls, header):
+        # Focus CCDs were numbered incorrectly in the readout software during
+        # commissioning run 2.  We need to map to the correct ones.
+        ccd = super(HyperSuprimeCamRawIngestTask, cls).getSensorId(header)
+        try:
+            tjd = cls.getTruncatedModifiedJulianDate(header)
+        except Exception:
+            return ccd
+
+        if tjd > 390 and tjd < 405:
+            ccd = cls.CCD_MAP_COMMISSIONING_2.get(ccd, ccd)
+
+        return ccd
+
+    def makeVisitInfo(self, headers, exposureId):
+        maker = MakeHscRawVisitInfo(self.log)
+        return maker(headers[0], exposureId)
+
+    def getFormatter(self, file, headers, dataId):
+        if dataId["sensor"] in (100, 101, 102, 103):
+            return HyperSuprimeCamCornerRawFormatter()
+        else:
+            return HyperSuprimeCamRawFormatter()

python/lsst/obs/subaru/gen3/hsc/instrument.py

@@ -0,0 +1,76 @@
+# This file is part of obs_subaru.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
+"""
+
+import re
+
+from lsst.daf.butler.instrument import Instrument
+
+__all__ = ("HyperSuprimeCam",)
+
+# Regular expression that matches HSC PhysicalFilter names (the same as Gen2
+# filternames), with a group that can be lowercased to yield the
+# associated AbstractFilter.
+FILTER_REGEX = re.compile(r"HSC\-([GRIZY])2?")
+
+
+class HyperSuprimeCam(Instrument):
+    """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
+
+    The current implementation simply retrieves the information it needs
+    from a Gen2 HscMapper instance (the only constructor argument).  This
+    will obviously need to be changed before Gen2 is retired, but it avoids
+    duplication for now.
+    """
+
+    camera = "HSC"
+
+    def __init__(self, mapper):
+        self.sensors = [
+            dict(
+                sensor=sensor.getId(),
+                name=sensor.getName(),
+                # getType() returns a pybind11-wrapped enum, which
+                # unfortunately has no way to extract the name of just
+                # the value (it's always prefixed by the enum type name).
+                purpose=str(sensor.getType()).split(".")[-1],
+                # The most useful grouping of sensors in HSC is by their
+                # orientation w.r.t. the focal plane, so that's what
+                # we put in the 'group' field.
+                group="NQUARTER{:d}".format(sensor.getOrientation().getNQuarter() % 4)
+            )
+            for sensor in mapper.camera
+        ]
+        self.physicalFilters = []
+        for name in mapper.filters:
+            # We use one of grizy for the abstract filter, when appropriate,
+            # which we identify as when the physical filter starts with
+            # "HSC-[GRIZY]".  Note that this means that e.g. "HSC-I" and
+            # "HSC-I2" are both mapped to abstract filter "i".
+            m = FILTER_REGEX.match(name)
+            self.physicalFilters.append(
+                dict(
+                    physical_filter=name,
+                    abstract_filter=m.group(1).lower() if m is not None else None
+                )
+            )

python/lsst/obs/subaru/gen3/hsc/rawFormatter.py

@@ -0,0 +1,58 @@
+# This file is part of obs_subaru.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Gen3 Butler Formatters for HSC raw data.
+"""
+
+from lsst.afw.image import ImageU, bboxFromMetadata
+from lsst.afw.geom import makeSkyWcs, Point2D, makeFlippedWcs
+from lsst.afw.math import flipImage
+from lsst.daf.butler.formatters.fitsRawFormatterBase import FitsRawFormatterBase
+
+from ....hsc.makeHscRawVisitInfo import MakeHscRawVisitInfo
+
+__all__ = ("HyperSuprimeCamRawFormatter", "HyperSuprimeCamCornerRawFormatter")
+
+
+class HyperSuprimeCamRawFormatter(FitsRawFormatterBase):
+
+    FLIP_LR = True
+    FLIP_TB = False
+
+    def makeVisitInfo(self, metadata):
+        maker = MakeHscRawVisitInfo()
+        return maker(metadata, exposureId=0)  # TODO: read exposure ID from header, too
+
+    def makeWcs(self, metadata):
+        wcs = makeSkyWcs(metadata, strip=True)
+        dimensions = bboxFromMetadata(metadata).getDimensions()
+        center = Point2D(dimensions/2.0)
+        return makeFlippedWcs(wcs, self.FLIP_LR, self.FLIP_TB, center)
+
+    def readImage(self, fileDescriptor):
+        image = ImageU(fileDescriptor.location.path)
+        return flipImage(image, self.FLIP_LR, self.FLIP_LR)
+
+
+class HyperSuprimeCamCornerRawFormatter(HyperSuprimeCamRawFormatter):
+
+    FLIP_LR = False
+    FLIP_TB = True