-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #148 from lsst/tickets/DM-15189
DM-15189: Initial implementation of Gen3 raw-data ingest
- Loading branch information
Showing
11 changed files
with
647 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
# This file is part of obs_subaru. | ||
# | ||
# Developed for the LSST Data Management System. | ||
# This product includes software developed by the LSST Project | ||
# (http://www.lsst.org). | ||
# See the COPYRIGHT file at the top-level directory of this distribution | ||
# for details of code ownership. | ||
# | ||
# This program is free software: you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License as published by | ||
# the Free Software Foundation, either version 3 of the License, or | ||
# (at your option) any later version. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
||
from . import hsc # noqa | ||
|
||
# suprimecam is rarely used; not automatically imported by default. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
# This file is part of obs_subaru. | ||
# | ||
# Developed for the LSST Data Management System. | ||
# This product includes software developed by the LSST Project | ||
# (http://www.lsst.org). | ||
# See the COPYRIGHT file at the top-level directory of this distribution | ||
# for details of code ownership. | ||
# | ||
# This program is free software: you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License as published by | ||
# the Free Software Foundation, either version 3 of the License, or | ||
# (at your option) any later version. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
||
from .instrument import * # noqa | ||
from .gen2convert import * # noqa | ||
from .ingest import * # noqa | ||
from .rawFormatter import * # noqa |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
# This file is part of obs_subaru. | ||
# | ||
# Developed for the LSST Data Management System. | ||
# This product includes software developed by the LSST Project | ||
# (http://www.lsst.org). | ||
# See the COPYRIGHT file at the top-level directory of this distribution | ||
# for details of code ownership. | ||
# | ||
# This program is free software: you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License as published by | ||
# the Free Software Foundation, either version 3 of the License, or | ||
# (at your option) any later version. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
||
"""Gen3 Butler specializations for Hyper Suprime-Cam. | ||
""" | ||
|
||
import re | ||
|
||
from ....hsc.makeHscRawVisitInfo import MakeHscRawVisitInfo | ||
from ..ingest import SubaruRawIngestTask | ||
from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter | ||
|
||
__all__ = ("HyperSuprimeCamRawIngestTask", ) | ||
|
||
|
||
class HyperSuprimeCamRawIngestTask(SubaruRawIngestTask): | ||
"""Hyper Suprime-Cam Gen3 raw data ingest specialization. | ||
""" | ||
|
||
DAY0 = 55927 # Zero point for 2012-01-01 51544 -> 2000-01-01 | ||
|
||
@staticmethod | ||
def getExposureId(header): | ||
expId = header.getScalar("EXP-ID").strip() | ||
m = re.search(r"^HSCE(\d{8})$", expId) # 2016-06-14 and new scheme | ||
if m: | ||
return int(m.group(1)) | ||
|
||
# Fallback to old scheme | ||
m = re.search(r"^HSC([A-Z])(\d{6})00$", expId) | ||
if not m: | ||
raise RuntimeError("Unable to interpret EXP-ID: %s" % expId) | ||
letter, exposure = m.groups() | ||
exposure = int(exposure) | ||
if exposure == 0: | ||
# Don't believe it | ||
frameId = header.getScalar("FRAMEID").strip() | ||
m = re.search(r"^HSC([A-Z])(\d{6})\d{2}$", frameId) | ||
if not m: | ||
raise RuntimeError("Unable to interpret FRAMEID: %s" % frameId) | ||
letter, exposure = m.groups() | ||
exposure = int(exposure) | ||
if exposure % 2: # Odd? | ||
exposure -= 1 | ||
return exposure + 1000000*(ord(letter) - ord("A")) | ||
|
||
# CCD index mapping for commissioning run 2 | ||
CCD_MAP_COMMISSIONING_2 = {112: 106, | ||
107: 105, | ||
113: 107, | ||
115: 109, | ||
108: 110, | ||
114: 108, | ||
} | ||
|
||
@classmethod | ||
def getSensorId(cls, header): | ||
# Focus CCDs were numbered incorrectly in the readout software during | ||
# commissioning run 2. We need to map to the correct ones. | ||
ccd = super(HyperSuprimeCamRawIngestTask, cls).getSensorId(header) | ||
try: | ||
tjd = cls.getTruncatedModifiedJulianDate(header) | ||
except Exception: | ||
return ccd | ||
|
||
if tjd > 390 and tjd < 405: | ||
ccd = cls.CCD_MAP_COMMISSIONING_2.get(ccd, ccd) | ||
|
||
return ccd | ||
|
||
def makeVisitInfo(self, headers, exposureId): | ||
maker = MakeHscRawVisitInfo(self.log) | ||
return maker(headers[0], exposureId) | ||
|
||
def getFormatter(self, file, headers, dataId): | ||
if dataId["sensor"] in (100, 101, 102, 103): | ||
return HyperSuprimeCamCornerRawFormatter() | ||
else: | ||
return HyperSuprimeCamRawFormatter() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
# This file is part of obs_subaru. | ||
# | ||
# Developed for the LSST Data Management System. | ||
# This product includes software developed by the LSST Project | ||
# (http://www.lsst.org). | ||
# See the COPYRIGHT file at the top-level directory of this distribution | ||
# for details of code ownership. | ||
# | ||
# This program is free software: you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License as published by | ||
# the Free Software Foundation, either version 3 of the License, or | ||
# (at your option) any later version. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
||
"""Gen3 Butler registry declarations for Hyper Suprime-Cam. | ||
""" | ||
|
||
import re | ||
|
||
from lsst.daf.butler.instrument import Instrument | ||
|
||
__all__ = ("HyperSuprimeCam",) | ||
|
||
# Regular expression that matches HSC PhysicalFilter names (the same as Gen2 | ||
# filternames), with a group that can be lowercased to yield the | ||
# associated AbstractFilter. | ||
FILTER_REGEX = re.compile(r"HSC\-([GRIZY])2?") | ||
|
||
|
||
class HyperSuprimeCam(Instrument): | ||
"""Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam. | ||
The current implementation simply retrieves the information it needs | ||
from a Gen2 HscMapper instance (the only constructor argument). This | ||
will obviously need to be changed before Gen2 is retired, but it avoids | ||
duplication for now. | ||
""" | ||
|
||
camera = "HSC" | ||
|
||
def __init__(self, mapper): | ||
self.sensors = [ | ||
dict( | ||
sensor=sensor.getId(), | ||
name=sensor.getName(), | ||
# getType() returns a pybind11-wrapped enum, which | ||
# unfortunately has no way to extract the name of just | ||
# the value (it's always prefixed by the enum type name). | ||
purpose=str(sensor.getType()).split(".")[-1], | ||
# The most useful grouping of sensors in HSC is by their | ||
# orientation w.r.t. the focal plane, so that's what | ||
# we put in the 'group' field. | ||
group="NQUARTER{:d}".format(sensor.getOrientation().getNQuarter() % 4) | ||
) | ||
for sensor in mapper.camera | ||
] | ||
self.physicalFilters = [] | ||
for name in mapper.filters: | ||
# We use one of grizy for the abstract filter, when appropriate, | ||
# which we identify as when the physical filter starts with | ||
# "HSC-[GRIZY]". Note that this means that e.g. "HSC-I" and | ||
# "HSC-I2" are both mapped to abstract filter "i". | ||
m = FILTER_REGEX.match(name) | ||
self.physicalFilters.append( | ||
dict( | ||
physical_filter=name, | ||
abstract_filter=m.group(1).lower() if m is not None else None | ||
) | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# This file is part of obs_subaru. | ||
# | ||
# Developed for the LSST Data Management System. | ||
# This product includes software developed by the LSST Project | ||
# (http://www.lsst.org). | ||
# See the COPYRIGHT file at the top-level directory of this distribution | ||
# for details of code ownership. | ||
# | ||
# This program is free software: you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License as published by | ||
# the Free Software Foundation, either version 3 of the License, or | ||
# (at your option) any later version. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
||
"""Gen3 Butler Formatters for HSC raw data. | ||
""" | ||
|
||
from lsst.afw.image import ImageU, bboxFromMetadata | ||
from lsst.afw.geom import makeSkyWcs, Point2D, makeFlippedWcs | ||
from lsst.afw.math import flipImage | ||
from lsst.daf.butler.formatters.fitsRawFormatterBase import FitsRawFormatterBase | ||
|
||
from ....hsc.makeHscRawVisitInfo import MakeHscRawVisitInfo | ||
|
||
__all__ = ("HyperSuprimeCamRawFormatter", "HyperSuprimeCamCornerRawFormatter") | ||
|
||
|
||
class HyperSuprimeCamRawFormatter(FitsRawFormatterBase): | ||
|
||
FLIP_LR = True | ||
FLIP_TB = False | ||
|
||
def makeVisitInfo(self, metadata): | ||
maker = MakeHscRawVisitInfo() | ||
return maker(metadata, exposureId=0) # TODO: read exposure ID from header, too | ||
|
||
def makeWcs(self, metadata): | ||
wcs = makeSkyWcs(metadata, strip=True) | ||
dimensions = bboxFromMetadata(metadata).getDimensions() | ||
center = Point2D(dimensions/2.0) | ||
return makeFlippedWcs(wcs, self.FLIP_LR, self.FLIP_TB, center) | ||
|
||
def readImage(self, fileDescriptor): | ||
image = ImageU(fileDescriptor.location.path) | ||
return flipImage(image, self.FLIP_LR, self.FLIP_LR) | ||
|
||
|
||
class HyperSuprimeCamCornerRawFormatter(HyperSuprimeCamRawFormatter): | ||
|
||
FLIP_LR = False | ||
FLIP_TB = True |
Oops, something went wrong.