Skip to content

Commit

Permalink
Use astro_metadata_translator for all header translations and unit ex…
Browse files Browse the repository at this point in the history
…traction

Many methods no longer abstract because astro_metadata_translator is handling
the conversions. VisitInfo is now derived directly from an
ObservationInfo.

Leave the existing VisitInfo classes to allow for staged
migration of obs packages to the new approach.
  • Loading branch information
timj committed Oct 19, 2018
1 parent ed3ab9d commit 6c7cff2
Show file tree
Hide file tree
Showing 5 changed files with 289 additions and 103 deletions.
1 change: 1 addition & 0 deletions python/lsst/obs/base/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@
from .cameraMapper import *
from .exposureIdInfo import *
from .makeRawVisitInfo import *
from .makeRawVisitInfoViaObsInfo import *
from .utils import *
128 changes: 25 additions & 103 deletions python/lsst/obs/base/gen3/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.


__all__ = ("RawIngestTask", "RawIngestConfig", "VisitInfoRawIngestTask")
__all__ = ("RawIngestTask", "RawIngestConfig")

import os.path
from abc import ABCMeta, abstractmethod
from abc import ABCMeta

from astro_metadata_translator import ObservationInfo
from lsst.afw.image import readMetadata
from lsst.daf.butler import DatasetType, StorageClassFactory, Run
from lsst.daf.butler.instrument import makeExposureEntryFromVisitInfo, makeVisitEntryFromVisitInfo
from lsst.daf.butler.instrument import makeExposureEntryFromObsInfo, makeVisitEntryFromObsInfo
from lsst.pex.config import Config, Field, ChoiceField
from lsst.pipe.base import Task

Expand Down Expand Up @@ -339,7 +340,6 @@ def processFile(self, file):
else:
self.log.infof("Conflict on {} ({}); ignoring.", dataId, file)

@abstractmethod
def extractDataId(self, file, headers):
"""Return the Data ID dictionary that should be used to label a file.
Expand All @@ -358,9 +358,15 @@ def extractDataId(self, file, headers):
"physical_filter" and "visit" keys should be provided as well
(respectively).
"""
raise NotImplementedError("Must be implemented by subclasses.")
obsInfo = ObservationInfo(headers[0])
return {
"camera": obsInfo.instrument,
"exposure": obsInfo.exposure,
"visit": obsInfo.visit,
"sensor": obsInfo.detector_num,
"physical_filter": obsInfo.physical_filter,
}

@abstractmethod
def extractVisitEntry(self, file, headers, dataId, associated):
"""Create a Visit DataUnit entry from raw file metadata.
Expand All @@ -379,20 +385,21 @@ def extractVisitEntry(self, file, headers, dataId, associated):
Guaranteed to have "Camera", "Sensor", and "PhysicalFilter" keys,
but the last may map to ``None`` if `extractDataId` either did not
contain a "physical_filter" key or mapped it to ``None``.
Subclasses may add new keys to this dict to pass arbitrary data to
`extractExposureEntry` (`extractVisitEntry` is always called
first), but note that when a Visit is comprised of multiple
Exposures, `extractVisitEntry` may not be called at all.
Also adds a "VisitInfo" key containing an `afw.image.VisitInfo`
object for use by `extractExposureEntry`.
Returns
-------
entry : `dict`
Dictionary corresponding to an Visit database table row.
Must have all non-null columns in the Visit table as keys.
"""
raise NotImplementedError("Must be implemented by subclasses.")
obsInfo = ObservationInfo(headers[0])
associated["ObsInfo"] = obsInfo
del dataId["sensor"]
del dataId["exposure"]
return makeVisitEntryFromObsInfo(dataId, obsInfo)

@abstractmethod
def extractExposureEntry(self, file, headers, dataId, associated):
"""Create an Exposure DataUnit entry from raw file metadata.
Expand Down Expand Up @@ -420,7 +427,12 @@ def extractExposureEntry(self, file, headers, dataId, associated):
Dictionary corresponding to an Exposure database table row.
Must have all non-null columns in the Exposure table as keys.
"""
raise NotImplementedError("Must be implemented by subclasses.")
try:
obsInfo = associated["ObsInfo"]
except KeyError:
obsInfo = ObservationInfo(headers[0])
del dataId["sensor"]
return makeExposureEntryFromObsInfo(dataId, obsInfo)

def getFormatter(self, file, headers, dataId):
"""Return the Formatter that should be used to read this file after
Expand All @@ -430,93 +442,3 @@ def getFormatter(self, file, headers, dataId):
configured for this DatasetType/StorageClass in the Butler.
"""
return None


class VisitInfoRawIngestTask(RawIngestTask):
"""An intermediate base class of RawIngestTask for cameras that already
implement constructing a `afw.image.VisitInfo` object from raw data.
Subclasses must provide (at least) implementations of `extractDataId` and
the new `makeVisitInfo` method; the latter is used to provide concrete
implementations of `extractVisitEntry` and `extractExposureEntry`.
"""

@abstractmethod
def makeVisitInfo(self, headers, exposureId):
"""Return an `afw.image.VisitInfo` object from the given header and ID.
Parameters
----------
headers : `list` of `~lsst.daf.base.PropertyList`
All headers returned by `readHeaders()`.
exposureId : `int`
Integer ID to pass to the `VisitInfo` constructor.
"""
raise NotImplementedError("Must be implemented by subclasses.")

def extractVisitEntry(self, file, headers, dataId, associated):
"""Create a Visit DataUnit entry from raw file metadata.
Parameters
----------
file : `str` or path-like object
Absolute path to the file being ingested (prior to any transfers).
headers : `list` of `~lsst.daf.base.PropertyList`
All headers returned by `readHeaders()`.
dataId : `dict`
The data ID for this file. Implementations are permitted to
modify this dictionary (generally by stripping off "sensor" and
"exposure" and adding new metadata key-value pairs) and return it.
associated : `dict`
A dictionary containing other associated DataUnit entries.
Guaranteed to have "Camera", "Sensor", and "PhysicalFilter" keys,
but the last may map to ``None`` if `extractDataId` either did not
contain a "physical_filter" key or mapped it to ``None``.
Also adds a "VisitInfo" key containing an `afw.image.VisitInfo`
object for use by `extractExposureEntry`.
Returns
-------
entry : `dict`
Dictionary corresponding to an Visit database table row.
Must have all non-null columns in the Visit table as keys.
"""
visitInfo = self.makeVisitInfo(headers, exposureId=dataId["exposure"])
associated["VisitInfo"] = visitInfo
del dataId["sensor"]
del dataId["exposure"]
return makeVisitEntryFromVisitInfo(dataId, visitInfo)

def extractExposureEntry(self, file, headers, dataId, associated):
"""Create an Exposure DataUnit entry from raw file metadata.
Parameters
----------
file : `str` or path-like object
Absolute path to the file being ingested (prior to any transfers).
headers : `list` of `~lsst.daf.base.PropertyList`
All headers returned by `readHeaders()`.
dataId : `dict`
The data ID for this file. Implementations are permitted to
modify this dictionary (generally by stripping off "sensor" and
adding new metadata key-value pairs) and return it.
associated : `dict`
A dictionary containing other associated DataUnit entries.
Guaranteed to have "Camera", "Sensor", "PhysicalFilter", and
"Visit" keys, but the latter two may map to ``None`` if
`extractDataId` did not contain keys for these or mapped them to
``None``. May also contain additional keys added by
`extractVisitEntry`.
Returns
-------
entry : `dict`
Dictionary corresponding to an Exposure database table row.
Must have all non-null columns in the Exposure table as keys.
"""
try:
visitInfo = associated["VisitInfo"]
except KeyError:
visitInfo = self.makeVisitInfo(headers, exposureId=dataId["exposure"])
del dataId["sensor"]
return makeExposureEntryFromVisitInfo(dataId, visitInfo)
176 changes: 176 additions & 0 deletions python/lsst/obs/base/makeRawVisitInfoViaObsInfo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# This file is part of obs_base.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

import astropy.units
from astropy.utils import iers

# This is an unofficial ERFA interface provided by Astropy.
# We need to use this to calculate the Earth rotation angle.
# If Astropy change their ERFA support we will need to either bring the
# calculation into this package or use another python ERFA binding.
import astropy._erfa as erfa

from astro_metadata_translator import ObservationInfo

from lsst.log import Log
from lsst.daf.base import DateTime
from lsst.geom import degrees, radians
from lsst.afw.image import VisitInfo, RotType
from lsst.afw.coord import Observatory, Weather
from lsst.geom import SpherePoint

__all__ = ["MakeRawVisitInfoViaObsInfo"]


class MakeRawVisitInfoViaObsInfo(object):
"""Base class functor to make a VisitInfo from the FITS header of a
raw image using `~astro_metadata_translator.ObservationInfo` translators.
Subclasses can be used if a specific
`~astro_metadata_translator.MetadataTranslator` translator should be used.
The design philosophy is to make a best effort and log warnings of
problems, rather than raising exceptions, in order to extract as much
VisitInfo information as possible from a messy FITS header without the
user needing to add a lot of error handling.
Parameters
----------
log : `lsst.log.Log` or None
Logger to use for messages.
(None to use ``Log.getLogger("MakeRawVisitInfoViaObsInfo")``).
"""

metadataTranslator = None
"""Header translator to use to construct VisitInfo, defaulting to
automatic determination."""

def __init__(self, log=None):
if log is None:
log = Log.getLogger("MakeRawVisitInfoViaObsInfo")
self.log = log

def __call__(self, md, exposureId=None):
"""Construct a VisitInfo and strip associated data from the metadata.
Parameters
----------
md : `lsst.daf.base.PropertyList` or `lsst.daf.base.PropertySet`
Metadata to pull from.
Items that are used are stripped from the metadata.
exposureId : `int`, optional
Ignored. Here for compatibility with `MakeRawVisitInfo`.
Returns
-------
visitInfo : `lsst.afw.image.VisitInfo`
`~lsst.afw.image.VisitInfo` derived from the header using
a `~astro_metadata_translator.MetadataTranslator`.
"""
argDict = dict()

obsInfo = ObservationInfo(md, translator_class=self.metadataTranslator)

# Strip all the cards out that were used
for c in obsInfo.cards_used:
del md[c]

# Map the translated information into a form suitable for VisitInfo
if obsInfo.exposure_time is not None:
argDict["exposureTime"] = obsInfo.exposure_time.to_value("s")
if obsInfo.dark_time is not None:
argDict["darkTime"] = obsInfo.dark_time.to_value("s")
argDict["exposureId"] = obsInfo.detector_exposure_id

# VisitInfo uses the middle of the observation for the date
if obsInfo.datetime_begin is not None and obsInfo.datetime_end is not None:
tdelta = obsInfo.datetime_end - obsInfo.datetime_begin
middle = obsInfo.datetime_begin + 0.5*tdelta

# DateTime uses nanosecond resolution, regardless of the resolution
# of the original date
middle.precision = 9
# isot is ISO8601 format with "T" separating date and time and no
# time zone
argDict["date"] = DateTime(middle.tai.isot, DateTime.TAI)

# Derive earth rotation angle from UT1 (being out by a second is not
# a big deal given the uncertainty over exactly what part of the
# observation we are needing it for).
# ERFA needs a UT1 time split into two floats
# We ignore any problems with DUT1 not being defined for now.
try:
ut1time = middle.ut1
except iers.IERSRangeError:
ut1time = middle

era = erfa.era00(ut1time.jd1, ut1time.jd2)
argDict["era"] = era * radians
else:
argDict["date"] = DateTime()

# Coordinates
if obsInfo.tracking_radec is not None:
icrs = obsInfo.tracking_radec.transform_to("icrs")
argDict["boresightRaDec"] = SpherePoint(icrs.ra.degree,
icrs.dec.degree, units=degrees)

altaz = obsInfo.altaz_begin
if altaz is not None:
argDict["boresightAzAlt"] = SpherePoint(altaz.az.degree,
altaz.alt.degree, units=degrees)

argDict["boresightAirmass"] = obsInfo.boresight_airmass

if obsInfo.boresight_rotation_angle is not None:
argDict["boresightRotAngle"] = obsInfo.boresight_rotation_angle.degree*degrees

if obsInfo.boresight_rotation_coord is not None:
rotType = RotType.UNKNOWN
if obsInfo.boresight_rotation_coord == "sky":
rotType = RotType.SKY
argDict["rotType"] = rotType

# Weather and Observatory Location
temperature = float("nan")
if obsInfo.temperature is not None:
temperature = obsInfo.temperature.to_value("deg_C", astropy.units.temperature())
pressure = float("nan")
if obsInfo.pressure is not None:
pressure = obsInfo.pressure.to_value("Pa")
relative_humidity = float("nan")
if obsInfo.relative_humidity is not None:
relative_humidity = obsInfo.relative_humidity
argDict["weather"] = Weather(temperature, pressure, relative_humidity)

if obsInfo.location is not None:
geolocation = obsInfo.location.to_geodetic()
argDict["observatory"] = Observatory(geolocation.lon.degree*degrees,
geolocation.lat.degree*degrees,
geolocation.height.to_value("m"))

for key in list(argDict.keys()): # use a copy because we may delete items
if argDict[key] is None:
self.log.warn("argDict[{}] is None; stripping".format(key, argDict[key]))
del argDict[key]

return VisitInfo(**argDict)

0 comments on commit 6c7cff2

Please sign in to comment.