DM-29266: Add translator methods to read all relevant HDUs

python/astro_metadata_translator/translator.py

@@ -943,6 +943,39 @@ def to_observation_counter(self):
         """
         return 0
 
+    @classmethod
+    def read_all_headers(cls, filename, primary=None):
+        """Read all relevant headers from the given file.
+
+        This should simply return all headers, but for multi-extension FITS
+        files where each file includes data from multiple detectors,
+        all the headers for each detector should be returned.
+
+        Parameters
+        ----------
+        filename : `str`
+            Path to a file in a format understood by this translator.
+        primary : `dict`-like, optional
+            The primary header obtained by the caller. This is sometimes
+            already known, for example if a system is trying to bootstrap
+            without already knowing what data is in the file. For many
+            instruments where the primary header is the only relevant
+            header, the primary header will be returned with no further
+            action.
+
+        Yields
+        ------
+        headers : iterator of `dict`-like
+            Each relevant header in turn.
+        """
+        if primary is not None:
+            yield primary
+        else:
+            # Prevent circular import by deferring
+            from .file_helpers import read_basic_metadata_from_file
+
+            yield read_basic_metadata_from_file(filename, 0)
+
 
 def _make_abstract_translator_method(property, doc, return_typedoc, return_type):
     """Create a an abstract translation method for this property.

python/astro_metadata_translator/translators/decam.py

@@ -17,6 +17,7 @@
 import posixpath
 import logging
 
+from astropy.io import fits
 from astropy.coordinates import EarthLocation, Angle
 import astropy.units as u
 
@@ -297,3 +298,57 @@ def fix_header(cls, header, instrument, obsid, filename=None):
                       log_label, header["FILTER"], obstype)
 
         return modified
+
+    @classmethod
+    def read_all_headers(cls, filename, primary=None):
+        """Read all relevant headers from the given file.
+
+        Returns all non-guide headers and does not include the primary
+        header.
+
+        Parameters
+        ----------
+        filename : `str`
+            Path to a file in a format understood by this translator.
+        primary : `dict`-like, optional
+            The primary header obtained by the caller. This is sometimes
+            already known, for example if a system is trying to bootstrap
+            without already knowing what data is in the file. For many
+            instruments where the primary header is the only relevant
+            header, the primary header will be returned with no further
+            action.
+
+        Yields
+        ------
+        headers : iterator of `dict`-like
+            Each relevant header in turn.
+
+        Notes
+        -----
+        DECam data use INHERIT=T so all returned headers will be merged
+        with the primary header.
+        """
+        # Circular dependency so must defer import
+        from ..headers import merge_headers
+
+        # Since we want to scan many HDUs we use astropy directly to keep
+        # the file open rather than continually opening and closing it
+        # as we go to each HDU.
+        with fits.open(filename) as fits_file:
+            # Astropy does not automatically handle the INHERIT=T in
+            # DECam headers so the primary header must be merged.
+            first_pass = True
+
+            for hdu in fits_file:
+                if first_pass:
+                    if not primary:
+                        primary = hdu.header
+                    first_pass = False
+                    continue
+
+                header = hdu.header
+                if "CCDNUM" not in header:
+                    continue
+                if header["CCDNUM"] > 62:  # ignore guide CCDs
+                    continue
+                yield merge_headers([primary, header], mode="overwrite")

python/astro_metadata_translator/translators/megaprime.py

@@ -15,6 +15,8 @@
 
 import re
 import posixpath
+
+from astropy.io import fits
 from astropy.coordinates import EarthLocation, Angle
 import astropy.units as u
 
@@ -183,3 +185,62 @@ def to_observation_counter(self):
             The observation counter.
         """
         return self.to_exposure_id()
+
+    @classmethod
+    def read_all_headers(cls, filename, primary=None):
+        """Read all relevant headers from the given file.
+
+        Returns all non-guide headers and does not include the primary
+        header.
+
+        Parameters
+        ----------
+        filename : `str`
+            Path to a file in a format understood by this translator.
+        primary : `dict`-like, optional
+            The primary header obtained by the caller. This is sometimes
+            already known, for example if a system is trying to bootstrap
+            without already knowing what data is in the file. For many
+            instruments where the primary header is the only relevant
+            header, the primary header will be returned with no further
+            action.
+
+        Yields
+        ------
+        headers : iterator of `dict`-like
+            Each relevant header in turn.
+
+        Notes
+        -----
+        MegaPrime data use INHERIT=F such that the primary header will never
+        be returned.
+        """
+        # Since we want to scan many HDUs we use astropy directly to keep
+        # the file open rather than continually opening and closing it
+        # as we go to each HDU.
+        with fits.open(filename) as fits_file:
+            for hdu in fits_file:
+                # Astropy <=4.2 strips the EXTNAME header but some CFHT data
+                # have two EXTNAME headers and the CCD number is in the
+                # second one.
+                if hdu.name == "PRIMARY":
+                    continue
+
+                if hdu.name.startswith("ccd"):
+                    # It may only be some data files that are broken so
+                    # handle the expected form.
+                    header = hdu.header
+
+                    # Astropy strips EXTNAME but translator needs it
+                    header["EXTNAME"] = hdu.name
+                    yield header
+                    continue
+
+                # Some test data at least has the EXTNAME as
+                # COMPRESSED_IMAGE but the EXTVER as the detector number.
+                if hdu.name == "COMPRESSED_IMAGE":
+                    header = hdu.header
+
+                    # Astropy strips EXTNAME so put it back for the translator
+                    header["EXTNAME"] = f"ccd{hdu.ver:02d}"
+                    yield header