DM-20169: Simplify default search path for fix_header

corrections/CFHT/README.md

@@ -0,0 +1,4 @@
+# Header Corrections for CFHT data
+
+Files must be in YAML format with name of the form `<instrument>-<observation_id>.yaml`.
+See the documentation in `astro_metadata_translator.fix_header`.

corrections/DECam/README.md

@@ -0,0 +1,4 @@
+# Header Corrections for DECam data
+
+Files must be in YAML format with name of the form `<instrument>-<observation_id>.yaml`.
+See the documentation in `astro_metadata_translator.fix_header`.

corrections/HSC/README.md

@@ -0,0 +1,4 @@
+# Header Corrections for HSC data
+
+Files must be in YAML format with name of the form `<instrument>-<observation_id>.yaml`.
+See the documentation in `astro_metadata_translator.fix_header`.

corrections/SuprimeCam/README.md

@@ -0,0 +1,4 @@
+# Header Corrections for SuprimeCam data
+
+Files must be in YAML format with name of the form `<instrument>-<observation_id>.yaml`.
+See the documentation in `astro_metadata_translator.fix_header`.

python/astro_metadata_translator/headers.py

@@ -189,8 +189,9 @@ def fix_header(header, search_path=None, translator_class=None, filename=None):
     ----------
     header : `dict`-like
         Header to correct.
-    search_path : `list`, optional
+    search_path : `list` or `str`, optional
         Explicit directory paths to search for correction files.
+        A single directory path can be given as a string.
     translator_class : `MetadataTranslator`-class, optional
         If not `None`, the class to use to translate the supplied headers
         into standard form. Otherwise each registered translator class will
@@ -207,9 +208,6 @@ def fix_header(header, search_path=None, translator_class=None, filename=None):
 
     Raises
     ------
-    ValueError
-        Raised if the supplied header is not understood by any registered
-        translation classes.
     TypeError
         Raised if the supplied translation class is not a `MetadataTranslator`.
 
@@ -219,7 +217,9 @@ def fix_header(header, search_path=None, translator_class=None, filename=None):
     necessary for the header to be handled by the supplied translator
     class or else support automatic translation class determination.
     It is also required that the ``observation_id`` and ``instrument``
-    be calculable prior to header fix up.
+    be calculable prior to header fix up.  If a translator class can not
+    be found or if there is a problem determining the instrument or
+    observation ID, the function will return without action.
 
     Correction files use names of the form ``instrument-obsid.yaml`` (for
     example ``LATISS-AT_O_20190329_000022.yaml``).
@@ -252,7 +252,13 @@ class or else support automatic translation class determination.
         header_to_translate = header
 
     if translator_class is None:
-        translator_class = MetadataTranslator.determine_translator(header_to_translate, filename=filename)
+        try:
+            translator_class = MetadataTranslator.determine_translator(header_to_translate,
+                                                                       filename=filename)
+        except ValueError:
+            # if the header is not recognized, we should not complain
+            # and should not proceed further.
+            return False
     elif not issubclass(translator_class, MetadataTranslator):
         raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
 
@@ -273,6 +279,9 @@ class or else support automatic translation class determination.
     # Work out the search path
     paths = []
     if search_path is not None:
+        if isinstance(search_path, str):
+            # Allow a single path to be given as a string
+            search_path = [search_path]
         paths.extend(search_path)
     if ENV_VAR_NAME in os.environ and os.environ[ENV_VAR_NAME]:
         paths.extend(os.environ[ENV_VAR_NAME].split(os.path.pathsep))

python/astro_metadata_translator/translator.py

@@ -18,16 +18,26 @@
 import logging
 import warnings
 import math
+import os.path
 
 import astropy.units as u
 import astropy.io.fits.card
 from astropy.coordinates import Angle
 
 from .properties import PROPERTIES
 
-
 log = logging.getLogger(__name__)
 
+# Location of the default package header corrections directory
+CORRECTIONS_DIR = os.path.normpath(
+    os.path.join(
+        os.path.dirname(__file__),
+        os.pardir,  # python
+        os.pardir,  # <root>
+        "corrections",
+    )
+)
+
 
 def cache_translation(func, method=None):
     """Decorator to cache the result of a translation method.
@@ -75,6 +85,8 @@ class MetadataTranslator:
     """
 
     # These are all deliberately empty in the base class.
+    default_search_path = None
+    """Default search path to use to locate header correction files."""
 
     _trivial_map = {}
     """Dict of one-to-one mappings for header translation from standard
@@ -540,7 +552,13 @@ def search_paths(self):
         paths : `list`
             Directory paths to search. Can be an empty list if no special
             directories are defined.
+
+        Notes
+        -----
+        Uses the classes ``default_search_path`` property if defined.
         """
+        if self.default_search_path is not None:
+            return [self.default_search_path]
         return []
 
     def is_key_ok(self, keyword):

python/astro_metadata_translator/translators/decam.py

@@ -14,11 +14,12 @@
 __all__ = ("DecamTranslator", )
 
 import re
+import os.path
 
 from astropy.coordinates import EarthLocation, Angle
 import astropy.units as u
 
-from ..translator import cache_translation
+from ..translator import cache_translation, CORRECTIONS_DIR
 from .fits import FitsTranslator
 from .helpers import altaz_from_degree_headers, is_non_science, \
     tracking_from_degree_headers
@@ -34,6 +35,9 @@ class DecamTranslator(FitsTranslator):
     supported_instrument = "DECam"
     """Supports the DECam instrument."""
 
+    default_search_path = os.path.join(CORRECTIONS_DIR, "DECam")
+    """Default search path to use to locate header correction files."""
+
     _const_map = {"boresight_rotation_angle": Angle(float("nan")*u.deg),
                   "boresight_rotation_coord": "unknown",
                   }

python/astro_metadata_translator/translators/hsc.py

@@ -20,7 +20,7 @@
 import astropy.units as u
 from astropy.coordinates import Angle
 
-from ..translator import cache_translation
+from ..translator import cache_translation, CORRECTIONS_DIR
 from .suprimecam import SuprimeCamTranslator
 
 log = logging.getLogger(__name__)
@@ -36,6 +36,9 @@ class HscTranslator(SuprimeCamTranslator):
     supported_instrument = "HSC"
     """Supports the HSC instrument."""
 
+    default_search_path = os.path.join(CORRECTIONS_DIR, "HSC")
+    """Default search path to use to locate header correction files."""
+
     _const_map = {"instrument": "HSC",
                   "boresight_rotation_coord": "sky"}
     """Hard wire HSC even though modern headers call it Hyper Suprime-Cam"""
@@ -295,15 +298,3 @@ def to_detector_name(self):
         # Name is defined from unique name
         unique = self.to_detector_unique_name()
         return unique.split("_")[1]
-
-    def search_paths(self):
-        # Docstring is inherited from Translator.search_paths.
-        pkg_root = os.path.normpath(
-            os.path.join(
-                os.path.dirname(__file__),
-                os.pardir,  # python/astro_metadata_translator
-                os.pardir,  # python
-                os.pardir,  # <root>
-            )
-        )
-        return [os.path.join(pkg_root, "corrections", "HSC")]

python/astro_metadata_translator/translators/megaprime.py

@@ -14,10 +14,11 @@
 __all__ = ("MegaPrimeTranslator", )
 
 import re
+import os.path
 from astropy.coordinates import EarthLocation, Angle
 import astropy.units as u
 
-from ..translator import cache_translation
+from ..translator import cache_translation, CORRECTIONS_DIR
 from .fits import FitsTranslator
 from .helpers import tracking_from_degree_headers, altaz_from_degree_headers
 
@@ -45,6 +46,9 @@ class MegaPrimeTranslator(FitsTranslator):
     supported_instrument = "MegaPrime"
     """Supports the MegaPrime instrument."""
 
+    default_search_path = os.path.join(CORRECTIONS_DIR, "CFHT")
+    """Default search path to use to locate header correction files."""
+
     _const_map = {"boresight_rotation_angle": Angle(float("nan")*u.deg),
                   "boresight_rotation_coord": "unknown",
                   "detector_group": None}

python/astro_metadata_translator/translators/suprimecam.py

@@ -15,11 +15,12 @@
 
 import re
 import logging
+import os.path
 
 import astropy.units as u
 from astropy.coordinates import SkyCoord, Angle
 
-from ..translator import cache_translation
+from ..translator import cache_translation, CORRECTIONS_DIR
 from .subaru import SubaruTranslator
 from .helpers import altaz_from_degree_headers
 
@@ -36,6 +37,9 @@ class SuprimeCamTranslator(SubaruTranslator):
     supported_instrument = "SuprimeCam"
     """Supports the SuprimeCam instrument."""
 
+    default_search_path = os.path.join(CORRECTIONS_DIR, "SuprimeCam")
+    """Default search path to use to locate header correction files."""
+
     _const_map = {"boresight_rotation_coord": "unknown",
                   "detector_group": None}
     """Constant mappings"""

tests/data/corrections/DECam-ct4m20121211t220632.yaml

@@ -0,0 +1 @@
+DETECTOR: NEW-ID

tests/test_headers.py

@@ -10,8 +10,12 @@
 # license that can be found in the LICENSE file.
 
 import unittest
+import os.path
 
-from astro_metadata_translator import merge_headers
+from astro_metadata_translator import merge_headers, fix_header, HscTranslator
+from astro_metadata_translator.tests import read_test_file
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
 
 
 class HeadersTestCase(unittest.TestCase):
@@ -290,5 +294,49 @@ def test_merging_append_sort(self):
         self.assertEqual(merged, expected)
 
 
+class FixHeadersTestCase(unittest.TestCase):
+
+    def test_basic_fix_header(self):
+        """Test that a header can be fixed if we specify a local path.
+        """
+
+        header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
+        self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3")
+
+        # First fix header but using no search path (should work as no-op)
+        fixed = fix_header(header)
+        self.assertFalse(fixed)
+
+        # Now using the test corrections directory
+        fixed = fix_header(header, search_path=os.path.join(TESTDIR, "data", "corrections"))
+        self.assertTrue(fixed)
+        self.assertEqual(header["DETECTOR"], "NEW-ID")
+
+        # Test that fix_header of unknown header is allowed
+        header = {"SOMETHING": "UNKNOWN"}
+        fixed = fix_header(header)
+        self.assertFalse(fixed)
+
+    def test_hsc_fix_header(self):
+        """Check that one of the known HSC corrections is being applied
+        properly."""
+        header = {"EXP-ID": "HSCA00120800",
+                  "INSTRUME": "HSC",
+                  "DATA-TYP": "FLAT"}
+
+        fixed = fix_header(header, translator_class=HscTranslator)
+        self.assertTrue(fixed)
+        self.assertEqual(header["DATA-TYP"], "OBJECT")
+
+        # And that this header won't be corrected
+        header = {"EXP-ID": "HSCA00120800X",
+                  "INSTRUME": "HSC",
+                  "DATA-TYP": "FLAT"}
+
+        fixed = fix_header(header, translator_class=HscTranslator)
+        self.assertFalse(fixed)
+        self.assertEqual(header["DATA-TYP"], "FLAT")
+
+
 if __name__ == "__main__":
     unittest.main()