Merge pull request #55 from lsst/tickets/DM-34186

DM-34186: Add has_simulated_content and group_counter_*

python/astro_metadata_translator/indexing.py

@@ -11,7 +11,14 @@
 
 from __future__ import annotations
 
-__all__ = ("read_index", "calculate_index", "index_files", "process_index_data")
+__all__ = (
+    "read_index",
+    "read_sidecar",
+    "calculate_index",
+    "index_files",
+    "process_index_data",
+    "process_sidecar_data",
+)
 
 """Functions to support file indexing."""
 
@@ -21,7 +28,7 @@
 import os
 import sys
 from copy import deepcopy
-from typing import IO, Any, List, MutableMapping, Optional, Sequence, Tuple, Union
+from typing import IO, Any, List, Literal, MutableMapping, Optional, Sequence, Tuple, Union, overload
 
 from .file_helpers import read_file_info
 from .headers import merge_headers
@@ -178,9 +185,27 @@ def calculate_index(
     return output
 
 
+@overload
+def read_index(
+    path: str,
+    *,
+    force_dict: Literal[True],
+) -> MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]:
+    ...
+
+
+@overload
+def read_index(
+    path: str,
+    *,
+    force_dict: Literal[False],
+) -> Union[ObservationGroup, MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]]:
+    ...
+
+
 def read_index(
     path: str, force_dict: bool = False
-) -> Union[ObservationGroup, MutableMapping[str, Union[str, MutableMapping[str, Any], ObservationInfo]]]:
+) -> Union[ObservationGroup, MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]]:
     """Read an index file.
 
     Parameters
@@ -200,14 +225,44 @@ def read_index(
         raise ValueError(f"Index files must be in .json format; got {path}")
 
     with open(path, "r") as fd:
-        content = json.loads(fd.read())
+        content: MutableMapping[str, Any] = json.loads(fd.read())
+
+    if not isinstance(content, MutableMapping):
+        raise ValueError(f"The content of the JSON file is {type(content)} and not a dict.")
 
     return process_index_data(content, force_dict=force_dict)
 
 
+@overload
 def process_index_data(
-    content: MutableMapping[str, Any], force_metadata: bool = False, force_dict: bool = False
-) -> Union[ObservationGroup, MutableMapping[str, Union[str, MutableMapping[str, Any], ObservationInfo]]]:
+    content: MutableMapping[str, Any],
+    *,
+    force_metadata: Literal[True],
+    force_dict: Literal[False],
+) -> MutableMapping[str, Any]:
+    ...
+
+
+@overload
+def process_index_data(
+    content: MutableMapping[str, Any],
+    *,
+    force_metadata: Literal[False],
+    force_dict: Literal[True],
+) -> MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]:
+    ...
+
+
+@overload
+def process_index_data(
+    content: MutableMapping[str, Any], *, force_metadata: bool = False, force_dict: bool = False
+) -> Union[ObservationGroup, MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]]:
+    ...
+
+
+def process_index_data(
+    content: MutableMapping[str, Any], *, force_metadata: bool = False, force_dict: bool = False
+) -> Union[ObservationGroup, MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]]]:
     """Process the content read from a JSON index file.
 
     Parameters
@@ -266,7 +321,7 @@ def process_index_data(
     obs_infos: List[ObservationInfo] = []
     # This type annotation is really MutableMapping[str, ObservationInfo]
     # but mypy needs it to look like the function return value.
-    by_file: MutableMapping[str, Union[str, MutableMapping[str, Any], ObservationInfo]] = {}
+    by_file: MutableMapping[str, Union[MutableMapping[str, Any], ObservationInfo]] = {}
     for file, hdr in unpacked.items():
         info = ObservationInfo.from_simple(hdr)
         info.filename = file
@@ -278,7 +333,7 @@ def process_index_data(
     return ObservationGroup(obs_infos)
 
 
-def read_sidecar(path: str) -> Union[ObservationInfo, MutableMapping[str, MutableMapping[str, Any]]]:
+def read_sidecar(path: str) -> Union[ObservationInfo, MutableMapping[str, Any]]:
     """Read a metadata sidecar file.
 
     Parameters
@@ -296,14 +351,38 @@ def read_sidecar(path: str) -> Union[ObservationInfo, MutableMapping[str, Mutabl
         raise ValueError(f"Sidecar files must be in .json format; got {path}")
 
     with open(path, "r") as fd:
-        content = json.loads(fd.read())
+        content: MutableMapping[str, Any] = json.loads(fd.read())
+
+    if not isinstance(content, MutableMapping):
+        raise ValueError(f"The content of the JSON file is {type(content)} and not a dict.")
 
     return process_sidecar_data(content)
 
 
+@overload
+def process_sidecar_data(
+    content: MutableMapping[str, Any],
+) -> Union[ObservationInfo, MutableMapping[str, Any]]:
+    ...
+
+
+@overload
+def process_sidecar_data(
+    content: MutableMapping[str, Any], force_metadata: Literal[True]
+) -> MutableMapping[str, Any]:
+    ...
+
+
+@overload
+def process_sidecar_data(
+    content: MutableMapping[str, Any], force_metadata: Literal[False]
+) -> Union[ObservationInfo, MutableMapping[str, Any]]:
+    ...
+
+
 def process_sidecar_data(
     content: MutableMapping[str, Any], force_metadata: bool = False
-) -> Union[ObservationInfo, MutableMapping[str, MutableMapping[str, Any]]]:
+) -> Union[ObservationInfo, MutableMapping[str, Any]]:
     """Process the content read from a JSON sidecar file.
 
     Parameters
@@ -318,10 +397,11 @@ def process_sidecar_data(
 
     Returns
     -------
-    info : `ObservationInfo` or `dict` of [`str`, `dict`]
+    info : `ObservationInfo` or `dict` of [`str`, `Any`]
         If the sidecar file referred to `ObservationInfo` this will return
-        an `ObservationGroup`, otherwise a `dict` will be returned. This
-        can be overridden using the ``force_metadata`` parameter.
+        an `ObservationInfo`, otherwise a `dict` will be returned. This
+        can be overridden using the ``force_metadata`` parameter in which
+        case a `dict` will always be returned.
     """
 
     if not isinstance(content, dict):

python/astro_metadata_translator/observationInfo.py

@@ -124,6 +124,7 @@ class ObservationInfo:
         physical_filter: str
         datetime_begin: astropy.time.Time
         datetime_end: astropy.time.Time
+        exposure_group: str
         exposure_time: astropy.units.Quantity
         dark_time: astropy.units.Quantity
         boresight_airmass: float
@@ -141,8 +142,14 @@ class ObservationInfo:
         tracking_radec: astropy.coordinates.SkyCoord
         altaz_begin: astropy.coordinates.AltAz
         science_program: str
+        observation_counter: int
+        observation_reason: str
         observation_type: str
         observation_id: str
+        observing_day: int
+        group_counter_start: int
+        group_counter_end: int
+        has_simulated_content: bool
 
     def __init__(
         self,
@@ -655,6 +662,16 @@ def makeObservationInfo(  # noqa: N802
                 super(cls, obsinfo).__setattr__(property, value)  # allows setting write-protected extensions
                 unused.remove(p)
 
+        # Recent additions to ObservationInfo may not be present in
+        # serializations. In theory they can be derived from other
+        # values in the default case. This might not be the right thing
+        # to do.
+        for k in ("group_counter_start", "group_counter_end"):
+            if k not in kwargs and "observation_counter" in kwargs:
+                super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
+        if (k := "has_simulated_content") not in kwargs:
+            super(cls, obsinfo).__setattr__(f"_{k}", False)
+
         if unused:
             n = len(unused)
             raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")

python/astro_metadata_translator/properties.py

@@ -336,4 +336,25 @@ class PropertyDefinition:
         "int",
         int,
     ),
+    "has_simulated_content": PropertyDefinition(
+        "Boolean indicating whether any part of this observation was simulated.", "bool", bool, None, None
+    ),
+    "group_counter_start": PropertyDefinition(
+        "Observation counter for the start of the exposure group."
+        "Depending on the instrument the relevant group may be "
+        "visit_id or exposure_group.",
+        "int",
+        int,
+        None,
+        None,
+    ),
+    "group_counter_end": PropertyDefinition(
+        "Observation counter for the end of the exposure group. "
+        "Depending on the instrument the relevant group may be "
+        "visit_id or exposure_group.",
+        "int",
+        int,
+        None,
+        None,
+    ),
 }

python/astro_metadata_translator/tests.py

@@ -53,7 +53,7 @@ def pl_constructor(loader: yaml.Loader, node: yaml.Node) -> Any:
         elif dtype == "Int":
             pl[key] = int(value)
         elif dtype == "Bool":
-            pl[key] = True if value == "true" else False
+            pl[key] = value
         else:
             pl[key] = value
 

python/astro_metadata_translator/translator.py

@@ -1059,6 +1059,60 @@ def to_observation_counter(self) -> int:
         """
         return 0
 
+    @cache_translation
+    def to_group_counter_start(self) -> int:
+        """Return the observation counter of the observation that began
+        this group.
+
+        The definition of the relevant group is up to the metadata
+        translator. It can be the first observation in the exposure_group
+        or the first observation in the visit, but must be derivable
+        from the metadata of this observation.
+
+        Returns
+        -------
+        counter : `int`
+            The observation counter for the start of the relevant group.
+            Default implementation always returns the observation counter
+            of this observation.
+        """
+        return self.to_observation_counter()
+
+    @cache_translation
+    def to_group_counter_end(self) -> int:
+        """Return the observation counter of the observation that ends
+        this group.
+
+        The definition of the relevant group is up to the metadata
+        translator. It can be the last observation in the exposure_group
+        or the last observation in the visit, but must be derivable
+        from the metadata of this observation. It is of course possible
+        that the last observation in the group does not exist if a sequence
+        of observations was not completed.
+
+        Returns
+        -------
+        counter : `int`
+            The observation counter for the end of the relevant group.
+            Default implementation always returns the observation counter
+            of this observation.
+        """
+        return self.to_observation_counter()
+
+    @cache_translation
+    def to_has_simulated_content(self) -> bool:
+        """Return a boolean indicating whether any part of the observation
+        was simulated.
+
+        Returns
+        -------
+        is_simulated : `bool`
+            `True` if this exposure has simulated content. This can be
+            if some parts of the metadata or data were simulated. Default
+            implementation always returns `False`.
+        """
+        return False
+
     @classmethod
     def determine_translatable_headers(
         cls, filename: str, primary: Optional[MutableMapping[str, Any]] = None

tests/data/bad-sidecar.json

@@ -0,0 +1 @@
+[1,2,3]

tests/data/fitsheader-hsc-HSCA04090107.json

@@ -0,0 +1 @@
+{"SIMPLE": true, "BITPIX": 16, "NAXIS": 2, "NAXIS1": 2144, "NAXIS2": 4241, "EXTEND": false, "BUNIT": "ADU", "BSCALE": 1.0, "BZERO": 32768, "BLANK": -32768, "TIMESYS": "UTC", "DATE-OBS": "2015-10-10", "UT-STR": "08:20:06.598", "HST-STR": "22:20:06.598", "LST-STR": "23:12:41.241", "MJD-STR": 57305.34729859, "UT": "08:20:06.627", "HST": "22:20:06.627", "LST": "23:12:41.270", "MJD": 57305.34729891876, "UT-END": "08:22:38.454", "HST-END": "22:22:38.454", "LST-END": "23:15:13.513", "MJD-END": 57305.3490561787, "ZD-STR": 37.35041361, "ZD-END": 36.81340547, "SECZ-STR": 1.258, "SECZ-END": 1.249, "AIRMASS": 1.257282805496569, "AZIMUTH": 115.5571586, "ALTITUDE": 52.64958639, "PROP-ID": "o15426", "OBSERVER": "\"Utsumi, Sakurai (Remotely from mitaka)\"", "FRAMEID": "HSCA04090107", "EXP-ID": "HSCA04090000", "DATASET": "NODATA", "OBS-MOD": "IMAG", "OBS-ALOC": "Observation", "DATA-TYP": "OBJECT", "OBJECT": "SSP-Wide", "RA": "01:24:37.737", "DEC": "+01:15:35.13", "RA2000": "01:24:37.737", "DEC2000": "+01:15:35.13", "RADESYS": "FK5", "EQUINOX": 2000.0, "CTYPE1": "RA---TAN", "CRPIX1": -3220.833, "CRVAL1": 21.1572375, "CUNIT1": "degree", "CTYPE2": "DEC--TAN", "CRPIX2": 17801.167, "CRVAL2": 1.25975833333, "CUNIT2": "degree", "CD1_1": 2.89152716465347e-21, "CD1_2": 4.72222222222222e-05, "CD2_1": -4.7222222222222e-05, "CD2_2": 2.89152716465347e-21, "LONGPOLE": 180.0, "WCS-ORIG": "Rough estimation", "OBSERVAT": "NAOJ", "TELESCOP": "Subaru", "FOC-POS": "PRIME", "TELFOCUS": "P_OPT2", "FOC-VAL": 3.83, "FILTER01": "HSC-r", "EXPTIME": 150.0, "INSTRUME": "Hyper Suprime-Cam", "DETECTOR": "notSet", "DET-ID": 37, "DET-TMP": -99.775, "DET-TMED": 0.0, "DET-TMIN": 0.0, "DET-TMAX": 0.0, "GAIN": 0.0, "BIN-FCT1": 1, "BIN-FCT2": 1, "DET-VER": "notSet", "INS-VER": "notSet", "WEATHER": "Fine", "SEEING": 0.0, "ADC-TYPE": "LINK", "ADC-STR": 8.42, "ADC-END": 8.25, "INR-STR": -58.084469, "INR-END": -58.084469, "DOM-WND": 0.0, "OUT-WND": 0.7, "DOM-TMP": 279.65, "OUT-TMP": 278.35, "DOM-HUM": 7.4, "OUT-HUM": 8.6, "DOM-PRS": 625.4000000000001, "OUT-PRS": 625.4000000000001, "INST-PA": 0.0, "M2-POS1": -2.927017, "M2-POS2": -4.3757, "M2-POS3": 5.079848, "M2-ANG1": 0.749958, "M2-ANG2": 2.249968, "M2-ANG3": -8.8e-05, "COMMENT": "------------------------------------------------------------", "T_M2OFF1": -1.8, "T_M2OFF2": -2.6, "T_M2OFF3": 3.83, "T_PROG": "SSP-Wide", "T_PNTGID": "W-02-020132-01-00001", "T_PURPOS": "OBJECT", "T_DATSET": "NODATA", "T_SHTDIR": 2, "T_AG": false, "T_UFNAME": "/raid/S15B/20151009/object093_1_07.fits", "T_GAIN1": 3.18, "T_GAIN2": 3.19, "T_GAIN3": 3.35, "T_GAIN4": 3.45, "T_CCDID": "undefined", "T_XFLIP": false, "T_YFLIP": false, "T_CCDSN": "061", "T_NFRAME": 112, "T_CCDTV": -99.775, "T_CCDTM": "22:22:12", "T_HEDVER": "116 CCDs for Summit4", "T_CFGFIL": "20140306.cfg", "T_BEEID": 1, "T_SDOID": 7, "T_OSMN11": 521, "T_OSMX11": 536, "T_EFMN11": 9, "T_EFMX11": 520, "T_OSMN12": 4226, "T_OSMX12": 4241, "T_EFMN12": 50, "T_EFMX12": 4225, "T_OSMN21": 537, "T_OSMX21": 552, "T_EFMN21": 553, "T_EFMX21": 1064, "T_OSMN22": 4226, "T_OSMX22": 4241, "T_EFMN22": 50, "T_EFMX22": 4225, "T_OSMN31": 1593, "T_OSMX31": 1608, "T_EFMN31": 1081, "T_EFMX31": 1592, "T_OSMN32": 4226, "T_OSMX32": 4241, "T_EFMN32": 50, "T_EFMX32": 4225, "T_OSMN41": 1609, "T_OSMX41": 1624, "T_EFMN41": 1625, "T_EFMX41": 2136, "T_OSMN42": 4226, "T_OSMX42": 4241, "T_EFMN42": 50, "T_EFMX42": 4225, "__CONTENT__": "metadata"}

tests/data/fitsheader-hsc.json

@@ -0,0 +1 @@
+{"_translator": "HSC", "telescope": "Subaru", "instrument": "HSC", "location": [-5464492.282102364, -2493000.6266314997, 2150943.6513960036], "exposure_id": 904024, "visit_id": 904024, "physical_filter": "HSC-I", "datetime_begin": [2456599.0, -0.23853115740740743], "datetime_end": [2456599.0, -0.23816055555555554], "exposure_time": 30.0, "dark_time": 30.0, "boresight_airmass": 1.121626027604189, "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "detector_num": 50, "detector_name": "12", "detector_unique_name": "1_12", "detector_serial": "120", "detector_group": "1", "detector_exposure_id": 180804850, "object": "STRIPE82L", "temperature": 272.35, "pressure": 621.7, "relative_humidity": 33.1, "tracking_radec": [320.74992499999996, 0.5000194444444445], "altaz_begin": [226.68922661, 63.04359233], "science_program": "o13015", "observation_type": "science", "observation_id": "HSCA90402400", "observation_reason": "science", "exposure_group": "904024", "observing_day": 20131102, "observation_counter": 904024, "group_counter_end": 904024, "__CONTENT__": "translated"}

tests/test_cfht.py

@@ -41,6 +41,9 @@ def test_megaprime_translator(self):
                     exposure_id=1038843,
                     exposure_group="1038843",
                     exposure_time=615.037 * u.s,
+                    group_counter_end=1038843,
+                    group_counter_start=1038843,
+                    has_simulated_content=False,
                     object="w2.+2+2",
                     observation_counter=1038843,
                     observation_id="1038843",
@@ -71,6 +74,9 @@ def test_megaprime_translator(self):
                     exposure_id=849375,
                     exposure_group="849375",
                     exposure_time=300.202 * u.s,
+                    group_counter_end=849375,
+                    group_counter_start=849375,
+                    has_simulated_content=False,
                     object="D3",
                     observation_counter=849375,
                     observation_id="849375",

tests/test_decam.py

@@ -40,6 +40,9 @@ def test_decam_translator(self):
                     exposure_id=229388,
                     exposure_group="229388",
                     exposure_time=200.0 * u.s,
+                    group_counter_end=229388,
+                    group_counter_start=229388,
+                    has_simulated_content=False,
                     object="DES supernova hex SN-S1 tiling 22",
                     observation_counter=229388,
                     observation_id="ct4m20130901t060255",
@@ -72,6 +75,9 @@ def test_decam_translator(self):
                     exposure_id=160496,
                     exposure_group="160496",
                     exposure_time=0.0 * u.s,
+                    group_counter_end=160496,
+                    group_counter_start=160496,
+                    has_simulated_content=False,
                     object="postflats-BIAS",
                     observation_counter=160496,
                     observation_id="ct4m20121211t220632",
@@ -104,6 +110,9 @@ def test_decam_translator(self):
                     exposure_id=412037,
                     exposure_group="412037",
                     exposure_time=86.0 * u.s,
+                    group_counter_end=412037,
+                    group_counter_start=412037,
+                    has_simulated_content=False,
                     object="Blind15A_03",
                     observation_counter=412037,
                     observation_id="ct4m20150220t004721",
@@ -136,6 +145,9 @@ def test_decam_translator(self):
                     exposure_id=845291,
                     exposure_group="845291",
                     exposure_time=120.0 * u.s,
+                    group_counter_end=845291,
+                    group_counter_start=845291,
+                    has_simulated_content=False,
                     object="",
                     observation_counter=845291,
                     observation_id="ct4m20190402t050618",