DM-27476: Update raw ingest to use JSON metadata files

python/lsst/obs/base/ingest_tests.py

@@ -50,6 +50,12 @@ class IngestTestBase(metaclass=abc.ABCMeta):
     actual directory will be a tempdir under this one.
     """
 
+    ingestDatasetTypeName = "raw"
+    """The DatasetType to use for the ingest.
+
+    If this is not an Exposure dataset type the tests will be more limited.
+    """
+
     dataIds = []
     """list of butler data IDs of files that should have been ingested."""
 
@@ -150,34 +156,49 @@ def verifyIngest(self, files=None, cli=False, fullCheck=False):
         This only really affects files that contain multiple datasets.
         """
         butler = Butler(self.root, run=self.outputRun)
-        datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
+        datasets = list(butler.registry.queryDatasets(self.ingestDatasetTypeName, collections=self.outputRun))
         self.assertEqual(len(datasets), len(self.dataIds))
 
         # Get the URI to the first dataset and check it is inside the
         # datastore
         datasetUri = butler.getURI(datasets[0])
         self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
 
+        # Get the relevant dataset type
+        datasetType = butler.registry.getDatasetType(self.ingestDatasetTypeName)
+
         for dataId in self.dataIds:
+            # For testing we only read the entire dataset the first time
+            # round if this is an Exposure. If it's not an Exposure
+            # we always read it completely but we don't read components
+            # because for an arbitrary dataset type we can't easily tell
+            # what component to test.
+
+            if not datasetType.storageClass.name.startswith("Exposure"):
+                exposure = butler.get(self.ingestDatasetTypeName, dataId)
+                # Could be anything so nothing to test by default
+                continue
+
             # Check that we can read metadata from a raw
-            metadata = butler.get("raw.metadata", dataId)
+            metadata = butler.get(f"{self.ingestDatasetTypeName}.metadata", dataId)
             if not fullCheck:
                 continue
             fullCheck = False
-            exposure = butler.get("raw", dataId)
+            exposure = butler.get(self.ingestDatasetTypeName, dataId)
+
             self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
 
             # Since components follow a different code path we check that
             # WCS match and also we check that at least the shape
             # of the image is the same (rather than doing per-pixel equality)
-            wcs = butler.get("raw.wcs", dataId)
+            wcs = butler.get(f"{self.ingestDatasetTypeName}.wcs", dataId)
             self.assertEqual(wcs, exposure.getWcs())
 
-            rawImage = butler.get("raw.image", dataId)
+            rawImage = butler.get(f"{self.ingestDatasetTypeName}.image", dataId)
             self.assertEqual(rawImage.getBBox(), exposure.getBBox())
 
             # check that the filter label got the correct band
-            filterLabel = butler.get("raw.filterLabel", dataId)
+            filterLabel = butler.get(f"{self.ingestDatasetTypeName}.filterLabel", dataId)
             self.assertEqual(filterLabel, self.filterLabel)
 
         self.checkRepo(files=files)
@@ -253,9 +274,9 @@ def testDirect(self):
         self._ingestRaws(transfer="direct")
 
         # Check that it really did have a URI outside of datastore
-        srcUri = ButlerURI(self.file)
+        srcUri = ButlerURI(self.file, forceAbsolute=True)
         butler = Butler(self.root, run=self.outputRun)
-        datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
+        datasets = list(butler.registry.queryDatasets(self.ingestDatasetTypeName, collections=self.outputRun))
         datastoreUri = butler.getURI(datasets[0])
         self.assertEqual(datastoreUri, srcUri)
 
@@ -282,11 +303,36 @@ def testInPlace(self):
         """Test that files already in the directory can be added to the
         registry in-place.
         """
-        # symlink into repo root manually
         butler = Butler(self.root, run=self.outputRun)
-        pathInStore = "prefix-" + os.path.basename(self.file)
+
+        # If the test uses an index file the index file needs to also
+        # appear in the datastore root along with the file to be ingested.
+        # In that scenario the file name being used for ingest can not
+        # be modified and must have the same name as found in the index
+        # file itself.
+        source_file_uri = ButlerURI(self.file)
+        index_file = source_file_uri.dirname().join("_index.json")
+        pathInStore = source_file_uri.basename()
+        if index_file.exists():
+            os.symlink(index_file.ospath, butler.datastore.root.join("_index.json").ospath)
+        else:
+            # No index file so we are free to pick any name
+            pathInStore = "prefix-" + pathInStore
+
+        # Create a symlink to the original file so that it looks like it
+        # is now inside the datastore.
         newPath = butler.datastore.root.join(pathInStore)
         os.symlink(os.path.abspath(self.file), newPath.ospath)
+
+        # If there is a sidecar file it needs to be linked in as well
+        # since ingest code does not follow symlinks.
+        sidecar_uri = ButlerURI(source_file_uri).updatedExtension(".json")
+        if sidecar_uri.exists():
+            newSidecar = ButlerURI(newPath).updatedExtension(".json")
+            os.symlink(sidecar_uri.ospath, newSidecar.ospath)
+
+        # Run ingest with auto mode since that should automatically determine
+        # that an in-place ingest is happening.
         self._ingestRaws(transfer="auto", file=newPath.ospath)
         self.verifyIngest()
 
@@ -295,7 +341,7 @@ def testInPlace(self):
         butler = Butler(self.root, run=self.outputRun)
 
         # Check that the URI associated with this path is the right one
-        uri = butler.getURI("raw", self.dataIds[0])
+        uri = butler.getURI(self.ingestDatasetTypeName, self.dataIds[0])
         self.assertEqual(uri.relative_to(butler.datastore.root), pathInStore)
 
     def testFailOnConflict(self):

python/lsst/obs/base/instrument_tests.py

@@ -27,11 +27,81 @@
 
 import abc
 import dataclasses
+import pkg_resources
 
-from lsst.obs.base import Instrument
+from lsst.obs.base import Instrument, FilterDefinitionCollection, FilterDefinition
 from lsst.obs.base.gen2to3 import TranslatorFactory
+from lsst.obs.base.yamlCamera import makeCamera
 from lsst.daf.butler import Registry
 from lsst.daf.butler import RegistryConfig
+from lsst.daf.butler.core.utils import getFullTypeName
+from lsst.daf.butler.formatters.yaml import YamlFormatter
+
+from .utils import createInitialSkyWcsFromBoresight
+
+DUMMY_FILTER_DEFINITIONS = FilterDefinitionCollection(
+    FilterDefinition(physical_filter="dummy_u", band="u", lambdaEff=0),
+    FilterDefinition(physical_filter="dummy_g", band="g", lambdaEff=0),
+)
+
+
+class DummyCamYamlWcsFormatter(YamlFormatter):
+    """Specialist formatter for tests that can make a WCS."""
+
+    @classmethod
+    def makeRawSkyWcsFromBoresight(cls, boresight, orientation, detector):
+        """Class method to make a raw sky WCS from boresight and detector.
+
+        This uses the API expected by define-visits. A working example
+        can be found in `FitsRawFormatterBase`.
+
+        Notes
+        -----
+        This makes no attempt to create a proper WCS from geometry.
+        """
+        return createInitialSkyWcsFromBoresight(boresight, orientation, detector, flipX=False)
+
+
+class DummyCam(Instrument):
+
+    filterDefinitions = DUMMY_FILTER_DEFINITIONS
+
+    @classmethod
+    def getName(cls):
+        return "DummyCam"
+
+    def getCamera(self):
+        # Return something that can be indexed by detector number
+        # but also has to support getIdIter
+        filename = pkg_resources.resource_filename("lsst.obs.base", "test/dummycam.yaml")
+        return makeCamera(filename)
+
+    def register(self, registry):
+        """Insert Instrument, physical_filter, and detector entries into a
+        `Registry`.
+        """
+        detector_max = 2
+        dataId = {"instrument": self.getName(), "class_name": getFullTypeName(DummyCam),
+                  "detector_max": detector_max}
+        with registry.transaction():
+            registry.syncDimensionData("instrument", dataId)
+            self._registerFilters(registry)
+            for d in range(detector_max):
+                registry.syncDimensionData("detector",
+                                           dict(dataId, id=d, full_name=f"RXX_S0{d}"))
+
+    def getRawFormatter(self, dataId):
+        # Docstring inherited fromt Instrument.getRawFormatter.
+        return DummyCamYamlWcsFormatter
+
+    def writeCuratedCalibrations(self, butler):
+        pass
+
+    def applyConfigOverrides(self, name, config):
+        pass
+
+    def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
+        return TranslatorFactory()
 
 
 @dataclasses.dataclass

python/lsst/obs/base/test/dummycam.yaml

@@ -0,0 +1,114 @@
+# This file is part of obs_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+---
+
+# Simplest possible single chip camera used for obs_base testing.
+
+name : DummyCam
+
+# A single amplifier ("segment" to the camera team)
+AMP : &AMP
+    perAmpData : True
+    # trimmed
+    dataExtent : [512, 2002]
+    readCorner : LL
+    #                         [[x0,  y0], [xsize, ysize]]
+    rawBBox                 : [[0,    0], [576,    2048]]  # total size of one amp's raw data
+
+    rawDataBBox             : [[10,   0], [512,    2002]]  # data region in raw data
+    rawSerialPrescanBBox    : [[0,    0], [10,     2002]]  # serial prescan (often == extended)
+    rawSerialOverscanBBox   : [[522,  0], [54,     2002]]  # serial overscan
+    rawParallelPrescanBBox  : [[0,    1], [0,         0]]  # pixels digitised before first parallel transfer
+    rawParallelOverscanBBox : [[10, 2002], [512,     46]]  # parallel overscan
+
+    saturation : 142857
+
+    # Linearity correction
+    linearityType : PROPORTIONAL
+    linearityThreshold : 0
+    linearityMax : 142857                # == saturation
+    linearityCoeffs : [0, 142857]        # == [linearityThreshold, linearityMax]
+
+# A single CCD
+CCD : &CCD
+    detectorType : 0
+    refpos : [2047.5, 2001.5]
+    offset : [.nan, .nan]
+    # This is the orientation we need to put the serial direction along the x-axis
+    bbox : [[0, 0], [4095, 4003]]
+    pixelSize : [0.010, 0.010]          # in mm
+    transformDict : {nativeSys : 'Pixels', transforms : None}
+    transposeDetector : False
+    pitch : 0.0                         # (degrees)
+    yaw : 0.0                           # rotation in plane of camera (degrees)
+    roll : 0.0                          # (degrees)
+
+    amplifiers:
+      C00: &C00
+        <<: *AMP
+        hdu : 1
+        flipXY : [False, False]
+        ixy : [0, 0]
+
+
+# Layout of CCDs within an raft
+RAFT:
+  ccds :
+    S00: &S00                      # Sensor (0, 0)
+      << : *CCD
+      id : 0
+      offset : [0.0, 0.0]
+
+# Specify the geometrical transformations relevant to the camera in all appropriate
+# (and known!) coordinate systems
+plateScale : 9.5695
+transforms : {'nativeSys': 'FocalPlane', 'FieldAngle': {'transformType': 'radial', 'coeffs': [0.0, 1.0, 0.0]}}
+
+# Define our specific devices
+#
+# All the CCDs present in this file
+CCDs :
+  RXX_S00 :
+    << : *S00
+    id : 0
+    serial : DummyCam-1
+    physicalType : FAKE
+    refpos : [2036.5, 2000.5]
+    offset : [-0.515, -0.055]
+    amplifiers :
+      C00 :
+        << : *C00
+        gain : 2.42
+        readNoise : 25.14
+  RXX_S01 :
+    << : *S00
+    id : 1
+    serial : DummyCam-2
+    physicalType : FAKE
+    refpos : [20.5, 20.5]
+    offset : [-00, -0.0]
+    amplifiers :
+      C00 :
+        << : *C00
+        gain : 2.42
+        readNoise : 25.14

tests/data/calexp.json

@@ -0,0 +1 @@
+{"telescope": "Subaru", "instrument": "HSC", "location": [-5464492.282102363, -2493000.626631501, 2150943.6513960036], "exposure_id": 903334, "visit_id": 903334, "physical_filter": "HSC-R", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "exposure_time": 30.0, "dark_time": 30.0, "boresight_airmass": 1.072389930512202, "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "detector_num": 23, "detector_name": "26", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "object": "STRIPE82L", "temperature": 273.75, "pressure": 621.3000000000001, "relative_humidity": 72.5, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558], "science_program": "o13015", "observation_type": "science", "observation_id": "HSCA90333400", "observation_reason": "science", "exposure_group": "903334", "observing_day": 20130617, "observation_counter": 903334, "__CONTENT__": "translated"}

tests/data/ingest/indexed_data/_index.json

@@ -0,0 +1 @@
+{"__CONTENT__": "translated", "__COMMON__": {"telescope": "Pretend", "instrument": "DummyCam", "location": [-5464492.282102363, -2493000.626631501, 2150943.6513960036], "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "object": "STRIPE82L", "exposure_time": 30.0, "dark_time": 30.0, "science_program": "o13015", "observation_type": "science", "relative_humidity": 72.5, "observation_reason": "science", "observing_day": 20130617, "temperature": 273.75, "pressure": 621.3000000000001}, "dataset_1.yaml": {"exposure_id": 100, "visit_id": 100, "physical_filter": "dummy_u", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "boresight_airmass": 1.072389930512202,  "detector_num": 0, "detector_name": "RXX_S00", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558],  "observation_id": "DummyDataset_1", "observation_reason": "science", "exposure_group": "1", "observation_counter": 1}, "dataset_2.yaml": {"exposure_id": 200, "visit_id": 200, "physical_filter": "dummy_g", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "boresight_airmass": 1.072389930512202, "detector_num": 1, "detector_name": "RXX_S01", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558], "observation_id": "DummyDataset_2", "exposure_group": "2", "observation_counter": 2}}

tests/data/ingest/indexed_data/bad_implied/_index.json

@@ -0,0 +1 @@
+bad index file content

tests/data/ingest/indexed_data/bad_implied/dataset_2.yaml

@@ -0,0 +1,2 @@
+value: 2
+name: "dataset_2"

tests/data/ingest/indexed_data/bad_index/_index.json

@@ -0,0 +1 @@
+bad index file content

tests/data/ingest/indexed_data/bad_instrument/_index.json

@@ -0,0 +1 @@
+{"__CONTENT__": "translated", "__COMMON__": {"telescope": "Pretend", "instrument": "HSC", "location": [-5464492.282102363, -2493000.626631501, 2150943.6513960036], "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "object": "STRIPE82L", "exposure_time": 30.0, "dark_time": 30.0, "science_program": "o13015", "observation_type": "science", "relative_humidity": 72.5, "observation_reason": "science", "observing_day": 20130617, "temperature": 273.75, "pressure": 621.3000000000001}, "../dataset_1.yaml": {"exposure_id": 100, "visit_id": 100, "physical_filter": "dummy_u", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "boresight_airmass": 1.072389930512202,  "detector_num": 0, "detector_name": "RXX_S00", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558],  "observation_id": "DummyDataset_1", "observation_reason": "science", "exposure_group": "1", "observation_counter": 1}, "../dataset_2.yaml": {"exposure_id": 200, "visit_id": 200, "physical_filter": "dummy_g", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "boresight_airmass": 1.072389930512202, "detector_num": 1, "detector_name": "RXX_S01", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558], "observation_id": "DummyDataset_2", "exposure_group": "2", "observation_counter": 2}}

tests/data/ingest/indexed_data/dataset_1.yaml

@@ -0,0 +1,2 @@
+value: 1
+name: "dataset_1"

tests/data/ingest/indexed_data/dataset_2.yaml

@@ -0,0 +1,2 @@
+value: 2
+name: "dataset_2"

tests/data/ingest/indexed_data/dataset_3.json

@@ -0,0 +1 @@
+{"telescope": "Pretend", "instrument": "DummyCam", "location": [-5464492.282102363, -2493000.626631501, 2150943.6513960036], "exposure_id": 400, "visit_id": 400, "physical_filter": "dummy_g", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "exposure_time": 30.0, "dark_time": 30.0, "boresight_airmass": 1.072389930512202, "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "detector_num": 1, "detector_name": "RXX_S01", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "object": "STRIPE82L", "temperature": 273.75, "pressure": 621.3000000000001, "relative_humidity": 72.5, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558], "science_program": "o13015", "observation_type": "science", "observation_id": "DummyDataset_3", "observation_reason": "science", "exposure_group": "2", "observing_day": 20130617, "observation_counter": 2, "__CONTENT__": "translated"}

tests/data/ingest/indexed_data/dataset_3.yaml

@@ -0,0 +1,2 @@
+value: 3
+name: "dataset_3"

tests/data/ingest/indexed_data/dataset_4.json

@@ -0,0 +1 @@
+{"telescope": "Pretend", "instrument": "DummyCam", "location": [-5464492.282102363, -2493000.626631501, 2150943.6513960036], "exposure_id": 600, "visit_id": 600, "physical_filter": "dummy_g", "datetime_begin": [2456461.0, 0.06193332175925925], "datetime_end": [2456461.0, 0.06230896990740742], "exposure_time": 30.0, "dark_time": 30.0, "boresight_airmass": 1.072389930512202, "boresight_rotation_angle": 270.0, "boresight_rotation_coord": "sky", "detector_num": 1, "detector_name": "RXX_S01", "detector_unique_name": "0_26", "detector_serial": "076", "detector_group": "0", "detector_exposure_id": 180666823, "object": "STRIPE82L", "temperature": 273.75, "pressure": 621.3000000000001, "relative_humidity": 72.5, "tracking_radec": [320.2499333333333, 1.9444444444444445e-05], "altaz_begin": [157.95888945, 68.80710558], "science_program": "o13015", "observation_type": "science", "observation_id": "DummyDataset_3", "observation_reason": "science", "exposure_group": "2", "observing_day": 20130617, "observation_counter": 2, "__CONTENT__": "translated"}

tests/data/ingest/indexed_data/dataset_4.yaml

@@ -0,0 +1,2 @@
+value: 4
+name: "dataset_4"