DM-27033: Integration branch for schema changes from Butler

python/lsst/obs/base/_instrument.py

@@ -732,7 +732,7 @@ def makeExposureRecordFromObsInfo(obsInfo, universe):
     return dimension.RecordClass(
         instrument=obsInfo.instrument,
         id=obsInfo.exposure_id,
-        name=obsInfo.observation_id,
+        obs_id=obsInfo.observation_id,
         group_name=obsInfo.exposure_group,
         group_id=obsInfo.visit_id,
         datetime_begin=obsInfo.datetime_begin,
@@ -742,6 +742,8 @@ def makeExposureRecordFromObsInfo(obsInfo, universe):
         dark_time=obsInfo.dark_time.to_value("s") if obsInfo.dark_time is not None else None,
         observation_type=obsInfo.observation_type,
         observation_reason=obsInfo.observation_reason,
+        day_obs=obsInfo.observing_day,
+        seq_num=obsInfo.observation_counter,
         physical_filter=obsInfo.physical_filter,
         science_program=obsInfo.science_program,
         target_name=obsInfo.object,

python/lsst/obs/base/defineVisits.py

@@ -372,6 +372,10 @@ def _buildVisitRecords(self, definition: VisitDefinitionData, *,
                                     (e.target_name for e in definition.exposures))
         science_program = _reduceOrNone(lambda a, b: a if a == b else None,
                                         (e.science_program for e in definition.exposures))
+
+        # observing day for a visit is defined by the earliest observation
+        # of the visit
+        observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures))
         observation_reason = _reduceOrNone(lambda a, b: a if a == b else None,
                                            (e.observation_reason for e in definition.exposures))
         if observation_reason is None:
@@ -393,6 +397,7 @@ def _buildVisitRecords(self, definition: VisitDefinitionData, *,
                 target_name=target_name,
                 science_program=science_program,
                 observation_reason=observation_reason,
+                day_obs=observing_day,
                 zenith_angle=zenith_angle,
                 visit_system=self.groupExposures.getVisitSystem()[0],
                 exposure_time=exposure_time,
@@ -594,7 +599,7 @@ def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionDat
             yield VisitDefinitionData(
                 instrument=exposure.instrument,
                 id=exposure.id,
-                name=exposure.name,
+                name=exposure.obs_id,
                 exposures=[exposure],
             )
 

python/lsst/obs/base/gen2to3/rootRepoConverter.py

@@ -120,7 +120,7 @@ def runRawIngest(self, pool=None):
         dataPaths = getDataPaths(dataRefs)
         self.task.log.info("Ingesting raws from root %s into run %s.", self.root, self.task.raws.butler.run)
         self._rawRefs.extend(self.task.raws.run(dataPaths, pool=pool))
-        self._chain = {self.task.raws.butler.run: {self.task.raws.datasetType.name}}
+        self._chain = [self.task.raws.butler.run]
 
     def runDefineVisits(self, pool=None):
         if self.task.defineVisits is None:

python/lsst/obs/base/gen2to3/standardRepoConverter.py

@@ -24,7 +24,7 @@
 
 import os.path
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Tuple
 
 from lsst.log import Log
 from lsst.log.utils import temporaryLogLevel
@@ -91,7 +91,7 @@ def __init__(self, **kwds):
                 self.butler2 = Butler2(self.root)
                 self.mapper = self.butler2.getMapperClass(self.root)(root=self.root)
         self._foundSkyMapsByCoaddName = {}
-        self._chain = {}
+        self._chain = []
 
     def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
         # Docstring inherited from RepoConverter.
@@ -202,15 +202,14 @@ def getRun(self, datasetTypeName: str, calibDate: Optional[str] = None) -> str:
         if run is None:
             raise ValueError(f"No default run for repo at {self.root}, and no "
                              f"override for dataset {datasetTypeName}.")
-        self._chain.setdefault(run, set()).add(datasetTypeName)
+        self._chain.append(run)
         return run
 
-    def getCollectionChain(self) -> List[Tuple[str, Set[str]]]:
-        """Return tuples of run name and associated dataset type names that
-        can be used to construct a chained collection that refers to the
-        converted repository (`list` [ `tuple` ]).
+    def getCollectionChain(self) -> List[str]:
+        """Return run names that can be used to construct a chained collection
+        that refers to the converted repository (`list` [ `str` ]).
         """
-        return list(self._chain.items())
+        return self._chain
 
     # Class attributes that will be shadowed by public instance attributes;
     # defined here only for documentation purposes.

python/lsst/obs/base/ingest.py

@@ -536,14 +536,14 @@ def run(self, files, *, pool: Optional[Pool] = None, processes: int = 1, run: Op
 
             self.log.debug("Attempting to ingest %d file%s from exposure %s:%s",
                            len(exposure.files), "" if len(exposure.files) == 1 else "s",
-                           exposure.record.instrument, exposure.record.name)
+                           exposure.record.instrument, exposure.record.obs_id)
 
             try:
                 self.butler.registry.syncDimensionData("exposure", exposure.record)
             except Exception as e:
                 n_exposures_failed += 1
                 self.log.warning("Exposure %s:%s could not be registered: %s",
-                                 exposure.record.instrument, exposure.record.name, e)
+                                 exposure.record.instrument, exposure.record.obs_id, e)
                 continue
 
             # Override default run if nothing specified explicitly
@@ -568,7 +568,7 @@ def run(self, files, *, pool: Optional[Pool] = None, processes: int = 1, run: Op
             # Success for this exposure
             n_exposures += 1
             self.log.info("Exposure %s:%s ingested successfully",
-                          exposure.record.instrument, exposure.record.name)
+                          exposure.record.instrument, exposure.record.obs_id)
 
         had_failure = False
 

python/lsst/obs/base/instrument_tests.py

@@ -31,7 +31,7 @@
 from lsst.obs.base import Instrument
 from lsst.obs.base.gen2to3 import TranslatorFactory
 from lsst.daf.butler import Registry
-from lsst.daf.butler import ButlerConfig
+from lsst.daf.butler import RegistryConfig
 
 
 @dataclasses.dataclass
@@ -79,7 +79,9 @@ def test_getCamera(self):
     def test_register(self):
         """Test that register() sets appropriate Dimensions.
         """
-        registry = Registry.fromConfig(ButlerConfig())
+        registryConfig = RegistryConfig()
+        registryConfig["db"] = "sqlite://"
+        registry = Registry.createFromConfig(registryConfig)
         # check that the registry starts out empty
         self.assertFalse(registry.queryDataIds(["instrument"]).toSequence())
         self.assertFalse(registry.queryDataIds(["detector"]).toSequence())