Merge branch 'tickets/DM-24923'

python/lsst/obs/base/cli/butler_cmd_test.py

@@ -29,36 +29,41 @@
 import click.testing
 
 from lsst.daf.butler.cli import butler
+from lsst.utils import doImport
 
 
 class ButlerCmdTestBase(metaclass=abc.ABCMeta):
     """Base class for tests of butler command line interface subcommands.
     Subclass from this, then `unittest.TestCase` to get a working test suite.
     """
 
-    @staticmethod
+    @property
     @abc.abstractmethod
-    def instrumentClass():
-        """Get the fully qualified instrument class.
+    def instrumentClassName(self):
+        """The fully qualified instrument class name.
 
         Returns
         -------
         `str`
-            The fully qualified instrument class.
+            The fully qualified instrument class name.
         """
         pass
 
-    @staticmethod
-    @abc.abstractmethod
-    def instrumentName():
-        """Get the instrument name.
+    @property
+    def instrument(self):
+        """The instrument class."""
+        return doImport(self.instrumentClassName)
+
+    @property
+    def instrumentName(self):
+        """The name of the instrument.
 
         Returns
         -------
         `str`
             The name of the instrument.
         """
-        pass
+        return self.instrument.getName()
 
     def test_cli(self):
         runner = click.testing.CliRunner()
@@ -67,10 +72,10 @@ def test_cli(self):
             self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
             result = runner.invoke(butler.cli, ["register-instrument",
                                                 "here",
-                                                "-i", self.instrumentClass()])
+                                                "-i", self.instrumentClassName])
             self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
             result = runner.invoke(butler.cli, ["write-curated-calibrations",
                                                 "here",
-                                                "-i", self.instrumentName(),
+                                                "-i", self.instrumentName,
                                                 "--output-run", "output_run"])
             self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")

python/lsst/obs/base/ingest_tests.py

@@ -33,6 +33,7 @@
 from lsst.daf.butler import Butler
 from lsst.daf.butler.script import createRepo
 import lsst.obs.base
+from lsst.utils import doImport
 from .utils import getInstrument
 from .script import ingestRaws, registerInstrument, writeCuratedCalibrations
 
@@ -47,57 +48,72 @@ class IngestTestBase(metaclass=abc.ABCMeta):
     actual directory will be a tempdir under this one.
     """
 
-    instrument = None
-    """The instrument to be registered and tested."""
-
     dataIds = []
     """list of butler data IDs of files that should have been ingested."""
 
     file = ""
     """Full path to a file to ingest in tests."""
 
-    RawIngestTask = "lsst.obs.base.RawIngestTask"
+    rawIngestTask = "lsst.obs.base.RawIngestTask"
     """The task to use in the Ingest test."""
 
     curatedCalibrationDatasetTypes = None
     """List or tuple of Datasets types that should be present after calling
     writeCuratedCalibrations. If `None` writeCuratedCalibrations will
     not be called and the test will be skipped."""
 
-    DefineVisitsTask = lsst.obs.base.DefineVisitsTask
+    defineVisitsTask = lsst.obs.base.DefineVisitsTask
     """The task to use to define visits from groups of exposures.
-
     This is ignored if ``visits`` is `None`.
     """
 
     visits = {}
     """A dictionary mapping visit data IDs the lists of exposure data IDs that
     are associated with them.
-
     If this is empty (but not `None`), visit definition will be run but no
     visits will be expected (e.g. because no exposures are on-sky
     observations).
     """
 
-    instrument = ""
-    """The fully qualified name of the instrument.
-    """
-
-    instrumentName = ""
-    """The name of the instrument.
-    """
-
     outputRun = "raw"
     """The name of the output run to use in tests.
     """
 
+    @property
+    @abc.abstractmethod
+    def instrumentClassName(self):
+        """The fully qualified instrument class name.
+
+        Returns
+        -------
+        `str`
+            The fully qualified instrument class name.
+        """
+        pass
+
+    @property
+    def instrumentClass(self):
+        """The instrument class."""
+        return doImport(self.instrumentClassName)
+
+    @property
+    def instrumentName(self):
+        """The name of the instrument.
+
+        Returns
+        -------
+        `str`
+            The name of the instrument.
+        """
+        return self.instrumentClass.getName()
+
     def setUp(self):
         # Use a temporary working directory
         self.root = tempfile.mkdtemp(dir=self.ingestDir)
         createRepo(self.root)
 
         # Register the instrument and its static metadata
-        registerInstrument(self.root, self.instrument)
+        registerInstrument(self.root, self.instrumentClassName)
 
     def tearDown(self):
         if os.path.exists(self.root):
@@ -145,22 +161,24 @@ def checkRepo(self, files=None):
         pass
 
     def testLink(self):
-        ingestRaws(self.root, self.outputRun, file=self.file, transfer="link", ingest_task=self.RawIngestTask)
+        ingestRaws(self.root, self.outputRun, file=self.file, transfer="link",
+                   ingest_task=self.rawIngestTask)
         self.verifyIngest()
 
     def testSymLink(self):
         ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
-                   ingest_task=self.RawIngestTask)
+                   ingest_task=self.rawIngestTask)
         self.verifyIngest()
 
     def testCopy(self):
-        ingestRaws(self.root, self.outputRun, file=self.file, transfer="copy", ingest_task=self.RawIngestTask)
+        ingestRaws(self.root, self.outputRun, file=self.file, transfer="copy",
+                   ingest_task=self.rawIngestTask)
         self.verifyIngest()
 
     def testHardLink(self):
         try:
             ingestRaws(self.root, self.outputRun, file=self.file, transfer="hardlink",
-                       ingest_task=self.RawIngestTask)
+                       ingest_task=self.rawIngestTask)
             self.verifyIngest()
         except PermissionError as err:
             raise unittest.SkipTest("Skipping hard-link test because input data"
@@ -174,17 +192,17 @@ def testInPlace(self):
         butler = Butler(self.root, run=self.outputRun)
         newPath = os.path.join(butler.datastore.root, os.path.basename(self.file))
         os.symlink(os.path.abspath(self.file), newPath)
-        ingestRaws(self.root, self.outputRun, file=newPath, transfer=None, ingest_task=self.RawIngestTask)
+        ingestRaws(self.root, self.outputRun, file=newPath, transfer=None, ingest_task=self.rawIngestTask)
         self.verifyIngest()
 
     def testFailOnConflict(self):
         """Re-ingesting the same data into the repository should fail.
         """
         ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
-                   ingest_task=self.RawIngestTask)
+                   ingest_task=self.rawIngestTask)
         with self.assertRaises(Exception):
             ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
-                       ingest_task=self.RawIngestTask)
+                       ingest_task=self.rawIngestTask)
 
     def testWriteCuratedCalibrations(self):
         """Test that we can ingest the curated calibrations"""
@@ -204,19 +222,20 @@ def testWriteCuratedCalibrations(self):
     def testDefineVisits(self):
         if self.visits is None:
             self.skipTest("Expected visits were not defined.")
-        ingestRaws(self.root, self.outputRun, file=self.file, transfer="link", ingest_task=self.RawIngestTask)
+        ingestRaws(self.root, self.outputRun, file=self.file, transfer="link",
+                   ingest_task=self.rawIngestTask)
 
-        config = self.DefineVisitsTask.ConfigClass()
+        config = self.defineVisitsTask.ConfigClass()
         butler = Butler(self.root, run=self.outputRun)
-        instrument = getInstrument(self.instrumentName, butler.registry)
-        instrument.applyConfigOverrides(self.DefineVisitsTask._DefaultName, config)
-        task = self.DefineVisitsTask(config=config, butler=butler)
+        instr = getInstrument(self.instrumentName, butler.registry)
+        instr.applyConfigOverrides(self.defineVisitsTask._DefaultName, config)
+        task = self.defineVisitsTask(config=config, butler=butler)
         task.run(self.dataIds)
 
         # Test that we got the visits we expected.
         visits = set(butler.registry.queryDimensions(["visit"], expand=True))
         self.assertCountEqual(visits, self.visits.keys())
-        camera = instrument.getCamera()
+        camera = instr.getCamera()
         for foundVisit, (expectedVisit, expectedExposures) in zip(visits, self.visits.items()):
             # Test that this visit is associated with the expected exposures.
             foundExposures = set(butler.registry.queryDimensions(["exposure"], dataId=expectedVisit,

python/lsst/obs/base/script/ingestRaws.py

@@ -68,6 +68,7 @@ def ingestRaws(repo, output_run, config=None, config_file=None, directory=None,
     ingester = TaskClass(config=ingestConfig, butler=butler)
     files = [file] if file is not None else []
     if directory is not None:
+        suffixes = (".fits", ".fits.gz", ".fits.fz")
         files.extend(
-            [os.path.join(directory, f) for f in os.listdir(directory) if f.lower().endswith("fits")])
+            [os.path.join(directory, f) for f in os.listdir(directory) if f.lower().endswith(suffixes)])
     ingester.run(files)