Merge pull request #403 from lsst/tickets/DM-32817

DM-32817: Switch from ButlerURI to ResourcePath

python/lsst/obs/base/gen2to3/convertRepo.py

@@ -29,9 +29,10 @@
 from typing import Iterable, List, Optional, Tuple
 
 from lsst.daf.butler import Butler as Butler3
-from lsst.daf.butler import ButlerURI, CollectionType, SkyPixDimension
+from lsst.daf.butler import CollectionType, SkyPixDimension
 from lsst.pex.config import Config, ConfigDictField, ConfigurableField, DictField, Field, ListField
 from lsst.pipe.base import Task
+from lsst.resources import ResourcePath
 from lsst.skymap import BaseSkyMap, skyMapRegistry
 
 from .._instrument import Instrument
@@ -134,8 +135,8 @@ def guessCollectionNames(self, instrument: Instrument, root: str) -> None:
             return
         if self.chainName is None:
             if os.path.isabs(self.path):
-                rerunURI = ButlerURI(self.path)
-                rootURI = ButlerURI(root)
+                rerunURI = ResourcePath(self.path)
+                rootURI = ResourcePath(root)
                 chainName = rerunURI.relative_to(rootURI)
                 if chainName is None:
                     raise ValueError(

python/lsst/obs/base/ingest.py

@@ -34,7 +34,6 @@
 from lsst.afw.fits import readMetadata
 from lsst.daf.butler import (
     Butler,
-    ButlerURI,
     CollectionType,
     DataCoordinate,
     DatasetIdGenEnum,
@@ -48,6 +47,7 @@
 )
 from lsst.pex.config import ChoiceField, Config, Field
 from lsst.pipe.base import Task
+from lsst.resources import ResourcePath
 from lsst.utils.timer import timeMethod
 
 from ._fitsRawFormatterBase import FitsRawFormatterBase
@@ -117,7 +117,7 @@ class RawFileData:
     (`list` of `RawFileDatasetInfo`)
     """
 
-    filename: ButlerURI
+    filename: ResourcePath
     """URI of the file this information was extracted from (`str`).
 
     This is the path prior to ingest, not the path after ingest.
@@ -301,7 +301,7 @@ def _determine_instrument_formatter(self, dataId, filename):
         ----------
         dataId : `lsst.daf.butler.DataCoordinate`
             The dataId associated with this dataset.
-        filename : `ButlerURI`
+        filename : `lsst.resources.ResourcePath`
             URI of file used for error reporting.
 
         Returns
@@ -333,12 +333,12 @@ def _determine_instrument_formatter(self, dataId, filename):
             FormatterClass = instrument.getRawFormatter(dataId)
         return instrument, FormatterClass
 
-    def extractMetadata(self, filename: ButlerURI) -> RawFileData:
+    def extractMetadata(self, filename: ResourcePath) -> RawFileData:
         """Extract and process metadata from a single raw file.
 
         Parameters
         ----------
-        filename : `ButlerURI`
+        filename : `lsst.resources.ResourcePath`
             URI to the file.
 
         Returns
@@ -435,7 +435,7 @@ def _calculate_dataset_info(self, header, filename):
         ----------
         header : Mapping or `astro_metadata_translator.ObservationInfo`
             Header from the dataset or previously-translated content.
-        filename : `ButlerURI`
+        filename : `lsst.resources.ResourcePath`
             Filename to use for error messages.
 
         Returns
@@ -516,7 +516,7 @@ def locateAndReadIndexFiles(self, files):
 
         Parameters
         ----------
-        files : iterable over `ButlerURI`
+        files : iterable over `lsst.resources.ResourcePath`
             URIs to the files to be ingested.
 
         Returns
@@ -949,7 +949,7 @@ def ingestFiles(
 
         Parameters
         ----------
-        files : iterable over `ButlerURI`
+        files : iterable over `lsst.resources.ResourcePath`
             URIs to the files to be ingested.
         pool : `multiprocessing.Pool`, optional
             If not `None`, a process pool with which to parallelize some
@@ -1099,7 +1099,7 @@ def run(
 
         Parameters
         ----------
-        files : iterable over `ButlerURI`, `str` or path-like objects
+        files : iterable `lsst.resources.ResourcePath`, `str` or path-like
             Paths to the files to be ingested.  Can refer to directories.
             Will be made absolute if they are not already.
         pool : `multiprocessing.Pool`, optional
@@ -1157,7 +1157,7 @@ def run(
         n_exposures_failed = 0
         n_ingests_failed = 0
         if group_files:
-            for group in ButlerURI.findFileResources(files, file_filter, group_files):
+            for group in ResourcePath.findFileResources(files, file_filter, group_files):
                 new_refs, bad, n_exp, n_exp_fail, n_ingest_fail = self.ingestFiles(
                     group,
                     pool=pool,
@@ -1173,7 +1173,7 @@ def run(
                 n_ingests_failed += n_ingest_fail
         else:
             refs, bad_files, n_exposures, n_exposures_failed, n_ingests_failed = self.ingestFiles(
-                ButlerURI.findFileResources(files, file_filter, group_files),
+                ResourcePath.findFileResources(files, file_filter, group_files),
                 pool=pool,
                 processes=processes,
                 run=run,

python/lsst/obs/base/ingest_tests.py

@@ -32,9 +32,10 @@
 
 import lsst.afw.cameraGeom
 import lsst.obs.base
-from lsst.daf.butler import Butler, ButlerURI
+from lsst.daf.butler import Butler
 from lsst.daf.butler.cli.butler import cli as butlerCli
 from lsst.daf.butler.cli.utils import LogCliRunner
+from lsst.resources import ResourcePath
 from lsst.utils import doImport
 
 from . import script
@@ -298,7 +299,7 @@ def testDirect(self):
         self._ingestRaws(transfer="direct")
 
         # Check that it really did have a URI outside of datastore.
-        srcUri = ButlerURI(self.file, forceAbsolute=True)
+        srcUri = ResourcePath(self.file, forceAbsolute=True)
         butler = Butler(self.root, run=self.outputRun)
         datasets = list(butler.registry.queryDatasets(self.ingestDatasetTypeName, collections=self.outputRun))
         datastoreUri = butler.getURI(datasets[0])
@@ -335,7 +336,7 @@ def testInPlace(self):
         # In that scenario the file name being used for ingest can not
         # be modified and must have the same name as found in the index
         # file itself.
-        source_file_uri = ButlerURI(self.file)
+        source_file_uri = ResourcePath(self.file)
         index_file = source_file_uri.dirname().join("_index.json")
         pathInStore = source_file_uri.basename()
         if index_file.exists():
@@ -351,9 +352,9 @@ def testInPlace(self):
 
         # If there is a sidecar file it needs to be linked in as well
         # since ingest code does not follow symlinks.
-        sidecar_uri = ButlerURI(source_file_uri).updatedExtension(".json")
+        sidecar_uri = ResourcePath(source_file_uri).updatedExtension(".json")
         if sidecar_uri.exists():
-            newSidecar = ButlerURI(newPath).updatedExtension(".json")
+            newSidecar = ResourcePath(newPath).updatedExtension(".json")
             os.symlink(sidecar_uri.ospath, newSidecar.ospath)
 
         # Run ingest with auto mode since that should automatically determine

tests/test_fitsRawFormatter.py

@@ -27,6 +27,7 @@
 import lsst.daf.base
 import lsst.daf.butler
 import lsst.geom
+import lsst.resources
 import lsst.utils.tests
 from astro_metadata_translator import FitsTranslator, StubTranslator
 from astro_metadata_translator.translators.helpers import tracking_from_degree_headers
@@ -215,7 +216,7 @@ def test_amp_parameter(self):
                             # and has the right parameters.
                             formatter = SimpleFitsRawFormatter(
                                 lsst.daf.butler.FileDescriptor(
-                                    lsst.daf.butler.Location(None, path=lsst.daf.butler.ButlerURI(tmpFile)),
+                                    lsst.daf.butler.Location(None, path=lsst.resources.ResourcePath(tmpFile)),
                                     lsst.daf.butler.StorageClassFactory().getStorageClass("ExposureI"),
                                     parameters=parameters,
                                 ),

ups/obs_base.table

@@ -8,6 +8,7 @@ setupRequired(geom)
 setupRequired(daf_butler)
 setupRequired(pipe_base)
 setupOptional(skymap)
+setupRequired(resources)
 
 envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python)
 envPrepend(PATH, ${PRODUCT_DIR}/bin)