DM-32817: Switch from ButlerURI to ResourcePath

.github/workflows/mypy.yaml

@@ -26,7 +26,8 @@ jobs:
         run: pip install types-requests types-PyYAML types-click types-pkg_resources
 
       - name: Install lsst.utils
-        run: pip install git+git://github.com/lsst/utils@main#egg=lsst_utils
+        run: |
+          pip install -r requirements.txt
 
       - name: Fake a version
         run: |

python/lsst/daf/butler/_butler.py

@@ -56,11 +56,7 @@
     Union,
 )
 
-try:
-    import boto3
-except ImportError:
-    boto3 = None
-
+from lsst.resources import ResourcePath, ResourcePathExpression
 from lsst.utils import doImportType
 from lsst.utils.introspection import get_class_of
 from lsst.utils.logging import VERBOSE, getLogger
@@ -70,7 +66,6 @@
 from ._deferredDatasetHandle import DeferredDatasetHandle
 from .core import (
     AmbiguousDatasetError,
-    ButlerURI,
     Config,
     ConfigSubset,
     DataCoordinate,
@@ -295,7 +290,7 @@ def __init__(
     """
 
     @classmethod
-    def get_repo_uri(cls, label: str) -> ButlerURI:
+    def get_repo_uri(cls, label: str) -> ResourcePath:
         """Look up the label in a butler repository index.
 
         Parameters
@@ -305,7 +300,7 @@ def get_repo_uri(cls, label: str) -> ButlerURI:
 
         Returns
         -------
-        uri : `ButlerURI`
+        uri : `lsst.resources.ResourcePath`
             URI to the Butler repository associated with the given label.
 
         Raises
@@ -339,21 +334,21 @@ def get_known_repos(cls) -> Set[str]:
 
     @staticmethod
     def makeRepo(
-        root: str,
+        root: ResourcePathExpression,
         config: Union[Config, str, None] = None,
         dimensionConfig: Union[Config, str, None] = None,
         standalone: bool = False,
         searchPaths: Optional[List[str]] = None,
         forceConfigRoot: bool = True,
-        outfile: Optional[str] = None,
+        outfile: Optional[ResourcePathExpression] = None,
         overwrite: bool = False,
     ) -> Config:
         """Create an empty data repository by adding a butler.yaml config
         to a repository root directory.
 
         Parameters
         ----------
-        root : `str` or `ButlerURI`
+        root : `lsst.resources.ResourcePathExpression`
             Path or URI to the root location of the new repository. Will be
             created if it does not exist.
         config : `Config` or `str`, optional
@@ -384,7 +379,7 @@ def makeRepo(
             of the root directory for a datastore or registry to be given.
             If this parameter is `True` the values for ``root`` will be
             forced into the resulting config if appropriate.
-        outfile : `str`, optional
+        outfile : `lss.resources.ResourcePathExpression`, optional
             If not-`None`, the output configuration will be written to this
             location rather than into the repository itself. Can be a URI
             string.  Can refer to a directory that will be used to write
@@ -422,7 +417,7 @@ def makeRepo(
             raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
 
         # Ensure that the root of the repository exists or can be made
-        uri = ButlerURI(root, forceDirectory=True)
+        uri = ResourcePath(root, forceDirectory=True)
         uri.mkdir()
 
         config = Config(config)
@@ -462,7 +457,7 @@ def makeRepo(
             # branch, _everything_ in the config is expanded, so there's no
             # need to special case this.
             Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
-        configURI: Union[str, ButlerURI]
+        configURI: Union[str, ResourcePathExpression]
         if outfile is not None:
             # When writing to a separate location we must include
             # the root of the butler repo in the config else it won't know
@@ -1291,7 +1286,7 @@ def getURIs(
         collections: Any = None,
         run: Optional[str] = None,
         **kwargs: Any,
-    ) -> Tuple[Optional[ButlerURI], Dict[str, ButlerURI]]:
+    ) -> Tuple[Optional[ResourcePath], Dict[str, ResourcePath]]:
         """Returns the URIs associated with the dataset.
 
         Parameters
@@ -1319,7 +1314,7 @@ def getURIs(
 
         Returns
         -------
-        primary : `ButlerURI`
+        primary : `lsst.resources.ResourcePath`
             The URI to the primary artifact associated with this dataset.
             If the dataset was disassembled within the datastore this
             may be `None`.
@@ -1349,7 +1344,7 @@ def getURI(
         collections: Any = None,
         run: Optional[str] = None,
         **kwargs: Any,
-    ) -> ButlerURI:
+    ) -> ResourcePath:
         """Return the URI to the Dataset.
 
         Parameters
@@ -1377,7 +1372,7 @@ def getURI(
 
         Returns
         -------
-        uri : `ButlerURI`
+        uri : `lsst.resources.ResourcePath`
             URI pointing to the Dataset within the datastore. If the
             Dataset does not exist in the datastore, and if ``predict`` is
             `True`, the URI will be a prediction and will include a URI
@@ -1414,11 +1409,11 @@ def getURI(
     def retrieveArtifacts(
         self,
         refs: Iterable[DatasetRef],
-        destination: Union[str, ButlerURI],
+        destination: ResourcePathExpression,
         transfer: str = "auto",
         preserve_path: bool = True,
         overwrite: bool = False,
-    ) -> List[ButlerURI]:
+    ) -> List[ResourcePath]:
         """Retrieve the artifacts associated with the supplied refs.
 
         Parameters
@@ -1427,11 +1422,12 @@ def retrieveArtifacts(
             The datasets for which artifacts are to be retrieved.
             A single ref can result in multiple artifacts. The refs must
             be resolved.
-        destination : `ButlerURI` or `str`
+        destination : `lsst.resources.ResourcePath` or `str`
             Location to write the artifacts.
         transfer : `str`, optional
             Method to use to transfer the artifacts. Must be one of the options
-            supported by `ButlerURI.transfer_from()`. "move" is not allowed.
+            supported by `~lsst.resources.ResourcePath.transfer_from()`.
+            "move" is not allowed.
         preserve_path : `bool`, optional
             If `True` the full path of the artifact within the datastore
             is preserved. If `False` the final file component of the path
@@ -1442,7 +1438,7 @@ def retrieveArtifacts(
 
         Returns
         -------
-        targets : `list` of `ButlerURI`
+        targets : `list` of `lsst.resources.ResourcePath`
             URIs of file artifacts in destination location. Order is not
             preserved.
 
@@ -1454,7 +1450,11 @@ def retrieveArtifacts(
         as a JSON file.
         """
         return self.datastore.retrieveArtifacts(
-            refs, ButlerURI(destination), transfer=transfer, preserve_path=preserve_path, overwrite=overwrite
+            refs,
+            ResourcePath(destination),
+            transfer=transfer,
+            preserve_path=preserve_path,
+            overwrite=overwrite,
         )
 
     def datasetExists(
@@ -2096,7 +2096,7 @@ def transfer_from(
         # purged, we have to ask for the (predicted) URI and check
         # existence explicitly. Execution butler is set up exactly like
         # this with no datastore records.
-        artifact_existence: Dict[ButlerURI, bool] = {}
+        artifact_existence: Dict[ResourcePath, bool] = {}
         if skip_missing:
             dataset_existence = source_butler.datastore.mexists(
                 source_refs, artifact_existence=artifact_existence

python/lsst/daf/butler/_butlerConfig.py

@@ -30,7 +30,9 @@
 import os
 from typing import Optional, Sequence, Union
 
-from .core import ButlerURI, Config, DatastoreConfig, StorageClassConfig
+from lsst.resources import ResourcePath, ResourcePathExpression
+
+from .core import Config, DatastoreConfig, StorageClassConfig
 from .registry import RegistryConfig
 from .transfers import RepoTransferFormatConfig
 
@@ -63,11 +65,11 @@ class ButlerConfig(Config):
 
     def __init__(
         self,
-        other: Optional[Union[str, ButlerURI, Config]] = None,
-        searchPaths: Sequence[Union[str, ButlerURI]] = None,
+        other: Optional[Union[ResourcePathExpression, Config]] = None,
+        searchPaths: Sequence[ResourcePathExpression] = None,
     ):
 
-        self.configDir: Optional[ButlerURI] = None
+        self.configDir: Optional[ResourcePath] = None
 
         # If this is already a ButlerConfig we assume that defaults
         # have already been loaded.
@@ -79,7 +81,7 @@ def __init__(
 
         if isinstance(other, (str, os.PathLike)):
             # This will only allow supported schemes
-            uri = ButlerURI(other)
+            uri = ResourcePath(other)
 
             # We allow the butler configuration file to be left off the
             # URI supplied by the user. If a directory-like URI is given
@@ -89,7 +91,7 @@ def __init__(
             # we have been given a directory-like URI if there is no
             # file extension. Local URIs do not need any guess work.
             if not uri.isLocal and not uri.getExtension():
-                uri = ButlerURI(other, forceDirectory=True)
+                uri = ResourcePath(other, forceDirectory=True)
 
             if uri.isdir():
                 # Could also be butler.json (for example in the butler
@@ -107,7 +109,7 @@ def __init__(
 
         configFile = butlerConfig.configFile
         if configFile is not None:
-            uri = ButlerURI(configFile)
+            uri = ResourcePath(configFile)
             self.configFile = uri
             self.configDir = uri.dirname()
 

python/lsst/daf/butler/_butlerRepoIndex.py

@@ -26,7 +26,9 @@
 import os
 from typing import ClassVar, Dict, Set
 
-from .core import ButlerURI, Config
+from lsst.resources import ResourcePath, ResourcePathExpression
+
+from .core import Config
 
 
 class ButlerRepoIndex:
@@ -49,17 +51,17 @@ class ButlerRepoIndex:
     index_env_var: ClassVar[str] = "DAF_BUTLER_REPOSITORY_INDEX"
     """The name of the environment variable to read to locate the index."""
 
-    _cache: ClassVar[Dict[ButlerURI, Config]] = {}
+    _cache: ClassVar[Dict[ResourcePath, Config]] = {}
     """Cache of indexes. In most scenarios only one index will be found
     and the environment will not change. In tests this may not be true."""
 
     @classmethod
-    def _read_repository_index(cls, index_uri: ButlerURI) -> Config:
+    def _read_repository_index(cls, index_uri: ResourcePathExpression) -> Config:
         """Read the repository index from the supplied URI.
 
         Parameters
         ----------
-        index_uri : `str` or `ButlerURI`
+        index_uri : `lsst.resources.ResourcePathExpression`
             URI of the repository index.
 
         Returns
@@ -85,12 +87,12 @@ def _read_repository_index(cls, index_uri: ButlerURI) -> Config:
         return repo_index
 
     @classmethod
-    def _get_index_uri(cls) -> ButlerURI:
+    def _get_index_uri(cls) -> ResourcePath:
         """Find the URI to the repository index.
 
         Returns
         -------
-        index_uri : `ButlerURI`
+        index_uri : `lsst.resources.ResourcePath`
             URI to the repository index.
 
         Raises
@@ -101,7 +103,7 @@ def _get_index_uri(cls) -> ButlerURI:
         index_uri = os.environ.get(cls.index_env_var)
         if index_uri is None:
             raise KeyError(f"No repository index defined in enviroment variable {cls.index_env_var}")
-        return ButlerURI(index_uri)
+        return ResourcePath(index_uri)
 
     @classmethod
     def _read_repository_index_from_environment(cls) -> Config:
@@ -131,7 +133,7 @@ def get_known_repos(cls) -> Set[str]:
         return set(repo_index)
 
     @classmethod
-    def get_repo_uri(cls, label: str) -> ButlerURI:
+    def get_repo_uri(cls, label: str) -> ResourcePath:
         """Look up the label in a butler repository index.
 
         Parameters
@@ -141,7 +143,7 @@ def get_repo_uri(cls, label: str) -> ButlerURI:
 
         Returns
         -------
-        uri : `ButlerURI`
+        uri : `lsst.resources.ResourcePath`
             URI to the Butler repository associated with the given label.
 
         Raises
@@ -162,4 +164,4 @@ def get_repo_uri(cls, label: str) -> ButlerURI:
             except KeyError:
                 index_uri = "<environment variable not defined>"
             raise KeyError(f"Label '{label}' not known to repository index at {index_uri}")
-        return ButlerURI(repo_uri)
+        return ResourcePath(repo_uri)