DM-33528: Use importlib.resources rather than deprecated pkg_resources

doc/changes/DM-33528.feature.rst

@@ -0,0 +1,2 @@
+``resource`` URI schemes now use `importlib.resources` (or `importlib_resources`) rather than the deprecated ``pkg_resources``.
+Due to this change, ``resource`` URI schemes now also support ``walk`` and ``findFileResources``.

pyproject.toml

@@ -22,6 +22,7 @@ classifiers = [
 keywords = ["lsst"]
 dependencies = [
     "lsst-utils",
+    "importlib_resources",
 ]
 dynamic = ["version"]
 

python/lsst/resources/packageresource.py

@@ -9,11 +9,19 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
+from __future__ import annotations
+
 import contextlib
 import logging
-from typing import Iterator, Optional
+import re
+import sys
+
+if sys.version_info >= (3, 11, 0):
+    from importlib import resources
+else:
+    import importlib_resources as resources  # type: ignore[no-redef]
 
-import pkg_resources
+from typing import Iterator
 
 __all__ = ("PackageResourcePath",)
 
@@ -33,27 +41,59 @@ class PackageResourcePath(ResourcePath):
 
     def exists(self) -> bool:
         """Check that the python resource exists."""
-        return pkg_resources.resource_exists(self.netloc, self.relativeToPathRoot)
+        ref = resources.files(self.netloc).joinpath(self.relativeToPathRoot)
+        return ref.is_file() or ref.is_dir()
 
     def read(self, size: int = -1) -> bytes:
         """Read the contents of the resource."""
-        with pkg_resources.resource_stream(self.netloc, self.relativeToPathRoot) as fh:
+        ref = resources.files(self.netloc).joinpath(self.relativeToPathRoot)
+        with ref.open("rb") as fh:
             return fh.read(size)
 
     @contextlib.contextmanager
     def open(
         self,
         mode: str = "r",
         *,
-        encoding: Optional[str] = None,
+        encoding: str | None = None,
         prefer_file_temporary: bool = False,
     ) -> Iterator[ResourceHandleProtocol]:
         # Docstring inherited.
         if "r" not in mode or "+" in mode:
             raise RuntimeError(f"Package resource URI {self} is read-only.")
-        if "b" in mode:
-            with pkg_resources.resource_stream(self.netloc, self.relativeToPathRoot) as buffer:
-                yield buffer
+        ref = resources.files(self.netloc).joinpath(self.relativeToPathRoot)
+        with ref.open(mode, encoding=encoding) as buffer:
+            yield buffer
+
+    def walk(
+        self, file_filter: str | re.Pattern | None = None
+    ) -> Iterator[list | tuple[ResourcePath, list[str], list[str]]]:
+        # Docstring inherited.
+        if not self.dirLike:
+            raise ValueError("Can not walk a non-directory URI")
+
+        if isinstance(file_filter, str):
+            file_filter = re.compile(file_filter)
+
+        ref = resources.files(self.netloc).joinpath(self.relativeToPathRoot)
+
+        files: list[str] = []
+        dirs: list[str] = []
+        for item in ref.iterdir():
+            if item.is_file():
+                files.append(item.name)
+            else:
+                # This is a directory.
+                dirs.append(item.name)
+
+        if file_filter is not None:
+            files = [f for f in files if file_filter.search(f)]
+
+        if not dirs and not files:
+            return
         else:
-            with super().open(mode, encoding=encoding, prefer_file_temporary=prefer_file_temporary) as buffer:
-                yield buffer
+            yield type(self)(self, forceAbsolute=False, forceDirectory=True), dirs, files
+
+        for dir in dirs:
+            new_uri = self.join(dir, forceDirectory=True)
+            yield from new_uri.walk(file_filter)

requirements.txt

@@ -1,4 +1,6 @@
 git+https://github.com/lsst/utils@main#egg=lsst-utils
+# Needed for Python <=3.10
+importlib_resources
 # optional
 backoff >= 1.10
 boto3 >= 1.13

tests/test_resource.py

@@ -9,6 +9,7 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
+import re
 import unittest
 
 from lsst.resources import ResourcePath
@@ -76,6 +77,42 @@ def test_open(self):
             with uri.open("w") as buffer:
                 pass
 
+    def test_walk(self):
+        """Test that we can find file resources.
+
+        Try to find resources in this package. Python does not care whether
+        a resource is a Python file or anything else.
+        """
+
+        resource = ResourcePath("resource://lsst.resources/")
+        resources = set(ResourcePath.findFileResources([resource]))
+
+        # Do not try to list all possible options. Files can move around
+        # and cache files can appear.
+        subset = {
+            ResourcePath("resource://lsst.resources/_resourceHandles/_s3ResourceHandle.py"),
+            ResourcePath("resource://lsst.resources/http.py"),
+        }
+        for r in subset:
+            self.assertIn(r, resources)
+
+        resources = set(
+            ResourcePath.findFileResources(
+                [ResourcePath("resource://lsst.resources/")], file_filter=r".*\.txt"
+            )
+        )
+        self.assertEqual(resources, set())
+
+        # Compare regex with str.
+        regex = r".*\.py"
+        py_files_str = list(resource.walk(file_filter=regex))
+        py_files_re = list(resource.walk(file_filter=re.compile(regex)))
+        self.assertGreater(len(py_files_str), 1)
+        self.assertEqual(py_files_str, py_files_re)
+
+        with self.assertRaises(ValueError):
+            list(ResourcePath("resource://lsst.resources/http.py").walk())
+
 
 if __name__ == "__main__":
     unittest.main()