Merge pull request #300 from WilliamJamieson/bugfix/circular-imports

Fix for circular dependencies between asdf packages

CHANGES.rst

@@ -1,4 +1,9 @@
-1.0.0 (unreleased)
+1.0.1 (unreleased)
+------------------
+
+- Remove asdf as an install dependency for the asdf-standard package. [#300]
+
+1.0.0 (2022-02-14)
 -------------------
 
 - Add installable Python package to replace use of this repo as a submodule.  [#292]

setup.cfg

@@ -20,7 +20,6 @@ setup_requires =
     setuptools
     setuptools_scm
 install_requires =
-    asdf>=2.8.0
     importlib_resources>=3;python_version<"3.9"
 
 [options.extras_require]

src/asdf_standard/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["DirectoryResourceMapping"]
+
+from .resource import DirectoryResourceMapping

src/asdf_standard/integration.py

@@ -6,8 +6,6 @@
 else:
     import importlib.resources as importlib_resources
 
-from asdf.resource import DirectoryResourceMapping
-
 import asdf_standard
 
 
@@ -21,11 +19,13 @@ def get_resource_mappings():
             raise RuntimeError("Missing resources directory")
 
     return [
-        DirectoryResourceMapping(resources_root / "schemas" / "stsci.edu", "http://stsci.edu/schemas/", recursive=True),
-        DirectoryResourceMapping(
+        asdf_standard.DirectoryResourceMapping(
+            resources_root / "schemas" / "stsci.edu", "http://stsci.edu/schemas/", recursive=True
+        ),
+        asdf_standard.DirectoryResourceMapping(
             resources_root / "schemas" / "asdf-format.org" / "core", "asdf://asdf-format.org/core/schemas/"
         ),
-        DirectoryResourceMapping(
+        asdf_standard.DirectoryResourceMapping(
             resources_root / "manifests" / "asdf-format.org" / "core",
             "asdf://asdf-format.org/core/manifests/",
         ),

src/asdf_standard/resource.py

@@ -0,0 +1,81 @@
+import fnmatch
+import os
+from collections.abc import Mapping
+from pathlib import Path
+
+
+class DirectoryResourceMapping(Mapping):
+    """
+    Resource mapping that reads resource content
+    from a directory or directory tree.
+    Parameters
+    ----------
+    root : str or importlib.abc.Traversable
+        Root directory (or directory-like Traversable) of the resource
+        files.  `str` will be interpreted as a filesystem path.
+    uri_prefix : str
+        Prefix used to construct URIs from file paths.  The
+        prefix will be prepended to paths relative to the root
+        directory.
+    recursive : bool, optional
+        If `True`, recurse into subdirectories.  Defaults to `False`.
+    filename_pattern : str, optional
+        Glob pattern that identifies relevant filenames.
+        Defaults to `"*.yaml"`.
+    stem_filename : bool, optional
+        If `True`, remove the filename's extension when
+        constructing its URI.
+    """
+
+    def __init__(self, root, uri_prefix, recursive=False, filename_pattern="*.yaml", stem_filename=True):
+        self._uri_to_file = {}
+        self._recursive = recursive
+        self._filename_pattern = filename_pattern
+        self._stem_filename = stem_filename
+
+        if isinstance(root, str):
+            self._root = Path(root)
+        else:
+            self._root = root
+
+        if uri_prefix.endswith("/"):
+            self._uri_prefix = uri_prefix[:-1]
+        else:
+            self._uri_prefix = uri_prefix
+
+        for file, path_components in self._iterate_files(self._root, []):
+            self._uri_to_file[self._make_uri(file, path_components)] = file
+
+    def _iterate_files(self, directory, path_components):
+        for obj in directory.iterdir():
+            if obj.is_file() and fnmatch.fnmatch(obj.name, self._filename_pattern):
+                yield obj, path_components
+            elif obj.is_dir() and self._recursive:
+                yield from self._iterate_files(obj, path_components + [obj.name])
+
+    def _make_uri(self, file, path_components):
+        if self._stem_filename:
+            filename = os.path.splitext(file.name)[0]
+        else:
+            filename = file.name
+
+        return "/".join([self._uri_prefix] + path_components + [filename])
+
+    def __getitem__(self, uri):
+        return self._uri_to_file[uri].read_bytes()
+
+    def __len__(self):
+        return len(self._uri_to_file)
+
+    def __iter__(self):
+        yield from self._uri_to_file
+
+    def __repr__(self):
+        return "{}({!r}, {!r}, recursive={!r}, filename_pattern={!r}, stem_filename={!r})".format(
+            self.__class__.__name__,
+            self._root,
+            self._uri_prefix,
+            self._recursive,
+            self._filename_pattern,
+            self._stem_filename,
+        )

tests/test_resource.py

@@ -0,0 +1,172 @@
+import io
+import sys
+from collections.abc import Mapping
+from pathlib import Path
+
+if sys.version_info < (3, 9):
+    import importlib_resources as importlib
+else:
+    import importlib
+
+from asdf_standard import DirectoryResourceMapping
+
+
+def test_directory_resource_mapping(tmpdir):
+    tmpdir.mkdir("schemas")
+    (tmpdir / "schemas").mkdir("nested")
+    with (tmpdir / "schemas" / "foo-1.2.3.yaml").open("w") as f:
+        f.write("id: http://somewhere.org/schemas/foo-1.2.3\n")
+    with (tmpdir / "schemas" / "nested" / "bar-4.5.6.yaml").open("w") as f:
+        f.write("id: http://somewhere.org/schemas/nested/bar-4.5.6\n")
+    with (tmpdir / "schemas" / "baz-7.8.9").open("w") as f:
+        f.write("id: http://somewhere.org/schemas/baz-7.8.9\n")
+
+    mapping = DirectoryResourceMapping(str(tmpdir / "schemas"), "http://somewhere.org/schemas")
+    assert isinstance(mapping, Mapping)
+    assert len(mapping) == 1
+    assert set(mapping) == {"http://somewhere.org/schemas/foo-1.2.3"}
+    assert "http://somewhere.org/schemas/foo-1.2.3" in mapping
+    assert b"http://somewhere.org/schemas/foo-1.2.3" in mapping["http://somewhere.org/schemas/foo-1.2.3"]
+    assert "http://somewhere.org/schemas/baz-7.8.9" not in mapping
+    assert "http://somewhere.org/schemas/baz-7.8" not in mapping
+    assert "http://somewhere.org/schemas/foo-1.2.3.yaml" not in mapping
+    assert "http://somewhere.org/schemas/nested/bar-4.5.6" not in mapping
+
+    mapping = DirectoryResourceMapping(str(tmpdir / "schemas"), "http://somewhere.org/schemas", recursive=True)
+    assert len(mapping) == 2
+    assert set(mapping) == {"http://somewhere.org/schemas/foo-1.2.3", "http://somewhere.org/schemas/nested/bar-4.5.6"}
+    assert "http://somewhere.org/schemas/foo-1.2.3" in mapping
+    assert b"http://somewhere.org/schemas/foo-1.2.3" in mapping["http://somewhere.org/schemas/foo-1.2.3"]
+    assert "http://somewhere.org/schemas/baz-7.8.9" not in mapping
+    assert "http://somewhere.org/schemas/baz-7.8" not in mapping
+    assert "http://somewhere.org/schemas/nested/bar-4.5.6" in mapping
+    assert b"http://somewhere.org/schemas/nested/bar-4.5.6" in mapping["http://somewhere.org/schemas/nested/bar-4.5.6"]
+
+    mapping = DirectoryResourceMapping(
+        str(tmpdir / "schemas"),
+        "http://somewhere.org/schemas",
+        recursive=True,
+        filename_pattern="baz-*",
+        stem_filename=False,
+    )
+
+    assert len(mapping) == 1
+    assert set(mapping) == {"http://somewhere.org/schemas/baz-7.8.9"}
+    assert "http://somewhere.org/schemas/foo-1.2.3" not in mapping
+    assert "http://somewhere.org/schemas/baz-7.8.9" in mapping
+    assert b"http://somewhere.org/schemas/baz-7.8.9" in mapping["http://somewhere.org/schemas/baz-7.8.9"]
+    assert "http://somewhere.org/schemas/nested/bar-4.5.6" not in mapping
+
+    # Check that the repr is reasonable
+    # Need to be careful checking the path string because
+    # pathlib normalizes Windows paths.
+    assert repr(Path(str(tmpdir / "schemas"))) in repr(mapping)
+    assert "http://somewhere.org/schemas" in repr(mapping)
+    assert "recursive=True" in repr(mapping)
+    assert "filename_pattern='baz-*'" in repr(mapping)
+    assert "stem_filename=False" in repr(mapping)
+
+    # Make sure trailing slash is handled correctly
+    mapping = DirectoryResourceMapping(str(tmpdir / "schemas"), "http://somewhere.org/schemas/")
+    assert len(mapping) == 1
+    assert set(mapping) == {"http://somewhere.org/schemas/foo-1.2.3"}
+    assert "http://somewhere.org/schemas/foo-1.2.3" in mapping
+    assert b"http://somewhere.org/schemas/foo-1.2.3" in mapping["http://somewhere.org/schemas/foo-1.2.3"]
+
+
+def test_directory_resource_mapping_with_traversable():
+    """
+    Confirm that DirectoryResourceMapping doesn't use pathlib.Path
+    methods outside of the Traversable interface.
+    """
+
+    class MockTraversable(importlib.abc.Traversable):
+        def __init__(self, name, value):
+            self._name = name
+            self._value = value
+
+        def iterdir(self):
+            if isinstance(self._value, dict):
+                for key, child in self._value.items():
+                    yield MockTraversable(key, child)
+
+        def read_bytes(self):
+            if not isinstance(self._value, bytes):
+                raise RuntimeError("Not a file")
+            return self._value
+
+        def read_text(self, encoding="utf-8"):
+            return self.read_bytes().decode(encoding)
+
+        def is_dir(self):
+            return isinstance(self._value, dict)
+
+        def is_file(self):
+            return self._value is not None and not isinstance(self._value, dict)
+
+        def joinpath(self, child):
+            if isinstance(self._value, dict):
+                child_value = self._value.get(child)
+            else:
+                child_value = None
+
+            return MockTraversable(child, child_value)
+
+        def __truediv__(self, child):
+            return self.joinpath(child)
+
+        def open(self, mode="r", *args, **kwargs):
+            if not self.is_file():
+                raise RuntimeError("Not a file")
+
+            if mode == "r":
+                return io.TextIOWrapper(io.BytesIO(self._value), *args, **kwargs)
+            elif mode == "rb":
+                return io.BytesIO(self._value)
+            else:
+                raise "Not a valid mode"
+
+        @property
+        def name(self):
+            return self._name
+
+    root = MockTraversable(
+        "/path/to/some/root",
+        {"foo-1.0.0.yaml": b"foo", "bar-1.0.0.yaml": b"bar", "baz-1.0.0": b"baz", "nested": {"foz-1.0.0.yaml": b"foz"}},
+    )
+
+    mapping = DirectoryResourceMapping(root, "http://somewhere.org/schemas")
+    assert len(mapping) == 2
+    assert set(mapping) == {"http://somewhere.org/schemas/foo-1.0.0", "http://somewhere.org/schemas/bar-1.0.0"}
+    assert "http://somewhere.org/schemas/foo-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/foo-1.0.0"] == b"foo"
+    assert "http://somewhere.org/schemas/bar-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/bar-1.0.0"] == b"bar"
+    assert "http://somewhere.org/schemas/baz-1.0.0" not in mapping
+    assert "http://somewhere.org/schemas/nested/foz-1.0.0" not in mapping
+
+    mapping = DirectoryResourceMapping(root, "http://somewhere.org/schemas", recursive=True)
+    assert len(mapping) == 3
+    assert set(mapping) == {
+        "http://somewhere.org/schemas/foo-1.0.0",
+        "http://somewhere.org/schemas/bar-1.0.0",
+        "http://somewhere.org/schemas/nested/foz-1.0.0",
+    }
+    assert "http://somewhere.org/schemas/foo-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/foo-1.0.0"] == b"foo"
+    assert "http://somewhere.org/schemas/bar-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/bar-1.0.0"] == b"bar"
+    assert "http://somewhere.org/schemas/baz-1.0.0" not in mapping
+    assert "http://somewhere.org/schemas/nested/foz-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/nested/foz-1.0.0"] == b"foz"
+
+    mapping = DirectoryResourceMapping(
+        root, "http://somewhere.org/schemas", filename_pattern="baz-*", stem_filename=False
+    )
+    assert len(mapping) == 1
+    assert set(mapping) == {"http://somewhere.org/schemas/baz-1.0.0"}
+    assert "http://somewhere.org/schemas/foo-1.0.0" not in mapping
+    assert "http://somewhere.org/schemas/bar-1.0.0" not in mapping
+    assert "http://somewhere.org/schemas/baz-1.0.0" in mapping
+    assert mapping["http://somewhere.org/schemas/baz-1.0.0"] == b"baz"
+    assert "http://somewhere.org/schemas/nested/foz-1.0.0" not in mapping