Merge pull request fsspec#515 from bstrdsmkr/support_entry_points

Add support for filesystem discovery via entry points

fsspec/__init__.py

@@ -1,15 +1,23 @@
-from ._version import get_versions
+try:
+    from importlib.metadata import entry_points
+except ImportError:  # python < 3.8
+    try:
+        from importlib_metadata import entry_points
+    except ImportError:
+        entry_points = None
 
-from .spec import AbstractFileSystem
+
+from . import caching
+from ._version import get_versions
+from .core import get_fs_token_paths, open, open_files, open_local
+from .mapping import FSMap, get_mapper
 from .registry import (
-    get_filesystem_class,
-    registry,
     filesystem,
+    get_filesystem_class,
     register_implementation,
+    registry,
 )
-from .mapping import FSMap, get_mapper
-from .core import open_files, get_fs_token_paths, open, open_local
-from . import caching
+from .spec import AbstractFileSystem
 
 __version__ = get_versions()["version"]
 del get_versions
@@ -29,3 +37,9 @@
     "registry",
     "caching",
 ]
+
+if entry_points is not None:
+    entry_points = entry_points()
+    for spec in entry_points.get("fsspec.specs", []):
+        err_msg = f"Unable to load filesystem from {spec}"
+        register_implementation(spec.name, spec.module, errtxt=err_msg)

fsspec/spec.py

@@ -80,10 +80,14 @@ def __call__(cls, *args, **kwargs):
 
 pa_version = get_package_version_without_import("pyarrow")
 if pa_version and LooseVersion(pa_version) < LooseVersion("2.0"):
-    import pyarrow as pa
-
-    up = pa.filesystem.DaskFileSystem
-else:
+    try:
+        import pyarrow as pa
+
+        up = pa.filesystem.DaskFileSystem
+    except ImportError:  # pragma: no cover
+        # pyarrow exists but doesn't import for some reason
+        up = object
+else:  # pragma: no cover
     up = object
 
 

fsspec/tests/test_registry.py

@@ -1,14 +1,23 @@
+import sys
+from unittest.mock import create_autospec, patch
+
 import pytest
+
 from fsspec.registry import (
-    get_filesystem_class,
-    _registry,
-    registry,
-    register_implementation,
     ReadOnlyError,
+    _registry,
+    get_filesystem_class,
     known_implementations,
+    register_implementation,
+    registry,
 )
 from fsspec.spec import AbstractFileSystem
 
+try:
+    from importlib.metadata import EntryPoint
+except ImportError:  # python < 3.8
+    from importlib_metadata import EntryPoint
+
 
 @pytest.fixture()
 def clear_registry():
@@ -19,6 +28,16 @@ def clear_registry():
         known_implementations.pop("test", None)
 
 
+@pytest.fixture()
+def clean_imports():
+    try:
+        real_module = sys.modules["fsspec"]
+        del sys.modules["fsspec"]
+        yield
+    finally:
+        sys.modules["fsspec"] = real_module
+
+
 @pytest.mark.parametrize(
     "protocol,module,minversion,oldversion",
     [("s3", "s3fs", "0.3.0", "0.1.0"), ("gs", "gcsfs", "0.3.0", "0.1.0")],
@@ -85,3 +104,18 @@ def test_register_fail(clear_registry):
     with pytest.raises(ValueError):
         register_implementation("test", AbstractFileSystem, clobber=False)
     register_implementation("test", AbstractFileSystem, clobber=True)
+
+
+def test_entry_points_registered_on_import(clear_registry, clean_imports):
+    mock_ep = create_autospec(EntryPoint, module="fsspec.spec.AbstractFileSystem")
+    mock_ep.name = "test"  # this can't be set in the constructor...
+    if sys.version_info < (3, 8):
+        import_location = "importlib_metadata.entry_points"
+    else:
+        import_location = "importlib.metadata.entry_points"
+    with patch(import_location, return_value={"fsspec.specs": [mock_ep]}):
+        assert "test" not in registry
+        import fsspec
+
+        get_filesystem_class("test")
+        assert "test" in registry

setup.py

@@ -33,6 +33,7 @@
     python_requires=">3.6",
     install_requires=open("requirements.txt").read().strip().split("\n"),
     extras_require={
+        ":python_version < '3.8'": ['importlib_metadata'],
         "abfs": ["adlfs"],
         "adl": ["adlfs"],
         "dask": ["dask", "distributed"],

tox.ini

@@ -36,6 +36,7 @@ conda_deps=
 deps=
     hadoop-test-cluster==0.1.0
     smbprotocol
+    py36,py37: importlib_metadata
 
 [testenv]
 description=Run test suite against target versions.
@@ -46,7 +47,7 @@ conda_deps=
 deps=
     {[core]deps}
 commands =
-    py.test --cov=fsspec -v -r s
+    py.test --cov=fsspec -v -r s {posargs}
 passenv = CIRUN
 
 [testenv:s3fs]