Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 103 additions & 101 deletions fsspec/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,156 +60,158 @@ def register_implementation(name, cls, clobber=False, errtxt=None):
# protocols mapped to the class which implements them. This dict can be
# updated with register_implementation
known_implementations = {
"data": {"class": "fsspec.implementations.data.DataFileSystem"},
"file": {"class": "fsspec.implementations.local.LocalFileSystem"},
"local": {"class": "fsspec.implementations.local.LocalFileSystem"},
"memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"},
"dropbox": {
"class": "dropboxdrivefs.DropboxDriveFileSystem",
"err": (
'DropboxFileSystem requires "dropboxdrivefs",'
'"requests" and "dropbox" to be installed'
),
},
"http": {
"class": "fsspec.implementations.http.HTTPFileSystem",
"err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed',
},
"https": {
"class": "fsspec.implementations.http.HTTPFileSystem",
"err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed',
},
"zip": {"class": "fsspec.implementations.zip.ZipFileSystem"},
"tar": {"class": "fsspec.implementations.tar.TarFileSystem"},
"gcs": {
"class": "gcsfs.GCSFileSystem",
"err": "Please install gcsfs to access Google Storage",
},
"gs": {
"class": "gcsfs.GCSFileSystem",
"err": "Please install gcsfs to access Google Storage",
},
"gdrive": {
"class": "gdrivefs.GoogleDriveFileSystem",
"err": "Please install gdrivefs for access to Google Drive",
},
"sftp": {
"class": "fsspec.implementations.sftp.SFTPFileSystem",
"err": 'SFTPFileSystem requires "paramiko" to be installed',
},
"ssh": {
"class": "fsspec.implementations.sftp.SFTPFileSystem",
"err": 'SFTPFileSystem requires "paramiko" to be installed',
"abfs": {
"class": "adlfs.AzureBlobFileSystem",
"err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage",
},
"ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"},
"hdfs": {
"class": "fsspec.implementations.arrow.HadoopFileSystem",
"err": "pyarrow and local java libraries required for HDFS",
"adl": {
"class": "adlfs.AzureDatalakeFileSystem",
"err": "Install adlfs to access Azure Datalake Gen1",
},
"arrow_hdfs": {
"class": "fsspec.implementations.arrow.HadoopFileSystem",
"err": "pyarrow and local java libraries required for HDFS",
},
"webhdfs": {
"class": "fsspec.implementations.webhdfs.WebHDFS",
"err": 'webHDFS access requires "requests" to be installed',
},
"s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"},
"s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"},
"wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"},
"oci": {
"class": "ocifs.OCIFileSystem",
"err": "Install ocifs to access OCI Object Storage",
},
"ocilake": {
"class": "ocifs.OCIFileSystem",
"err": "Install ocifs to access OCI Data Lake",
},
"asynclocal": {
"class": "morefs.asyn_local.AsyncLocalFileSystem",
"err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem",
},
"adl": {
"class": "adlfs.AzureDatalakeFileSystem",
"err": "Install adlfs to access Azure Datalake Gen1",
},
"abfs": {
"class": "adlfs.AzureBlobFileSystem",
"err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage",
},
"az": {
"class": "adlfs.AzureBlobFileSystem",
"err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage",
},
"cached": {"class": "fsspec.implementations.cached.CachingFileSystem"},
"blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"},
"filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"},
"simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"},
"box": {
"class": "boxfs.BoxFileSystem",
"err": "Please install boxfs to access BoxFileSystem",
},
"cached": {"class": "fsspec.implementations.cached.CachingFileSystem"},
"dask": {
"class": "fsspec.implementations.dask.DaskWorkerFileSystem",
"err": "Install dask distributed to access worker file system",
},
"data": {"class": "fsspec.implementations.data.DataFileSystem"},
"dbfs": {
"class": "fsspec.implementations.dbfs.DatabricksFileSystem",
"err": "Install the requests package to use the DatabricksFileSystem",
},
"github": {
"class": "fsspec.implementations.github.GithubFileSystem",
"err": "Install the requests package to use the github FS",
"dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"},
"dropbox": {
"class": "dropboxdrivefs.DropboxDriveFileSystem",
"err": (
'DropboxFileSystem requires "dropboxdrivefs","requests" and "'
'"dropbox" to be installed'
),
},
"dvc": {
"class": "dvc.api.DVCFileSystem",
"err": "Install dvc to access DVCFileSystem",
},
"file": {"class": "fsspec.implementations.local.LocalFileSystem"},
"filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"},
"ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"},
"gcs": {
"class": "gcsfs.GCSFileSystem",
"err": "Please install gcsfs to access Google Storage",
},
"gdrive": {
"class": "gdrivefs.GoogleDriveFileSystem",
"err": "Please install gdrivefs for access to Google Drive",
},
"generic": {"class": "fsspec.generic.GenericFileSystem"},
"git": {
"class": "fsspec.implementations.git.GitFileSystem",
"err": "Install pygit2 to browse local git repos",
},
"smb": {
"class": "fsspec.implementations.smb.SMBFileSystem",
"err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed',
"github": {
"class": "fsspec.implementations.github.GithubFileSystem",
"err": "Install the requests package to use the github FS",
},
"jupyter": {
"gs": {
"class": "gcsfs.GCSFileSystem",
"err": "Please install gcsfs to access Google Storage",
},
"hdfs": {
"class": "fsspec.implementations.arrow.HadoopFileSystem",
"err": "pyarrow and local java libraries required for HDFS",
},
"hf": {
"class": "huggingface_hub.HfFileSystem",
"err": "Install huggingface_hub to access HfFileSystem",
},
"http": {
"class": "fsspec.implementations.http.HTTPFileSystem",
"err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed',
},
"https": {
"class": "fsspec.implementations.http.HTTPFileSystem",
"err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed',
},
"jlab": {
"class": "fsspec.implementations.jupyter.JupyterFileSystem",
"err": "Jupyter FS requires requests to be installed",
},
"jlab": {
"jupyter": {
"class": "fsspec.implementations.jupyter.JupyterFileSystem",
"err": "Jupyter FS requires requests to be installed",
},
"lakefs": {
"class": "lakefs_spec.LakeFSFileSystem",
"err": "Please install lakefs-spec to access LakeFSFileSystem",
},
"libarchive": {
"class": "fsspec.implementations.libarchive.LibArchiveFileSystem",
"err": "LibArchive requires to be installed",
},
"reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"},
"generic": {"class": "fsspec.generic.GenericFileSystem"},
"local": {"class": "fsspec.implementations.local.LocalFileSystem"},
"memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"},
"oci": {
"class": "ocifs.OCIFileSystem",
"err": "Install ocifs to access OCI Object Storage",
},
"ocilake": {
"class": "ocifs.OCIFileSystem",
"err": "Install ocifs to access OCI Data Lake",
},
"oss": {
"class": "ossfs.OSSFileSystem",
"err": "Install ossfs to access Alibaba Object Storage System",
},
"webdav": {
"class": "webdav4.fsspec.WebdavFileSystem",
"err": "Install webdav4 to access WebDAV",
"reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"},
"root": {
"class": "fsspec_xrootd.XRootDFileSystem",
"err": (
"Install fsspec-xrootd to access xrootd storage system. "
"Note: 'root' is the protocol name for xrootd storage systems, "
"not referring to root directories"
),
},
"dvc": {
"class": "dvc.api.DVCFileSystem",
"err": "Install dvc to access DVCFileSystem",
"s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"},
"s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"},
"sftp": {
"class": "fsspec.implementations.sftp.SFTPFileSystem",
"err": 'SFTPFileSystem requires "paramiko" to be installed',
},
"hf": {
"class": "huggingface_hub.HfFileSystem",
"err": "Install huggingface_hub to access HfFileSystem",
"simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"},
"smb": {
"class": "fsspec.implementations.smb.SMBFileSystem",
"err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed',
},
"root": {
"class": "fsspec_xrootd.XRootDFileSystem",
"err": "Install fsspec-xrootd to access xrootd storage system."
+ " Note: 'root' is the protocol name for xrootd storage systems,"
+ " not referring to root directories",
"ssh": {
"class": "fsspec.implementations.sftp.SFTPFileSystem",
"err": 'SFTPFileSystem requires "paramiko" to be installed',
},
"dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"},
"box": {
"class": "boxfs.BoxFileSystem",
"err": "Please install boxfs to access BoxFileSystem",
"tar": {"class": "fsspec.implementations.tar.TarFileSystem"},
"wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"},
"webdav": {
"class": "webdav4.fsspec.WebdavFileSystem",
"err": "Install webdav4 to access WebDAV",
},
"lakefs": {
"class": "lakefs_spec.LakeFSFileSystem",
"err": "Please install lakefs-spec to access LakeFSFileSystem",
"webhdfs": {
"class": "fsspec.implementations.webhdfs.WebHDFS",
"err": 'webHDFS access requires "requests" to be installed',
},
"zip": {"class": "fsspec.implementations.zip.ZipFileSystem"},
}


Expand Down
6 changes: 6 additions & 0 deletions fsspec/tests/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ def clean_imports():
sys.modules["fsspec"] = real_module


def test_sorted_known_implementations():
expected = sorted(known_implementations.keys())
actual = list(known_implementations.keys())
assert actual == expected


def test_registry_readonly():
get_filesystem_class("file")
assert "file" in registry
Expand Down