From 25d757c20690b8fadb209ae4b113694a37490267 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Wed, 6 Mar 2024 21:31:27 -0500 Subject: [PATCH 1/4] sort known_implementations --- fsspec/registry.py | 197 ++++++++++++++++++++++----------------------- 1 file changed, 96 insertions(+), 101 deletions(-) diff --git a/fsspec/registry.py b/fsspec/registry.py index 27dc87355..f22553b79 100644 --- a/fsspec/registry.py +++ b/fsspec/registry.py @@ -60,156 +60,151 @@ def register_implementation(name, cls, clobber=False, errtxt=None): # protocols mapped to the class which implements them. This dict can be # updated with register_implementation known_implementations = { - "data": {"class": "fsspec.implementations.data.DataFileSystem"}, - "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, - "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, - "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, - "dropbox": { - "class": "dropboxdrivefs.DropboxDriveFileSystem", - "err": ( - 'DropboxFileSystem requires "dropboxdrivefs",' - '"requests" and "dropbox" to be installed' - ), - }, - "http": { - "class": "fsspec.implementations.http.HTTPFileSystem", - "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', - }, - "https": { - "class": "fsspec.implementations.http.HTTPFileSystem", - "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', - }, - "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, - "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, - "gcs": { - "class": "gcsfs.GCSFileSystem", - "err": "Please install gcsfs to access Google Storage", - }, - "gs": { - "class": "gcsfs.GCSFileSystem", - "err": "Please install gcsfs to access Google Storage", - }, - "gdrive": { - "class": "gdrivefs.GoogleDriveFileSystem", - "err": "Please install gdrivefs for access to Google Drive", - }, - "sftp": { - "class": "fsspec.implementations.sftp.SFTPFileSystem", - "err": 'SFTPFileSystem requires "paramiko" to be installed', - }, - "ssh": { - "class": "fsspec.implementations.sftp.SFTPFileSystem", - "err": 'SFTPFileSystem requires "paramiko" to be installed', + "abfs": { + "class": "adlfs.AzureBlobFileSystem", + "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", }, - "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, - "hdfs": { - "class": "fsspec.implementations.arrow.HadoopFileSystem", - "err": "pyarrow and local java libraries required for HDFS", + "adl": { + "class": "adlfs.AzureDatalakeFileSystem", + "err": "Install adlfs to access Azure Datalake Gen1", }, "arrow_hdfs": { "class": "fsspec.implementations.arrow.HadoopFileSystem", "err": "pyarrow and local java libraries required for HDFS", }, - "webhdfs": { - "class": "fsspec.implementations.webhdfs.WebHDFS", - "err": 'webHDFS access requires "requests" to be installed', - }, - "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, - "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, - "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, - "oci": { - "class": "ocifs.OCIFileSystem", - "err": "Install ocifs to access OCI Object Storage", - }, - "ocilake": { - "class": "ocifs.OCIFileSystem", - "err": "Install ocifs to access OCI Data Lake", - }, "asynclocal": { "class": "morefs.asyn_local.AsyncLocalFileSystem", "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", }, - "adl": { - "class": "adlfs.AzureDatalakeFileSystem", - "err": "Install adlfs to access Azure Datalake Gen1", - }, - "abfs": { - "class": "adlfs.AzureBlobFileSystem", - "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", - }, "az": { "class": "adlfs.AzureBlobFileSystem", "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", }, - "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, "blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"}, - "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, - "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, + "box": { + "class": "boxfs.BoxFileSystem", + "err": "Please install boxfs to access BoxFileSystem", + }, + "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, "dask": { "class": "fsspec.implementations.dask.DaskWorkerFileSystem", "err": "Install dask distributed to access worker file system", }, + "data": {"class": "fsspec.implementations.data.DataFileSystem"}, "dbfs": { "class": "fsspec.implementations.dbfs.DatabricksFileSystem", "err": "Install the requests package to use the DatabricksFileSystem", }, - "github": { - "class": "fsspec.implementations.github.GithubFileSystem", - "err": "Install the requests package to use the github FS", + "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, + "dropbox": { + "class": "dropboxdrivefs.DropboxDriveFileSystem", + "err": 'DropboxFileSystem requires "dropboxdrivefs","requests" and "dropbox" to be installed', + }, + "dvc": { + "class": "dvc.api.DVCFileSystem", + "err": "Install dvc to access DVCFileSystem", + }, + "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, + "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, + "gcs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "gdrive": { + "class": "gdrivefs.GoogleDriveFileSystem", + "err": "Please install gdrivefs for access to Google Drive", }, + "generic": {"class": "fsspec.generic.GenericFileSystem"}, "git": { "class": "fsspec.implementations.git.GitFileSystem", "err": "Install pygit2 to browse local git repos", }, - "smb": { - "class": "fsspec.implementations.smb.SMBFileSystem", - "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', + "github": { + "class": "fsspec.implementations.github.GithubFileSystem", + "err": "Install the requests package to use the github FS", }, - "jupyter": { + "gs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "hdfs": { + "class": "fsspec.implementations.arrow.HadoopFileSystem", + "err": "pyarrow and local java libraries required for HDFS", + }, + "hf": { + "class": "huggingface_hub.HfFileSystem", + "err": "Install huggingface_hub to access HfFileSystem", + }, + "http": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "https": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "jlab": { "class": "fsspec.implementations.jupyter.JupyterFileSystem", "err": "Jupyter FS requires requests to be installed", }, - "jlab": { + "jupyter": { "class": "fsspec.implementations.jupyter.JupyterFileSystem", "err": "Jupyter FS requires requests to be installed", }, + "lakefs": { + "class": "lakefs_spec.LakeFSFileSystem", + "err": "Please install lakefs-spec to access LakeFSFileSystem", + }, "libarchive": { "class": "fsspec.implementations.libarchive.LibArchiveFileSystem", "err": "LibArchive requires to be installed", }, - "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, - "generic": {"class": "fsspec.generic.GenericFileSystem"}, + "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, + "oci": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Object Storage", + }, + "ocilake": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Data Lake", + }, "oss": { "class": "ossfs.OSSFileSystem", "err": "Install ossfs to access Alibaba Object Storage System", }, - "webdav": { - "class": "webdav4.fsspec.WebdavFileSystem", - "err": "Install webdav4 to access WebDAV", + "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, + "root": { + "class": "fsspec_xrootd.XRootDFileSystem", + "err": "Install fsspec-xrootd to access xrootd storage system. Note: 'root' is the protocol name for xrootd storage systems, not referring to root directories", }, - "dvc": { - "class": "dvc.api.DVCFileSystem", - "err": "Install dvc to access DVCFileSystem", + "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "sftp": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', }, - "hf": { - "class": "huggingface_hub.HfFileSystem", - "err": "Install huggingface_hub to access HfFileSystem", + "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, + "smb": { + "class": "fsspec.implementations.smb.SMBFileSystem", + "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', }, - "root": { - "class": "fsspec_xrootd.XRootDFileSystem", - "err": "Install fsspec-xrootd to access xrootd storage system." - + " Note: 'root' is the protocol name for xrootd storage systems," - + " not referring to root directories", + "ssh": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', }, - "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, - "box": { - "class": "boxfs.BoxFileSystem", - "err": "Please install boxfs to access BoxFileSystem", + "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, + "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, + "webdav": { + "class": "webdav4.fsspec.WebdavFileSystem", + "err": "Install webdav4 to access WebDAV", }, - "lakefs": { - "class": "lakefs_spec.LakeFSFileSystem", - "err": "Please install lakefs-spec to access LakeFSFileSystem", + "webhdfs": { + "class": "fsspec.implementations.webhdfs.WebHDFS", + "err": 'webHDFS access requires "requests" to be installed', }, + "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, } From d639c8ddb8905af38c403ca06368dd507b07aa01 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Thu, 7 Mar 2024 15:05:09 -0500 Subject: [PATCH 2/4] add test_sorted_known_implementations --- fsspec/tests/test_registry.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fsspec/tests/test_registry.py b/fsspec/tests/test_registry.py index 0664912a1..d7aa8f388 100644 --- a/fsspec/tests/test_registry.py +++ b/fsspec/tests/test_registry.py @@ -36,6 +36,12 @@ def clean_imports(): sys.modules["fsspec"] = real_module +def test_sorted_known_implementations(known_implementations): + expected = sorted(known_implementations.keys()) + actual = list(known_implementations.keys()) + assert actual == expected + + def test_registry_readonly(): get_filesystem_class("file") assert "file" in registry From 6958c884cb4dc9f58aedec5aaac077bec01aecdf Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Fri, 8 Mar 2024 12:00:13 -0500 Subject: [PATCH 3/4] rm as fixture --- fsspec/tests/test_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsspec/tests/test_registry.py b/fsspec/tests/test_registry.py index d7aa8f388..5e13c9ef8 100644 --- a/fsspec/tests/test_registry.py +++ b/fsspec/tests/test_registry.py @@ -36,7 +36,7 @@ def clean_imports(): sys.modules["fsspec"] = real_module -def test_sorted_known_implementations(known_implementations): +def test_sorted_known_implementations(): expected = sorted(known_implementations.keys()) actual = list(known_implementations.keys()) assert actual == expected From 4e521db07899a6d0b5b6ee4ff779540e16599df7 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Fri, 8 Mar 2024 12:02:02 -0500 Subject: [PATCH 4/4] lint --- fsspec/registry.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fsspec/registry.py b/fsspec/registry.py index f22553b79..8a36e0948 100644 --- a/fsspec/registry.py +++ b/fsspec/registry.py @@ -98,7 +98,10 @@ def register_implementation(name, cls, clobber=False, errtxt=None): "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, "dropbox": { "class": "dropboxdrivefs.DropboxDriveFileSystem", - "err": 'DropboxFileSystem requires "dropboxdrivefs","requests" and "dropbox" to be installed', + "err": ( + 'DropboxFileSystem requires "dropboxdrivefs","requests" and "' + '"dropbox" to be installed' + ), }, "dvc": { "class": "dvc.api.DVCFileSystem", @@ -177,7 +180,11 @@ def register_implementation(name, cls, clobber=False, errtxt=None): "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, "root": { "class": "fsspec_xrootd.XRootDFileSystem", - "err": "Install fsspec-xrootd to access xrootd storage system. Note: 'root' is the protocol name for xrootd storage systems, not referring to root directories", + "err": ( + "Install fsspec-xrootd to access xrootd storage system. " + "Note: 'root' is the protocol name for xrootd storage systems, " + "not referring to root directories" + ), }, "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"},