deps: bump dvc-data to 0.1.19

dvc/fs/__init__.py

@@ -19,23 +19,7 @@
     ConfigError,
     RemoteMissingDepsError,
 )
-from dvc_objects.fs.implementations.azure import AzureAuthError  # noqa: F401
-from dvc_objects.fs.implementations.azure import AzureFileSystem  # noqa: F401
-from dvc_objects.fs.implementations.gdrive import (  # noqa: F401
-    GDriveFileSystem,
-)
-from dvc_objects.fs.implementations.gs import GSFileSystem  # noqa: F401
-from dvc_objects.fs.implementations.hdfs import HDFSFileSystem  # noqa: F401
 from dvc_objects.fs.implementations.local import localfs  # noqa: F401
-from dvc_objects.fs.implementations.oss import OSSFileSystem  # noqa: F401
-from dvc_objects.fs.implementations.s3 import S3FileSystem  # noqa: F401
-from dvc_objects.fs.implementations.webdav import (  # noqa: F401
-    WebDAVFileSystem,
-    WebDAVSFileSystem,
-)
-from dvc_objects.fs.implementations.webhdfs import (  # noqa: F401
-    WebHDFSFileSystem,
-)
 from dvc_objects.fs.path import Path  # noqa: F401
 
 from .data import DataFileSystem  # noqa: F401
@@ -112,11 +96,8 @@ def get_cloud_fs(repo, **kwargs):
 
     cls = get_fs_cls(remote_conf)
 
-    if cls == GDriveFileSystem and repo:
-        remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
-
     url = remote_conf.pop("url")
-    if issubclass(cls, WebDAVFileSystem):
+    if cls.protocol in ["webdav", "webdavs"]:
         # For WebDAVFileSystem, provided url is the base path itself, so it
         # should be treated as being a root path.
         fs_path = cls.root_marker

dvc/output.py

@@ -28,14 +28,7 @@
 from dvc_data.transfer import transfer as otransfer
 from dvc_objects.errors import ObjectFormatError
 
-from .fs import (
-    HDFSFileSystem,
-    LocalFileSystem,
-    RemoteMissingDepsError,
-    S3FileSystem,
-    Schemes,
-    get_cloud_fs,
-)
+from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs
 from .utils import relpath
 from .utils.fs import path_isin
 
@@ -65,10 +58,12 @@
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
+HDFS_PARAM_CHECKSUM = "checksum"
+S3_PARAM_CHECKSUM = "etag"
 CHECKSUMS_SCHEMA = {
     LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
+    HDFS_PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3_PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
 }
 
 

setup.cfg

@@ -68,7 +68,7 @@ install_requires =
     dvc-render==0.0.9
     dvc-task==0.1.2
     dvclive>=0.10.0
-    dvc-data==0.1.18
+    dvc-data==0.1.20
     dvc-http==0.0.2
     hydra-core>=1.1.0
 
@@ -82,40 +82,27 @@ all =
     %(s3)s
     %(ssh)s
     %(webdav)s
+    %(webhdfs)s
 dev =
     %(all)s
     %(tests)s
     dvc-data[cli]
 # Extra dependencies for remote integrations
-azure =
-    adlfs>=2021.10.0
-    azure-identity>=1.4.0
-    knack
-gdrive = pydrive2[fsspec]>=1.10.1
-gs = gcsfs>=2021.11.1
-hdfs =
-    # due to https://github.com/python-poetry/poetry/issues/4683, we have to
-    # explicitly specify pyarrow here
-    pyarrow>=1
-    fsspec[arrow]
-oss = ossfs>=2021.8.0
-s3 =
-    s3fs[boto3]>=2022.02.0
-    aiobotocore[boto3]>2
-ssh =
-    # due to https://github.com/python-poetry/poetry/issues/4683, we have to
-    # explicitly specify bcrypt here
-    bcrypt
-    dvc-ssh==0.0.1
+azure = dvc-azure==0.0.1
+gdrive = dvc-gdrive==0.0.1
+gs = dvc-gs==0.0.1
+hdfs = dvc-hdfs==0.0.1
+oss = dvc-oss==0.0.1
+s3 = dvc-s3==0.0.1
+ssh = dvc-ssh==0.0.1
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for Linux, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself.
 ssh_gssapi = dvc-ssh[gssapi]==0.0.1
-webdav = webdav4>=0.9.3
-# not to break `dvc[webhdfs]`
-webhdfs =
+webdav = dvc-webdav==0.0.1
+webhdfs = dvc-webhdfs==0.0.2
 # requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
-webhdfs_kerberos = requests-kerberos==0.14.0
+webhdfs_kerberos = dvc-webhdfs[kerberos]==0.0.2
 terraform = tpi[ssh]>=2.1.0
 tests =
     %(terraform)s

tests/unit/fs/test_azure.py

@@ -1,6 +1,5 @@
 import pytest
-
-from dvc.fs import AzureAuthError, AzureFileSystem
+from dvc_azure import AzureAuthError, AzureFileSystem
 
 container_name = "container-name"
 connection_string = (

tests/unit/fs/test_fs.py

@@ -1,15 +1,11 @@
 import pytest
+from dvc_hdfs import HDFSFileSystem
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
+from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
 
 from dvc.config import RemoteNotFoundError
-from dvc.fs import (
-    HDFSFileSystem,
-    LocalFileSystem,
-    S3FileSystem,
-    get_fs_cls,
-    get_fs_config,
-)
+from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
 
 
 @pytest.mark.parametrize(

tests/unit/fs/test_s3.py

@@ -1,8 +1,9 @@
 import os
 
 import pytest
+from dvc_s3 import S3FileSystem
 
-from dvc.fs import ConfigError, S3FileSystem
+from dvc.fs import ConfigError
 
 bucket_name = "bucket-name"
 prefix = "some/prefix"

tests/unit/output/test_load.py

@@ -1,9 +1,10 @@
 # pylint: disable=unsubscriptable-object
 
 import pytest
+from dvc_s3 import S3FileSystem
 
 from dvc import output
-from dvc.fs import LocalFileSystem, S3FileSystem
+from dvc.fs import LocalFileSystem
 from dvc.output import Output
 from dvc.stage import Stage
 

tests/unit/remote/test_oss.py

@@ -1,4 +1,4 @@
-from dvc.fs import OSSFileSystem
+from dvc_oss import OSSFileSystem
 
 bucket_name = "bucket-name"
 endpoint = "endpoint"

tests/unit/remote/test_remote.py

@@ -1,8 +1,10 @@
 import os
 
 import pytest
+from dvc_gs import GSFileSystem
+from dvc_s3 import S3FileSystem
 
-from dvc.fs import GSFileSystem, S3FileSystem, get_cloud_fs
+from dvc.fs import get_cloud_fs
 from dvc_data.db import get_index
 
 

tests/unit/remote/test_webdav.py

@@ -1,8 +1,9 @@
 from unittest.mock import patch
 
 import pytest
+from dvc_webdav import WebDAVFileSystem, WebDAVSFileSystem
 
-from dvc.fs import WebDAVFileSystem, WebDAVSFileSystem, get_cloud_fs
+from dvc.fs import get_cloud_fs
 from tests.utils.asserts import issubset
 
 url_fmt = "{scheme}://{user}@example.com/public.php/webdav"
@@ -65,7 +66,7 @@ def test_token():
     )
 
 
-@patch("dvc_objects.fs.implementations.webdav.ask_password")
+@patch("dvc_webdav.ask_password")
 def test_ask_password(ask_password_mocked):
     ask_password_mocked.return_value = "pass"
     host = "host"

tests/unit/remote/test_webhdfs.py

@@ -2,8 +2,7 @@
 
 import pytest
 import requests
-
-from dvc.fs import WebHDFSFileSystem
+from dvc_webhdfs import WebHDFSFileSystem
 
 host = "host"
 kerberos = False
@@ -16,8 +15,8 @@
 user = "test"
 
 
-@pytest.fixture()
-def webhdfs_config():
+@pytest.fixture(name="webhdfs_config")
+def fixture_webhdfs_config():
     url = f"webhdfs://{user}@{host}:{port}"
     url_config = WebHDFSFileSystem._get_kwargs_from_urls(url)
     return {