From 9a56e221a79196e4595ee35e0ec20dc8543aa3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 23 Aug 2023 16:52:36 +0545 Subject: [PATCH 1/2] fix pylint error and remove TestAdd --- dvc_webhdfs/__init__.py | 9 +++++++-- dvc_webhdfs/tests/test_dvc.py | 17 ----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/dvc_webhdfs/__init__.py b/dvc_webhdfs/__init__.py index 3bc9469..dbcb237 100644 --- a/dvc_webhdfs/__init__.py +++ b/dvc_webhdfs/__init__.py @@ -1,8 +1,10 @@ import threading +from typing import Any # pylint:disable=abstract-method +from dvc.utils.objects import cached_property from dvc_objects.fs.base import FileSystem -from funcy import cached_property, wrap_prop +from funcy import wrap_prop class WebHDFSFileSystem(FileSystem): @@ -10,6 +12,10 @@ class WebHDFSFileSystem(FileSystem): REQUIRES = {"fsspec": "fsspec"} PARAM_CHECKSUM = "checksum" + def __init__(self, fs=None, **kwargs: Any): + self._ssl_verify = kwargs.pop("ssl_verify", True) + super().__init__(fs, **kwargs) + @classmethod def _strip_protocol(cls, path: str) -> str: from fsspec.utils import infer_storage_options @@ -33,7 +39,6 @@ def _get_kwargs_from_urls(urlpath): ) def _prepare_credentials(self, **config): - self._ssl_verify = config.pop("ssl_verify", True) principal = config.pop("kerberos_principal", None) if principal: config["kerb_kwargs"] = {"principal": principal} diff --git a/dvc_webhdfs/tests/test_dvc.py b/dvc_webhdfs/tests/test_dvc.py index 398d051..f3d62a7 100644 --- a/dvc_webhdfs/tests/test_dvc.py +++ b/dvc_webhdfs/tests/test_dvc.py @@ -5,7 +5,6 @@ from dvc.testing.remote_tests import ( # noqa, pylint: disable=unused-import TestRemote, ) -from dvc.testing.workspace_tests import TestAdd as _TestAdd from dvc.testing.workspace_tests import TestImport as _TestImport @@ -31,19 +30,3 @@ def is_object_storage(self): @pytest.fixture def dir_md5(self): return "32f6597da5c3c1dead9dc562faac09a2.dir" - - -class TestAdd(_TestAdd): - @pytest.fixture - def hash_name(self): - return "checksum" - - @pytest.fixture - def hash_value(self): - return ( - "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000" - ) - - @pytest.fixture - def dir_hash_value(self): - pytest.skip("external outputs are broken for hdfs dirs") From 52a452339c3cc0aabd90d37c2eaf9ca2aa3cd813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 23 Aug 2023 17:15:47 +0545 Subject: [PATCH 2/2] update hash in the TestImport test --- dvc_webhdfs/tests/test_dvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc_webhdfs/tests/test_dvc.py b/dvc_webhdfs/tests/test_dvc.py index f3d62a7..3dccb5b 100644 --- a/dvc_webhdfs/tests/test_dvc.py +++ b/dvc_webhdfs/tests/test_dvc.py @@ -21,7 +21,7 @@ def workspace(make_workspace): class TestImport(_TestImport): @pytest.fixture def stage_md5(self): - return "3869631c193f0a3c206c6f04e84cb2b6" + return "f5f3369ef4fa451ddd6ab51f7fb42f7f" @pytest.fixture def is_object_storage(self):