Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions dvc_webhdfs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import threading
from typing import Any

# pylint:disable=abstract-method
from dvc.utils.objects import cached_property
from dvc_objects.fs.base import FileSystem
from funcy import cached_property, wrap_prop
from funcy import wrap_prop


class WebHDFSFileSystem(FileSystem):
protocol = "webhdfs"
REQUIRES = {"fsspec": "fsspec"}
PARAM_CHECKSUM = "checksum"

def __init__(self, fs=None, **kwargs: Any):
self._ssl_verify = kwargs.pop("ssl_verify", True)
super().__init__(fs, **kwargs)

@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
Expand All @@ -33,7 +39,6 @@ def _get_kwargs_from_urls(urlpath):
)

def _prepare_credentials(self, **config):
self._ssl_verify = config.pop("ssl_verify", True)
principal = config.pop("kerberos_principal", None)
if principal:
config["kerb_kwargs"] = {"principal": principal}
Expand Down
19 changes: 1 addition & 18 deletions dvc_webhdfs/tests/test_dvc.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from dvc.testing.remote_tests import ( # noqa, pylint: disable=unused-import
TestRemote,
)
from dvc.testing.workspace_tests import TestAdd as _TestAdd
from dvc.testing.workspace_tests import TestImport as _TestImport


Expand All @@ -22,7 +21,7 @@ def workspace(make_workspace):
class TestImport(_TestImport):
@pytest.fixture
def stage_md5(self):
return "3869631c193f0a3c206c6f04e84cb2b6"
return "f5f3369ef4fa451ddd6ab51f7fb42f7f"

@pytest.fixture
def is_object_storage(self):
Expand All @@ -31,19 +30,3 @@ def is_object_storage(self):
@pytest.fixture
def dir_md5(self):
return "32f6597da5c3c1dead9dc562faac09a2.dir"


class TestAdd(_TestAdd):
@pytest.fixture
def hash_name(self):
return "checksum"

@pytest.fixture
def hash_value(self):
return (
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000"
)

@pytest.fixture
def dir_hash_value(self):
pytest.skip("external outputs are broken for hdfs dirs")