diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000..bc6092ea --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,29 @@ +name: Python package +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip nox + nox -s install + - name: Test with pytest + run: | + nox -s smoke + + - name: Publish package + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: andrewfulton9 + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..016388ce --- /dev/null +++ b/noxfile.py @@ -0,0 +1,18 @@ +import nox + + +@nox.session(python=False) +def develop(session): + session.install("flit") + session.run(*"flit install -s".split()) + + +@nox.session(python=False) +def install(session): + session.install(".") + + +@nox.session(python=False) +def smoke(session): + session.install(*"pytest".split()) + session.run(*"pytest --skiphdfs upath".split()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e5781070 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +build-backend = "flit_core.buildapi" +requires = ["flit_core>=2,<4"] + +[tool] +[tool.flit] +[tool.flit.metadata] +author = "Andrew Fulton" +author-email = "andrewfulton9@gmail.com" +classifiers = [] +home-page = "https://github.com/Quansight/universal_pathlib" +keywords = "" +license = "" +maintainer = "Andrew Fulton" +maintainer-email = "andrewfulton9@gmail.com" +module = "upath" +requires = ["fsspec"] +requires-python = ">=3.7" + +[tool.flit.metadata.urls] + +[tool.flit.metadata.requires-extra] +dev = [] +doc = [] +test = [ + "requests", + "s3fs", + "jupyter ", + "ipython", + "pytest", + "pylint", + "flake8", + "pyarrow", + "moto", + "hadoop-test-cluster", +] + +[tool.flit.scripts] + +[tool.flit.sdist] +include = [] + +[tool.flit.entrypoints] diff --git a/upath/__init__.py b/upath/__init__.py index 323b29ff..334d3529 100644 --- a/upath/__init__.py +++ b/upath/__init__.py @@ -1 +1,4 @@ +"""pathlib with fsspec""" +__version__ = "0.0.3" + from upath.core import UPath diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index 3129350d..c797b274 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -8,7 +8,6 @@ import pytest -from fsspec import filesystem from fsspec.implementations.local import LocalFileSystem from fsspec.registry import ( register_implementation, @@ -16,6 +15,28 @@ ) +def pytest_addoption(parser): + parser.addoption( + "--skiphdfs", + action="store_true", + default=False, + help="skip hdfs tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "hdfs: mark test as hdfs") + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--skiphdfs"): + return + skip_hdfs = pytest.mark.skip(reason="skipping hdfs") + for item in items: + if "hdfs" in item.keywords: + item.add_marker(skip_hdfs) + + class DummyTestFS(LocalFileSystem): protocol = "mock" @@ -28,16 +49,6 @@ def clear_registry(): finally: _registry.clear() - -# folder_structure = { -# 'folders': {'folder1': {'folders': {}, -# 'files': {'file1.txt': 'file1.txt', -# 'file2.txt': 'file2.txt'}} -# 'files': {'file1.txt': 'hello_world', -# 'file2.txt': 'hello_world'} - - -# } @pytest.fixture() def tempdir(clear_registry): @@ -45,8 +56,9 @@ def tempdir(clear_registry): tempdir = tempdir.name return tempdir + @pytest.fixture() -def local_testdir(tempdir, clear_registry): +def local_testdir(tempdir, clear_registry): tmp = Path(tempdir) tmp.mkdir() folder1 = tmp.joinpath('folder1') @@ -66,26 +78,28 @@ def local_testdir(tempdir, clear_registry): yield tempdir shutil.rmtree(tempdir) + @pytest.fixture(scope='session') def htcluster(): proc = subprocess.Popen(shlex.split("htcluster startup"), - stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) time.sleep(30) yield proc.terminate() proc.wait() proc1 = subprocess.Popen(shlex.split("htcluster shutdown"), - stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) proc1.terminate() proc1.wait() time.sleep(10) - + + @pytest.fixture() def hdfs(htcluster, tempdir, local_testdir): - pyarrow = pytest.importorskip('pyarrow') host, user, port = '0.0.0.0', 'hdfs', 9000 - #hdfs = filesystem('hdfs', host=host, user=user, port=port, driver='libhdfs3') hdfs = pyarrow.hdfs.connect(host='0.0.0.0', port=9000, user=user) hdfs.mkdir(tempdir, create_parents=True) for x in Path(local_testdir).glob('**/*'): @@ -96,10 +110,10 @@ def hdfs(htcluster, tempdir, local_testdir): with hdfs.open(str(x), 'wb') as f: f.write(text) else: - hdfs.mkdir(str(x)) + hdfs.mkdir(str(x)) hdfs.close() yield host, user, port - + @pytest.fixture(scope='session') def s3_server(): @@ -131,8 +145,7 @@ def s3_server(): time.sleep(0.1) # pragma: no cover anon = False s3so = dict(client_kwargs={'endpoint_url': endpoint_uri}, - use_listings_cache=False) - + use_listings_cache=False) yield anon, s3so proc.terminate() proc.wait() diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index dd6ea0b4..0f8874e0 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -194,7 +194,7 @@ def test_write_text(self, pathlib_base): path.write_text(s) assert path.read_text() == s - +@pytest.mark.hdfs class TestUPathHDFS(TestUpath): @pytest.fixture(autouse=True) @@ -254,6 +254,7 @@ def test_touch_unlink(self): # file doesn't exists, but missing_ok is True path.unlink(missing_ok=True) +@pytest.mark.hdfs def test_multiple_backend_paths(local_testdir, s3, hdfs): anon, s3so = s3 path = f's3:{local_testdir}'