Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GH-38431: [Python][CI] Update fs.type_name checks for s3fs tests #38455

Merged
merged 2 commits into from
Oct 26, 2023
Merged

GH-38431: [Python][CI] Update fs.type_name checks for s3fs tests #38455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ff0de95
Update checks for s3fs type_name
AlenkaF Oct 25, 2023
3a53d11
Update pin for s3fs
AlenkaF Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/conda_env_python.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ numpy>=1.16.6
pytest
pytest-faulthandler
pytest-lazy-fixture
s3fs>=2021.8.0
s3fs>=2023.10.0
setuptools
setuptools_scm<8.0.0
12 changes: 6 additions & 6 deletions python/pyarrow/tests/test_fs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ def check_mtime_or_absent(file_info):


def skip_fsspec_s3fs(fs):
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
pytest.xfail(reason="Not working with fsspec's s3fs")


Expand Down Expand Up @@ -631,7 +631,7 @@ def test_get_file_info(fs, pathfn):
assert aaa_info.path == aaa
assert 'aaa' in repr(aaa_info)
assert aaa_info.extension == ''
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs doesn't create empty directories
assert aaa_info.type == FileType.NotFound
else:
Expand All @@ -646,7 +646,7 @@ def test_get_file_info(fs, pathfn):
assert bb_info.type == FileType.File
assert 'FileType.File' in repr(bb_info)
assert bb_info.size == 0
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
check_mtime(bb_info)

assert c_info.path == str(c)
Expand All @@ -655,7 +655,7 @@ def test_get_file_info(fs, pathfn):
assert c_info.type == FileType.File
assert 'FileType.File' in repr(c_info)
assert c_info.size == 4
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
check_mtime(c_info)

assert zzz_info.path == str(zzz)
Expand Down Expand Up @@ -698,7 +698,7 @@ def test_get_file_info_with_selector(fs, pathfn):
assert selector.base_dir == base_dir

infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs only lists directories if they are not empty
len(infos) == 4
else:
Expand All @@ -719,7 +719,7 @@ def test_get_file_info_with_selector(fs, pathfn):
selector = FileSelector(base_dir, recursive=False)

infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs only lists directories if they are not empty
assert len(infos) == 3
else:
Expand Down