From 2ff2f049933724700cfa2dd5110102b69abce6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Mon, 16 May 2022 21:17:07 +0200 Subject: [PATCH 01/13] Updated listing method --- .../localfs_mock_bucketfs_location.py | 4 ++-- tests/test_localfs_mock_bucketfs_location.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 57fb8d2e..c60dbba0 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -85,8 +85,8 @@ def read_file_from_bucketfs_via_joblib(self, def list_files_in_bucketfs(self, bucket_file_path: str) -> list: path = self.get_complete_file_path_in_bucket(bucket_file_path) - Path(path).parent.mkdir(parents=True, exist_ok=True) - return ["."] + list_files = [str(p.name) for p in Path(path).iterdir()] + return list_files def delete_file_in_bucketfs( self, diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index 85c7bc20..b07195ed 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -111,3 +111,16 @@ def test_upload_read_fileobject (): output_test_byte_string = output_tmp_file.read() assert input_test_byte_string == output_test_byte_string + + +def test_list_files_in_bucketfs(): + with TemporaryDirectory() as path: + bucketfs_location_upload = LocalFSMockBucketFSLocation(path) + bucketfs_location_listing = LocalFSMockBucketFSLocation(path) + bucket_file_path = "test_file.txt" + test_value = TestValue("test_string") + bucketfs_location_upload.upload_object_to_bucketfs_via_joblib( + test_value, bucket_file_path) + + files = bucketfs_location_listing.list_files_in_bucketfs(path) + assert bucket_file_path in files From 51dd04c22e7ea70df9f79211aae378e969815d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Mon, 16 May 2022 21:19:17 +0200 Subject: [PATCH 02/13] Updated changelog --- doc/changes/changes_0.3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/changes_0.3.0.md b/doc/changes/changes_0.3.0.md index 7cacf4f4..cd15c1a7 100644 --- a/doc/changes/changes_0.3.0.md +++ b/doc/changes/changes_0.3.0.md @@ -10,6 +10,7 @@ ## Bug Fixes - #63: Corrected uploading fileobject method of the mock bucketfs + - #66: Corrected listing method of localfs mock bucketfs ## Refactoring From efb04b4d69532f74aecae0ca07e8450c006591e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 09:25:25 +0200 Subject: [PATCH 03/13] Visited subfolders recursively --- .../localfs_mock_bucketfs_location.py | 4 +++- tests/test_localfs_mock_bucketfs_location.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index c60dbba0..9a691b06 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -85,7 +85,9 @@ def read_file_from_bucketfs_via_joblib(self, def list_files_in_bucketfs(self, bucket_file_path: str) -> list: path = self.get_complete_file_path_in_bucket(bucket_file_path) - list_files = [str(p.name) for p in Path(path).iterdir()] + list_files = [str(p.relative_to(bucket_file_path)) + for p in Path(path).rglob('*') if p.is_file()] + return list_files def delete_file_in_bucketfs( diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index b07195ed..05e459e9 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -117,10 +117,15 @@ def test_list_files_in_bucketfs(): with TemporaryDirectory() as path: bucketfs_location_upload = LocalFSMockBucketFSLocation(path) bucketfs_location_listing = LocalFSMockBucketFSLocation(path) - bucket_file_path = "test_file.txt" + + bucket_files_path = [ + "path/in/bucket/file.txt", + "path/in/file.txt" + "file.txt"] test_value = TestValue("test_string") - bucketfs_location_upload.upload_object_to_bucketfs_via_joblib( - test_value, bucket_file_path) + for file_path in bucket_files_path: + bucketfs_location_upload.upload_object_to_bucketfs_via_joblib( + test_value, file_path) - files = bucketfs_location_listing.list_files_in_bucketfs(path) - assert bucket_file_path in files + listed_files = bucketfs_location_listing.list_files_in_bucketfs(path) + assert set(listed_files) == set(bucket_files_path) From 09ee42d456523d2c37e30c2201b3a6d947dc468e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 09:46:43 +0200 Subject: [PATCH 04/13] Fixed error in tests --- tests/test_upload_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index caf76887..df3639d3 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -32,8 +32,8 @@ def test_list_files(): "path/in/bucket/file.txt": ["."] } for bucket_path, expected in bucket_file_path_map.items(): - assert expected == list_files.list_files_in_bucketfs( - bucket_config, bucket_path) + assert set(expected) == set(list_files.list_files_in_bucketfs( + bucket_config, bucket_path)) finally: for path_in_bucket in path_list: delete_testfile_from_bucketfs( From 694b7bbfff4798cb860c6bf3d28698ba3e44769a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 13:34:27 +0200 Subject: [PATCH 05/13] Update listing current path --- exasol_bucketfs_utils_python/list_files.py | 5 +++-- tests/test_upload_list.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/exasol_bucketfs_utils_python/list_files.py b/exasol_bucketfs_utils_python/list_files.py index 20d247fc..6316d131 100644 --- a/exasol_bucketfs_utils_python/list_files.py +++ b/exasol_bucketfs_utils_python/list_files.py @@ -28,7 +28,8 @@ def list_files_in_bucketfs(bucket_config: BucketConfig, path_parts = Path(path).parts if path_parts[:len(bucket_file_path_parts)] == bucket_file_path_parts: relevant_parts = path_parts[len(bucket_file_path_parts):] - relevant_path = str(Path(*relevant_parts)) - files.append(relevant_path) + if relevant_parts != (): + relevant_path = str(Path(*relevant_parts)) + files.append(relevant_path) return files diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index df3639d3..1baca37f 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -29,7 +29,8 @@ def test_list_files(): "path/in/": ["bucket/file.txt"], "path/in/bucket": ["file.txt"], "path/in/bucket/": ["file.txt"], - "path/in/bucket/file.txt": ["."] + "path/in/bucket/file.txt": [], + ".": ["path/in/bucket/file.txt", "path/file2.txt"] } for bucket_path, expected in bucket_file_path_map.items(): assert set(expected) == set(list_files.list_files_in_bucketfs( From 83812e26316d763d6f167f3c9c5689776115457a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 13:40:54 +0200 Subject: [PATCH 06/13] Corrected path for localfs --- exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 9a691b06..87a48cf6 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -85,7 +85,7 @@ def read_file_from_bucketfs_via_joblib(self, def list_files_in_bucketfs(self, bucket_file_path: str) -> list: path = self.get_complete_file_path_in_bucket(bucket_file_path) - list_files = [str(p.relative_to(bucket_file_path)) + list_files = [str(p.relative_to(path)) for p in Path(path).rglob('*') if p.is_file()] return list_files From 21be02721f56f0cdf296902c179b155c58337da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 14:29:55 +0200 Subject: [PATCH 07/13] Fix checking error --- tests/test_upload_list.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index 1baca37f..f2398ef7 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -33,8 +33,9 @@ def test_list_files(): ".": ["path/in/bucket/file.txt", "path/file2.txt"] } for bucket_path, expected in bucket_file_path_map.items(): - assert set(expected) == set(list_files.list_files_in_bucketfs( - bucket_config, bucket_path)) + assert set(expected).issubset( + set(list_files.list_files_in_bucketfs( + bucket_config, bucket_path))) finally: for path_in_bucket in path_list: delete_testfile_from_bucketfs( From e17a32865c2ca12b183776932eddc1109fc78c20 Mon Sep 17 00:00:00 2001 From: Umit Cavus Buyuksahin Date: Tue, 17 May 2022 20:39:04 +0200 Subject: [PATCH 08/13] Update exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py Co-authored-by: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> --- exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 87a48cf6..aceb0c52 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -83,7 +83,7 @@ def read_file_from_bucketfs_via_joblib(self, return result def list_files_in_bucketfs(self, - bucket_file_path: str) -> list: + bucket_file_path: str) -> List[str]: path = self.get_complete_file_path_in_bucket(bucket_file_path) list_files = [str(p.relative_to(path)) for p in Path(path).rglob('*') if p.is_file()] From 60cd930b4da69f44a192343c2888ce2c0f3c3efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 17 May 2022 20:49:48 +0200 Subject: [PATCH 09/13] Updated tests --- .../localfs_mock_bucketfs_location.py | 2 +- tests/test_localfs_mock_bucketfs_location.py | 13 ++++++++----- tests/test_upload_list.py | 11 ++++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index aceb0c52..9747c8c7 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -1,4 +1,4 @@ -from typing import Any, IO +from typing import Any, IO, List from pathlib import PurePosixPath, Path from typing import Any import joblib diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index 05e459e9..d0109a30 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -118,14 +118,17 @@ def test_list_files_in_bucketfs(): bucketfs_location_upload = LocalFSMockBucketFSLocation(path) bucketfs_location_listing = LocalFSMockBucketFSLocation(path) + local_path = "path/in/" bucket_files_path = [ - "path/in/bucket/file.txt", - "path/in/file.txt" - "file.txt"] + f"{local_path}bucket/file.txt", + f"{local_path}file1.txt", + f"{local_path}file2.txt"] test_value = TestValue("test_string") for file_path in bucket_files_path: bucketfs_location_upload.upload_object_to_bucketfs_via_joblib( test_value, file_path) - listed_files = bucketfs_location_listing.list_files_in_bucketfs(path) - assert set(listed_files) == set(bucket_files_path) + expected_files = ['file1.txt', 'file2.txt', 'bucket/file.txt'] + listed_files = bucketfs_location_listing\ + .list_files_in_bucketfs(local_path) + assert set(listed_files) == set(expected_files) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index f2398ef7..63d54a97 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -29,13 +29,14 @@ def test_list_files(): "path/in/": ["bucket/file.txt"], "path/in/bucket": ["file.txt"], "path/in/bucket/": ["file.txt"], - "path/in/bucket/file.txt": [], - ".": ["path/in/bucket/file.txt", "path/file2.txt"] + "path/in/bucket/file.txt": [] } for bucket_path, expected in bucket_file_path_map.items(): - assert set(expected).issubset( - set(list_files.list_files_in_bucketfs( - bucket_config, bucket_path))) + assert set(expected) == set(list_files.list_files_in_bucketfs( + bucket_config, bucket_path)) + + assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset( + set(list_files.list_files_in_bucketfs(bucket_config, "."))) finally: for path_in_bucket in path_list: delete_testfile_from_bucketfs( From f570cf6afa1c882676444605b45672e0c6e603ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Wed, 18 May 2022 10:38:39 +0200 Subject: [PATCH 10/13] Raised file not found exception --- exasol_bucketfs_utils_python/list_files.py | 6 ++++++ .../localfs_mock_bucketfs_location.py | 12 ++++++++---- tests/test_upload_list.py | 5 ++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/exasol_bucketfs_utils_python/list_files.py b/exasol_bucketfs_utils_python/list_files.py index 6316d131..7d3b6980 100644 --- a/exasol_bucketfs_utils_python/list_files.py +++ b/exasol_bucketfs_utils_python/list_files.py @@ -23,13 +23,19 @@ def list_files_in_bucketfs(bucket_config: BucketConfig, response.raise_for_status() bucket_file_path_parts = Path(bucket_file_path).parts + path_exist = False files = [] for path in response.text.split(): path_parts = Path(path).parts if path_parts[:len(bucket_file_path_parts)] == bucket_file_path_parts: + path_exist = True relevant_parts = path_parts[len(bucket_file_path_parts):] if relevant_parts != (): relevant_path = str(Path(*relevant_parts)) files.append(relevant_path) + if not path_exist: + raise FileNotFoundError( + f"No such file or directory '{bucket_file_path}' in bucketfs") + return files diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 9747c8c7..8bca3aa2 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -84,10 +84,14 @@ def read_file_from_bucketfs_via_joblib(self, def list_files_in_bucketfs(self, bucket_file_path: str) -> List[str]: - path = self.get_complete_file_path_in_bucket(bucket_file_path) - list_files = [str(p.relative_to(path)) - for p in Path(path).rglob('*') if p.is_file()] - + complete_path = self.get_complete_file_path_in_bucket(bucket_file_path) + path = Path(complete_path) + if not path.exists(): + raise FileNotFoundError( + f"No such file or directory '{bucket_file_path}' in bucketfs") + + list_files = [str(p.relative_to(complete_path)) + for p in path.rglob('*') if p.is_file()] return list_files def delete_file_in_bucketfs( diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index 63d54a97..3c9a86f4 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -1,7 +1,8 @@ from exasol_bucketfs_utils_python import upload, list_files from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config \ + import BucketFSConnectionConfig from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs @@ -22,6 +23,8 @@ def test_list_files(): bucket_file_path=path_in_bucket, string=test_string) + + bucket_file_path_map = { "path": ["in/bucket/file.txt", "file2.txt"], "path/": ["in/bucket/file.txt", "file2.txt"], From 9e990d4b654c5d9eaae121a6783f54d611838f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Wed, 18 May 2022 11:21:50 +0200 Subject: [PATCH 11/13] Added tests for file not found error --- tests/conftest.py | 3 +- tests/fixtures/prepare_bucket_fixture.py | 32 +++++++++ tests/test_localfs_mock_bucketfs_location.py | 12 ++++ tests/test_upload_list.py | 68 ++++++++------------ 4 files changed, 72 insertions(+), 43 deletions(-) create mode 100644 tests/fixtures/prepare_bucket_fixture.py diff --git a/tests/conftest.py b/tests/conftest.py index 5855bdb1..b0fffe62 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,5 +3,6 @@ pytest_plugins = [ "tests.fixtures.build_language_container_fixture", "tests.fixtures.bucketfs_location_fixture", - "tests.fixtures.upload_language_container_fixture" + "tests.fixtures.upload_language_container_fixture", + "tests.fixtures.prepare_bucket_fixture" ] \ No newline at end of file diff --git a/tests/fixtures/prepare_bucket_fixture.py b/tests/fixtures/prepare_bucket_fixture.py new file mode 100644 index 00000000..c26240fd --- /dev/null +++ b/tests/fixtures/prepare_bucket_fixture.py @@ -0,0 +1,32 @@ +import pytest +from exasol_bucketfs_utils_python import upload +from exasol_bucketfs_utils_python.bucket_config import BucketConfig +from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import \ + BucketFSConnectionConfig +from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs + + +@pytest.fixture(scope="module") +def prepare_bucket(): + connection_config = BucketFSConnectionConfig( + host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig( + connection_config=connection_config, bucketfs_name="bfsdefault") + bucket_config = BucketConfig( + bucket_name="default", bucketfs_config=bucketfs_config) + test_string = "test_string" + + path_list = ["path/in/bucket/file.txt", "path/file2.txt"] + try: + for path_in_bucket in path_list: + upload.upload_string_to_bucketfs( + bucket_config=bucket_config, + bucket_file_path=path_in_bucket, + string=test_string) + yield bucket_config + finally: + for path_in_bucket in path_list: + delete_testfile_from_bucketfs( + file_path=path_in_bucket, + bucket_config=bucket_config) \ No newline at end of file diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index d0109a30..debabb9c 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -1,6 +1,8 @@ from tempfile import TemporaryDirectory, NamedTemporaryFile from pathlib import Path, PurePosixPath +import pytest + from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import LocalFSMockBucketFSLocation def test_upload_download_string_from_different_instance(): @@ -132,3 +134,13 @@ def test_list_files_in_bucketfs(): listed_files = bucketfs_location_listing\ .list_files_in_bucketfs(local_path) assert set(listed_files) == set(expected_files) + + +def test_list_files_not_found_error(): + with TemporaryDirectory() as path: + bucketfs_location_listing = LocalFSMockBucketFSLocation(path) + + local_path = "path/in/" + bucket_path = f"{local_path}not_existing_path" + with pytest.raises(FileNotFoundError): + bucketfs_location_listing.list_files_in_bucketfs(bucket_path) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index 3c9a86f4..ed27451c 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -1,47 +1,31 @@ -from exasol_bucketfs_utils_python import upload, list_files -from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config \ - import BucketFSConnectionConfig -from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs +import unittest +import pytest +from exasol_bucketfs_utils_python import list_files -def test_list_files(): - connection_config = BucketFSConnectionConfig( - host="localhost", port=6666, user="w", pwd="write", is_https=False) - bucketfs_config = BucketFSConfig( - connection_config=connection_config, bucketfs_name="bfsdefault") - bucket_config = BucketConfig( - bucket_name="default", bucketfs_config=bucketfs_config) - test_string = "test_string" +@pytest.mark.parametrize( + "bucket_path,expected_list", + [ + ("path", ["in/bucket/file.txt", "file2.txt"]), + ("path/", ["in/bucket/file.txt", "file2.txt"]), + ("path/in", ["bucket/file.txt"]), + ("path/in/", ["bucket/file.txt"]), + ("path/in/bucket", ["file.txt"]), + ("path/in/bucket/", ["file.txt"]), + ("path/in/bucket/file.txt", []) + ] +) +def test_list_files(bucket_path, expected_list, prepare_bucket): + bucket_config = prepare_bucket + assert set(expected_list) == set(list_files.list_files_in_bucketfs( + bucket_config, bucket_path)) - path_list = ["path/in/bucket/file.txt", "path/file2.txt"] - try: - for path_in_bucket in path_list: - upload.upload_string_to_bucketfs( - bucket_config=bucket_config, - bucket_file_path=path_in_bucket, - string=test_string) + assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset( + set(list_files.list_files_in_bucketfs(bucket_config, "."))) - - bucket_file_path_map = { - "path": ["in/bucket/file.txt", "file2.txt"], - "path/": ["in/bucket/file.txt", "file2.txt"], - "path/in": ["bucket/file.txt"], - "path/in/": ["bucket/file.txt"], - "path/in/bucket": ["file.txt"], - "path/in/bucket/": ["file.txt"], - "path/in/bucket/file.txt": [] - } - for bucket_path, expected in bucket_file_path_map.items(): - assert set(expected) == set(list_files.list_files_in_bucketfs( - bucket_config, bucket_path)) - - assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset( - set(list_files.list_files_in_bucketfs(bucket_config, "."))) - finally: - for path_in_bucket in path_list: - delete_testfile_from_bucketfs( - file_path=path_in_bucket, - bucket_config=bucket_config) +def test_file_not_found_error(prepare_bucket): + bucket_config = prepare_bucket + bucket_path = "not_existing_path" + with pytest.raises(FileNotFoundError): + list_files.list_files_in_bucketfs(bucket_config, bucket_path) From 6b5fe31aeb11cca26f9d75b6543280e52c21cb18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Wed, 18 May 2022 11:36:03 +0200 Subject: [PATCH 12/13] Fix deletion test --- tests/test_delete_file.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_delete_file.py b/tests/test_delete_file.py index 6fc40ba6..1df3cb53 100644 --- a/tests/test_delete_file.py +++ b/tests/test_delete_file.py @@ -1,3 +1,5 @@ +import pytest + from exasol_bucketfs_utils_python import upload, list_files, delete from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig @@ -42,9 +44,9 @@ def test_delete_files(): # # check files not exist for bucket_path, expected in bucket_file_path_map.items(): - listed_files = list_files.list_files_in_bucketfs( - bucket_config, bucket_path) - assert not listed_files + with pytest.raises(FileNotFoundError): + list_files.list_files_in_bucketfs(bucket_config, bucket_path) + finally: for path_in_bucket in path_list: delete_testfile_from_bucketfs( From a2b3cc0647f8a44ae4148fe1eab28a6fd515e8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Wed, 18 May 2022 13:34:48 +0200 Subject: [PATCH 13/13] Seperated list files tests --- tests/test_upload_list.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index ed27451c..298630b0 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -20,6 +20,9 @@ def test_list_files(bucket_path, expected_list, prepare_bucket): assert set(expected_list) == set(list_files.list_files_in_bucketfs( bucket_config, bucket_path)) + +def test_list_files_of_current_directory(prepare_bucket): + bucket_config = prepare_bucket assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset( set(list_files.list_files_in_bucketfs(bucket_config, ".")))