diff --git a/doc/changes/changes_0.3.0.md b/doc/changes/changes_0.3.0.md index 7cacf4f4..cd15c1a7 100644 --- a/doc/changes/changes_0.3.0.md +++ b/doc/changes/changes_0.3.0.md @@ -10,6 +10,7 @@ ## Bug Fixes - #63: Corrected uploading fileobject method of the mock bucketfs + - #66: Corrected listing method of localfs mock bucketfs ## Refactoring diff --git a/exasol_bucketfs_utils_python/list_files.py b/exasol_bucketfs_utils_python/list_files.py index 20d247fc..7d3b6980 100644 --- a/exasol_bucketfs_utils_python/list_files.py +++ b/exasol_bucketfs_utils_python/list_files.py @@ -23,12 +23,19 @@ def list_files_in_bucketfs(bucket_config: BucketConfig, response.raise_for_status() bucket_file_path_parts = Path(bucket_file_path).parts + path_exist = False files = [] for path in response.text.split(): path_parts = Path(path).parts if path_parts[:len(bucket_file_path_parts)] == bucket_file_path_parts: + path_exist = True relevant_parts = path_parts[len(bucket_file_path_parts):] - relevant_path = str(Path(*relevant_parts)) - files.append(relevant_path) + if relevant_parts != (): + relevant_path = str(Path(*relevant_parts)) + files.append(relevant_path) + + if not path_exist: + raise FileNotFoundError( + f"No such file or directory '{bucket_file_path}' in bucketfs") return files diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 57fb8d2e..8bca3aa2 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -1,4 +1,4 @@ -from typing import Any, IO +from typing import Any, IO, List from pathlib import PurePosixPath, Path from typing import Any import joblib @@ -83,10 +83,16 @@ def read_file_from_bucketfs_via_joblib(self, return result def list_files_in_bucketfs(self, - bucket_file_path: str) -> list: - path = self.get_complete_file_path_in_bucket(bucket_file_path) - Path(path).parent.mkdir(parents=True, exist_ok=True) - return ["."] + bucket_file_path: str) -> List[str]: + complete_path = self.get_complete_file_path_in_bucket(bucket_file_path) + path = Path(complete_path) + if not path.exists(): + raise FileNotFoundError( + f"No such file or directory '{bucket_file_path}' in bucketfs") + + list_files = [str(p.relative_to(complete_path)) + for p in path.rglob('*') if p.is_file()] + return list_files def delete_file_in_bucketfs( self, diff --git a/tests/conftest.py b/tests/conftest.py index 5855bdb1..b0fffe62 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,5 +3,6 @@ pytest_plugins = [ "tests.fixtures.build_language_container_fixture", "tests.fixtures.bucketfs_location_fixture", - "tests.fixtures.upload_language_container_fixture" + "tests.fixtures.upload_language_container_fixture", + "tests.fixtures.prepare_bucket_fixture" ] \ No newline at end of file diff --git a/tests/fixtures/prepare_bucket_fixture.py b/tests/fixtures/prepare_bucket_fixture.py new file mode 100644 index 00000000..c26240fd --- /dev/null +++ b/tests/fixtures/prepare_bucket_fixture.py @@ -0,0 +1,32 @@ +import pytest +from exasol_bucketfs_utils_python import upload +from exasol_bucketfs_utils_python.bucket_config import BucketConfig +from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import \ + BucketFSConnectionConfig +from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs + + +@pytest.fixture(scope="module") +def prepare_bucket(): + connection_config = BucketFSConnectionConfig( + host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig( + connection_config=connection_config, bucketfs_name="bfsdefault") + bucket_config = BucketConfig( + bucket_name="default", bucketfs_config=bucketfs_config) + test_string = "test_string" + + path_list = ["path/in/bucket/file.txt", "path/file2.txt"] + try: + for path_in_bucket in path_list: + upload.upload_string_to_bucketfs( + bucket_config=bucket_config, + bucket_file_path=path_in_bucket, + string=test_string) + yield bucket_config + finally: + for path_in_bucket in path_list: + delete_testfile_from_bucketfs( + file_path=path_in_bucket, + bucket_config=bucket_config) \ No newline at end of file diff --git a/tests/test_delete_file.py b/tests/test_delete_file.py index 6fc40ba6..1df3cb53 100644 --- a/tests/test_delete_file.py +++ b/tests/test_delete_file.py @@ -1,3 +1,5 @@ +import pytest + from exasol_bucketfs_utils_python import upload, list_files, delete from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig @@ -42,9 +44,9 @@ def test_delete_files(): # # check files not exist for bucket_path, expected in bucket_file_path_map.items(): - listed_files = list_files.list_files_in_bucketfs( - bucket_config, bucket_path) - assert not listed_files + with pytest.raises(FileNotFoundError): + list_files.list_files_in_bucketfs(bucket_config, bucket_path) + finally: for path_in_bucket in path_list: delete_testfile_from_bucketfs( diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index 85c7bc20..debabb9c 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -1,6 +1,8 @@ from tempfile import TemporaryDirectory, NamedTemporaryFile from pathlib import Path, PurePosixPath +import pytest + from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import LocalFSMockBucketFSLocation def test_upload_download_string_from_different_instance(): @@ -111,3 +113,34 @@ def test_upload_read_fileobject (): output_test_byte_string = output_tmp_file.read() assert input_test_byte_string == output_test_byte_string + + +def test_list_files_in_bucketfs(): + with TemporaryDirectory() as path: + bucketfs_location_upload = LocalFSMockBucketFSLocation(path) + bucketfs_location_listing = LocalFSMockBucketFSLocation(path) + + local_path = "path/in/" + bucket_files_path = [ + f"{local_path}bucket/file.txt", + f"{local_path}file1.txt", + f"{local_path}file2.txt"] + test_value = TestValue("test_string") + for file_path in bucket_files_path: + bucketfs_location_upload.upload_object_to_bucketfs_via_joblib( + test_value, file_path) + + expected_files = ['file1.txt', 'file2.txt', 'bucket/file.txt'] + listed_files = bucketfs_location_listing\ + .list_files_in_bucketfs(local_path) + assert set(listed_files) == set(expected_files) + + +def test_list_files_not_found_error(): + with TemporaryDirectory() as path: + bucketfs_location_listing = LocalFSMockBucketFSLocation(path) + + local_path = "path/in/" + bucket_path = f"{local_path}not_existing_path" + with pytest.raises(FileNotFoundError): + bucketfs_location_listing.list_files_in_bucketfs(bucket_path) diff --git a/tests/test_upload_list.py b/tests/test_upload_list.py index caf76887..298630b0 100644 --- a/tests/test_upload_list.py +++ b/tests/test_upload_list.py @@ -1,41 +1,34 @@ -from exasol_bucketfs_utils_python import upload, list_files -from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig -from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs +import unittest +import pytest +from exasol_bucketfs_utils_python import list_files -def test_list_files(): - connection_config = BucketFSConnectionConfig( - host="localhost", port=6666, user="w", pwd="write", is_https=False) - bucketfs_config = BucketFSConfig( - connection_config=connection_config, bucketfs_name="bfsdefault") - bucket_config = BucketConfig( - bucket_name="default", bucketfs_config=bucketfs_config) - test_string = "test_string" +@pytest.mark.parametrize( + "bucket_path,expected_list", + [ + ("path", ["in/bucket/file.txt", "file2.txt"]), + ("path/", ["in/bucket/file.txt", "file2.txt"]), + ("path/in", ["bucket/file.txt"]), + ("path/in/", ["bucket/file.txt"]), + ("path/in/bucket", ["file.txt"]), + ("path/in/bucket/", ["file.txt"]), + ("path/in/bucket/file.txt", []) + ] +) +def test_list_files(bucket_path, expected_list, prepare_bucket): + bucket_config = prepare_bucket + assert set(expected_list) == set(list_files.list_files_in_bucketfs( + bucket_config, bucket_path)) - path_list = ["path/in/bucket/file.txt", "path/file2.txt"] - try: - for path_in_bucket in path_list: - upload.upload_string_to_bucketfs( - bucket_config=bucket_config, - bucket_file_path=path_in_bucket, - string=test_string) - bucket_file_path_map = { - "path": ["in/bucket/file.txt", "file2.txt"], - "path/": ["in/bucket/file.txt", "file2.txt"], - "path/in": ["bucket/file.txt"], - "path/in/": ["bucket/file.txt"], - "path/in/bucket": ["file.txt"], - "path/in/bucket/": ["file.txt"], - "path/in/bucket/file.txt": ["."] - } - for bucket_path, expected in bucket_file_path_map.items(): - assert expected == list_files.list_files_in_bucketfs( - bucket_config, bucket_path) - finally: - for path_in_bucket in path_list: - delete_testfile_from_bucketfs( - file_path=path_in_bucket, - bucket_config=bucket_config) +def test_list_files_of_current_directory(prepare_bucket): + bucket_config = prepare_bucket + assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset( + set(list_files.list_files_in_bucketfs(bucket_config, "."))) + + +def test_file_not_found_error(prepare_bucket): + bucket_config = prepare_bucket + bucket_path = "not_existing_path" + with pytest.raises(FileNotFoundError): + list_files.list_files_in_bucketfs(bucket_config, bucket_path)