#66: Corrected listing method for local operations

doc/changes/changes_0.3.0.md

@@ -10,6 +10,7 @@
 ## Bug Fixes
 
  - #63: Corrected uploading fileobject method of the mock bucketfs
+ - #66: Corrected listing method of localfs mock bucketfs  
 
 ## Refactoring
 

exasol_bucketfs_utils_python/list_files.py

@@ -23,12 +23,19 @@ def list_files_in_bucketfs(bucket_config: BucketConfig,
     response.raise_for_status()
 
     bucket_file_path_parts = Path(bucket_file_path).parts
+    path_exist = False
     files = []
     for path in response.text.split():
         path_parts = Path(path).parts
         if path_parts[:len(bucket_file_path_parts)] == bucket_file_path_parts:
+            path_exist = True
             relevant_parts = path_parts[len(bucket_file_path_parts):]
-            relevant_path = str(Path(*relevant_parts))
-            files.append(relevant_path)
+            if relevant_parts != ():
+                relevant_path = str(Path(*relevant_parts))
+                files.append(relevant_path)
+
+    if not path_exist:
+        raise FileNotFoundError(
+            f"No such file or directory '{bucket_file_path}' in bucketfs")
 
     return files

exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py

@@ -1,4 +1,4 @@
-from typing import Any, IO
+from typing import Any, IO, List
 from pathlib import PurePosixPath, Path
 from typing import Any
 import joblib
@@ -83,10 +83,16 @@ def read_file_from_bucketfs_via_joblib(self,
         return result
 
     def list_files_in_bucketfs(self,
-                               bucket_file_path: str) -> list:
-        path = self.get_complete_file_path_in_bucket(bucket_file_path)
-        Path(path).parent.mkdir(parents=True, exist_ok=True)
-        return ["."]
+                               bucket_file_path: str) -> List[str]:
+        complete_path = self.get_complete_file_path_in_bucket(bucket_file_path)
+        path = Path(complete_path)
+        if not path.exists():
+            raise FileNotFoundError(
+                f"No such file or directory '{bucket_file_path}' in bucketfs")
+
+        list_files = [str(p.relative_to(complete_path))
+                      for p in path.rglob('*') if p.is_file()]
+        return list_files
 
     def delete_file_in_bucketfs(
             self,

tests/conftest.py

@@ -3,5 +3,6 @@
 pytest_plugins = [
     "tests.fixtures.build_language_container_fixture",
     "tests.fixtures.bucketfs_location_fixture",
-    "tests.fixtures.upload_language_container_fixture"
+    "tests.fixtures.upload_language_container_fixture",
+    "tests.fixtures.prepare_bucket_fixture"
 ]

tests/fixtures/prepare_bucket_fixture.py

@@ -0,0 +1,32 @@
+import pytest
+from exasol_bucketfs_utils_python import upload
+from exasol_bucketfs_utils_python.bucket_config import BucketConfig
+from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
+from exasol_bucketfs_utils_python.bucketfs_connection_config import \
+    BucketFSConnectionConfig
+from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs
+
+
+@pytest.fixture(scope="module")
+def prepare_bucket():
+    connection_config = BucketFSConnectionConfig(
+        host="localhost", port=6666, user="w", pwd="write", is_https=False)
+    bucketfs_config = BucketFSConfig(
+        connection_config=connection_config, bucketfs_name="bfsdefault")
+    bucket_config = BucketConfig(
+        bucket_name="default", bucketfs_config=bucketfs_config)
+    test_string = "test_string"
+
+    path_list = ["path/in/bucket/file.txt", "path/file2.txt"]
+    try:
+        for path_in_bucket in path_list:
+            upload.upload_string_to_bucketfs(
+                bucket_config=bucket_config,
+                bucket_file_path=path_in_bucket,
+                string=test_string)
+        yield bucket_config
+    finally:
+        for path_in_bucket in path_list:
+            delete_testfile_from_bucketfs(
+                file_path=path_in_bucket,
+                bucket_config=bucket_config)

tests/test_delete_file.py

@@ -1,3 +1,5 @@
+import pytest
+
 from exasol_bucketfs_utils_python import upload, list_files, delete
 from exasol_bucketfs_utils_python.bucket_config import BucketConfig
 from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
@@ -42,9 +44,9 @@ def test_delete_files():
 
         # # check files not exist
         for bucket_path, expected in bucket_file_path_map.items():
-            listed_files = list_files.list_files_in_bucketfs(
-                bucket_config, bucket_path)
-            assert not listed_files
+            with pytest.raises(FileNotFoundError):
+                list_files.list_files_in_bucketfs(bucket_config, bucket_path)
+
     finally:
         for path_in_bucket in path_list:
             delete_testfile_from_bucketfs(

tests/test_localfs_mock_bucketfs_location.py

@@ -1,6 +1,8 @@
 from tempfile import TemporaryDirectory, NamedTemporaryFile
 from pathlib import Path, PurePosixPath
 
+import pytest
+
 from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import LocalFSMockBucketFSLocation
 
 def test_upload_download_string_from_different_instance():
@@ -111,3 +113,34 @@ def test_upload_read_fileobject ():
             output_test_byte_string = output_tmp_file.read()
 
     assert input_test_byte_string == output_test_byte_string
+
+
+def test_list_files_in_bucketfs():
+    with TemporaryDirectory() as path:
+        bucketfs_location_upload = LocalFSMockBucketFSLocation(path)
+        bucketfs_location_listing = LocalFSMockBucketFSLocation(path)
+
+        local_path = "path/in/"
+        bucket_files_path = [
+            f"{local_path}bucket/file.txt",
+            f"{local_path}file1.txt",
+            f"{local_path}file2.txt"]
+        test_value = TestValue("test_string")
+        for file_path in bucket_files_path:
+            bucketfs_location_upload.upload_object_to_bucketfs_via_joblib(
+                test_value, file_path)
+
+        expected_files = ['file1.txt', 'file2.txt', 'bucket/file.txt']
+        listed_files = bucketfs_location_listing\
+            .list_files_in_bucketfs(local_path)
+        assert set(listed_files) == set(expected_files)
+
+
+def test_list_files_not_found_error():
+    with TemporaryDirectory() as path:
+        bucketfs_location_listing = LocalFSMockBucketFSLocation(path)
+
+        local_path = "path/in/"
+        bucket_path = f"{local_path}not_existing_path"
+        with pytest.raises(FileNotFoundError):
+            bucketfs_location_listing.list_files_in_bucketfs(bucket_path)

tests/test_upload_list.py

@@ -1,41 +1,34 @@
-from exasol_bucketfs_utils_python import upload, list_files
-from exasol_bucketfs_utils_python.bucket_config import BucketConfig
-from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
-from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
-from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs
+import unittest
+import pytest
+from exasol_bucketfs_utils_python import list_files
 
 
-def test_list_files():
-    connection_config = BucketFSConnectionConfig(
-        host="localhost", port=6666, user="w", pwd="write", is_https=False)
-    bucketfs_config = BucketFSConfig(
-        connection_config=connection_config, bucketfs_name="bfsdefault")
-    bucket_config = BucketConfig(
-        bucket_name="default", bucketfs_config=bucketfs_config)
-    test_string = "test_string"
+@pytest.mark.parametrize(
+    "bucket_path,expected_list",
+    [
+        ("path", ["in/bucket/file.txt", "file2.txt"]),
+        ("path/", ["in/bucket/file.txt", "file2.txt"]),
+        ("path/in", ["bucket/file.txt"]),
+        ("path/in/", ["bucket/file.txt"]),
+        ("path/in/bucket", ["file.txt"]),
+        ("path/in/bucket/", ["file.txt"]),
+        ("path/in/bucket/file.txt", [])
+     ]
+)
+def test_list_files(bucket_path, expected_list, prepare_bucket):
+    bucket_config = prepare_bucket
+    assert set(expected_list) == set(list_files.list_files_in_bucketfs(
+        bucket_config, bucket_path))
 
-    path_list = ["path/in/bucket/file.txt", "path/file2.txt"]
-    try:
-        for path_in_bucket in path_list:
-            upload.upload_string_to_bucketfs(
-                bucket_config=bucket_config,
-                bucket_file_path=path_in_bucket,
-                string=test_string)
 
-        bucket_file_path_map = {
-            "path": ["in/bucket/file.txt", "file2.txt"],
-            "path/": ["in/bucket/file.txt", "file2.txt"],
-            "path/in": ["bucket/file.txt"],
-            "path/in/": ["bucket/file.txt"],
-            "path/in/bucket": ["file.txt"],
-            "path/in/bucket/": ["file.txt"],
-            "path/in/bucket/file.txt": ["."]
-        }
-        for bucket_path, expected in bucket_file_path_map.items():
-            assert expected == list_files.list_files_in_bucketfs(
-                bucket_config, bucket_path)
-    finally:
-        for path_in_bucket in path_list:
-            delete_testfile_from_bucketfs(
-                file_path=path_in_bucket,
-                bucket_config=bucket_config)
+def test_list_files_of_current_directory(prepare_bucket):
+    bucket_config = prepare_bucket
+    assert {"path/in/bucket/file.txt", "path/file2.txt"}.issubset(
+        set(list_files.list_files_in_bucketfs(bucket_config, ".")))
+
+
+def test_file_not_found_error(prepare_bucket):
+    bucket_config = prepare_bucket
+    bucket_path = "not_existing_path"
+    with pytest.raises(FileNotFoundError):
+        list_files.list_files_in_bucketfs(bucket_config, bucket_path)