From ff0615b07548c96b56cd62d1a48f14a79a70cec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 11:09:17 +0200 Subject: [PATCH 1/9] Added generating udf path method --- .../abstract_bucketfs_location.py | 8 ++++++++ .../bucketfs_location.py | 10 ++++++++-- exasol_bucketfs_utils_python/bucketfs_utils.py | 10 +++++----- .../localfs_mock_bucketfs_location.py | 18 +++++++++++++++++- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py index f761f232..2e7773be 100644 --- a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py @@ -2,6 +2,7 @@ from typing import Any, Tuple, IO, Iterable from pathlib import PurePosixPath, Path from urllib.parse import ParseResult +from typing import Union class AbstractBucketFSLocation(ABC): @@ -10,6 +11,13 @@ class AbstractBucketFSLocation(ABC): fileobjects and joblib objects. Also able to read files from the BucketFS directly, if called from inside a UDF. """ + + @abstractmethod + def generate_bucket_udf_path( + self, path_in_bucket: Union[None, str, PurePosixPath]) \ + -> PurePosixPath: + pass + @abstractmethod def download_from_bucketfs_to_string( self, diff --git a/exasol_bucketfs_utils_python/bucketfs_location.py b/exasol_bucketfs_utils_python/bucketfs_location.py index 8f6e4865..e471e593 100644 --- a/exasol_bucketfs_utils_python/bucketfs_location.py +++ b/exasol_bucketfs_utils_python/bucketfs_location.py @@ -1,8 +1,8 @@ -from typing import Any, Tuple, IO, Iterable +from typing import Any, Tuple, IO, Iterable, Union from pathlib import PurePosixPath, Path from urllib.parse import ParseResult from exasol_bucketfs_utils_python import download, upload, list_files, \ - delete + delete, bucketfs_utils from exasol_bucketfs_utils_python import load_file_from_local_fs as from_BFS from exasol_bucketfs_utils_python.bucket_config import BucketConfig @@ -25,6 +25,12 @@ def __init__(self, bucket_config: BucketConfig, base_path: PurePosixPath): self.base_path = base_path self.bucket_config = bucket_config + def generate_bucket_udf_path( + self, path_in_bucket: Union[None, str, PurePosixPath]) \ + -> PurePosixPath: + return bucketfs_utils.generate_bucket_udf_path( + self.bucket_config, path_in_bucket) + def get_complete_file_path_in_bucket( self, bucket_file_path: str) -> str: diff --git a/exasol_bucketfs_utils_python/bucketfs_utils.py b/exasol_bucketfs_utils_python/bucketfs_utils.py index 7c6f71e5..48ee7d43 100644 --- a/exasol_bucketfs_utils_python/bucketfs_utils.py +++ b/exasol_bucketfs_utils_python/bucketfs_utils.py @@ -14,7 +14,7 @@ def _encode_url_part(part: str) -> str: return urlencoded -def _correct_path_in_bucket_for_archives(path_in_bucket: PurePosixPath) \ +def correct_path_in_bucket_for_archives(path_in_bucket: PurePosixPath) \ -> PurePosixPath: for extension in ARCHIVE_EXTENSIONS: if path_in_bucket.name.endswith(extension): @@ -24,7 +24,7 @@ def _correct_path_in_bucket_for_archives(path_in_bucket: PurePosixPath) \ return path_in_bucket -def _make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) \ +def make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) \ -> PurePosixPath: path_in_bucket = PurePosixPath(path_in_bucket) if path_in_bucket.is_absolute(): @@ -62,8 +62,8 @@ def generate_bucket_udf_path( path = PurePosixPath(bucketfs_path, bucket_config.bucket_name) if path_in_bucket is not None: - path_in_bucket = _make_path_relative(path_in_bucket) - path_in_bucket = _correct_path_in_bucket_for_archives(path_in_bucket) + path_in_bucket = make_path_relative(path_in_bucket) + path_in_bucket = correct_path_in_bucket_for_archives(path_in_bucket) else: path_in_bucket = "" path = PurePosixPath(path, path_in_bucket) @@ -120,7 +120,7 @@ def generate_bucket_http_url( url = generate_bucketfs_http_url(bucket_config.bucketfs_config, with_credentials) if path_in_bucket is not None: - path_in_bucket = _make_path_relative(path_in_bucket) + path_in_bucket = make_path_relative(path_in_bucket) else: path_in_bucket = "" encoded_bucket_and_path_in_bucket = \ diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 8bca3aa2..bb2fabe0 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -1,7 +1,9 @@ -from typing import Any, IO, List +from typing import Any, IO, List, Union from pathlib import PurePosixPath, Path from typing import Any import joblib + +from exasol_bucketfs_utils_python import bucketfs_utils from exasol_bucketfs_utils_python.abstract_bucketfs_location import \ AbstractBucketFSLocation @@ -20,6 +22,20 @@ def __init__(self, base_path: PurePosixPath): def get_complete_file_path_in_bucket(self, bucket_file_path) -> str: return str(PurePosixPath(self.base_path, bucket_file_path)) + def generate_bucket_udf_path( + self, path_in_bucket: Union[None, str, PurePosixPath]) \ + -> PurePosixPath: + + if path_in_bucket is not None: + path_in_bucket = bucketfs_utils.\ + make_path_relative(path_in_bucket) + path_in_bucket = bucketfs_utils.\ + correct_path_in_bucket_for_archives(path_in_bucket) + else: + path_in_bucket = "" + path = PurePosixPath(self.base_path, path_in_bucket) + return path + def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: with open(self.get_complete_file_path_in_bucket( bucket_file_path), "rt") as f: From 94158c6414f6c13748cea3c4a7aad400e8acd9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 11:09:27 +0200 Subject: [PATCH 2/9] Added tests --- tests/test_bucketfs_location.py | 24 ++++++++++++++++++-- tests/test_localfs_mock_bucketfs_location.py | 20 ++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/test_bucketfs_location.py b/tests/test_bucketfs_location.py index 5edcd6ea..b5f5c31d 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/test_bucketfs_location.py @@ -1,9 +1,7 @@ from pathlib import PurePosixPath - from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig - from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation import pytest import textwrap @@ -11,6 +9,28 @@ # TODO replace upload_testfile_to_BucketFS once missing funcs in BucketFSLocation are implemented +@pytest.mark.parametrize("path_in_bucket", [ + "/path/in/bucket/file.txt", + "path/in/bucket/file.txt", + "path/in/bucket/file.txt.tar.gz", + "path/in/bucket/file.txt.zip", + "path/in/bucket/file.txt.tgz", + "path/in/bucket/file.txt.tar"]) +def test_generate_bucket_udf_path(path_in_bucket): + connection_config = BucketFSConnectionConfig( + host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig( + connection_config=connection_config, bucketfs_name="bfsdefault") + bucket_config = BucketConfig( + bucket_name="default", bucketfs_config=bucketfs_config) + bucketfs_location = BucketFSLocation(bucket_config, "") + + udf_path = bucketfs_location.generate_bucket_udf_path( + path_in_bucket=path_in_bucket) + + assert str(udf_path) == "/buckets/bfsdefault/default/path/in/bucket/file.txt" + + def test_upload_download_string_from_different_instance(): connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index debabb9c..18244f34 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -1,9 +1,25 @@ +import tempfile from tempfile import TemporaryDirectory, NamedTemporaryFile from pathlib import Path, PurePosixPath - import pytest +from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import \ + LocalFSMockBucketFSLocation + + +@pytest.mark.parametrize("path_in_bucket", [ + "/path/in/bucket/file.txt", + "path/in/bucket/file.txt", + "path/in/bucket/file.txt.tar.gz", + "path/in/bucket/file.txt.zip", + "path/in/bucket/file.txt.tgz", + "path/in/bucket/file.txt.tar"]) +def test_generate_bucket_udf_path(path_in_bucket): + with tempfile.TemporaryDirectory() as tmpdir_name: + bucketfs_location = LocalFSMockBucketFSLocation(tmpdir_name) + udf_path = bucketfs_location.generate_bucket_udf_path(path_in_bucket) + + assert udf_path == PurePosixPath(tmpdir_name, "path/in/bucket/file.txt") -from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import LocalFSMockBucketFSLocation def test_upload_download_string_from_different_instance(): with TemporaryDirectory() as path: From baba17666681eeda787b8044f842248a1943377b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 14:26:02 +0200 Subject: [PATCH 3/9] Added with db test --- tests/test_bucketfs_location.py | 72 ++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/test_bucketfs_location.py b/tests/test_bucketfs_location.py index b5f5c31d..72d64326 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/test_bucketfs_location.py @@ -28,7 +28,76 @@ def test_generate_bucket_udf_path(path_in_bucket): udf_path = bucketfs_location.generate_bucket_udf_path( path_in_bucket=path_in_bucket) - assert str(udf_path) == "/buckets/bfsdefault/default/path/in/bucket/file.txt" + assert str(udf_path) == "/buckets/bfsdefault/default/" \ + "path/in/bucket/file.txt" + + +def test_generate_bucket_udf_path_with_db( + upload_language_container, pyexasol_connection): + + connection_config = BucketFSConnectionConfig( + host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig( + connection_config=connection_config, bucketfs_name="bfsdefault") + bucket_config = BucketConfig( + bucket_name="default", bucketfs_config=bucketfs_config) + bucketfs_location = BucketFSLocation(bucket_config, "") + + bucket_file_path = "test_file.txt" + test_string = "test_string" + bucketfs_location.upload_string_to_bucketfs(bucket_file_path, test_string) + + target_schema = "TARGET_SCHEMA" + try: + # access file from udf + udf_name = "AccessFileInBucketFSFromUDF" + pyexasol_connection.execute( + f"CREATE SCHEMA IF NOT EXISTS {target_schema};") + pyexasol_connection.execute( + f"OPEN SCHEMA {target_schema};") + udf_sql = textwrap.dedent(f""" + CREATE OR REPLACE PYTHON3_BFSUP SET SCRIPT {target_schema}."{udf_name}"( + "path_in_bucket" VARCHAR(20000)) + RETURNS BOOLEAN + AS + from exasol_bucketfs_utils_python.bucket_config import BucketConfig + from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig + from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig + from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation + from pathlib import PurePosixPath + + bucket_name = "default" + bucketfs_name = "bfsdefault" + def get_bucket_config(): + connection_config = BucketFSConnectionConfig(host="localhost", + port=6666, + user="r", pwd="read", + is_https=False) + bucketfs_config = BucketFSConfig(bucketfs_name, connection_config=connection_config) + return BucketConfig(bucket_name, bucketfs_config) + + def get_bucket_path(): + return PurePosixPath("/buckets", bucketfs_name, bucket_name) + + def run(ctx): + path_in_bucket = ctx.path_in_bucket + bucket_config = get_bucket_config() + bucket_path = get_bucket_path() + bucketfs_location = BucketFSLocation(bucket_config, "") + relative_file_path = bucketfs_location.generate_bucket_udf_path( + path_in_bucket).relative_to(bucket_path) + listed_files = bucketfs_location.list_files_in_bucketfs(".") + return str(relative_file_path) in listed_files + """) + pyexasol_connection.execute(udf_sql) + result = pyexasol_connection.execute( + f"""select {target_schema}."{udf_name}"('{bucket_file_path}')""").fetchall() + print(result) + assert result[0][0] + finally: + delete_testfile_from_bucketfs(file_path=bucket_file_path, + bucket_config=bucketfs_location.bucket_config) + pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {target_schema} CASCADE;") def test_upload_download_string_from_different_instance(): @@ -248,6 +317,7 @@ def run(ctx): bucket_config=bucketfs_location_read.bucket_config) pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {target_schema} CASCADE;") + def test_read_files_to_fileobj_from_bucketfs_inside_udf(upload_language_container, pyexasol_connection): connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) From 9b000c56fe3f708b5957bfb4091c17487225bf83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 14:27:20 +0200 Subject: [PATCH 4/9] Removed archive controller for localFS --- .../localfs_mock_bucketfs_location.py | 2 -- tests/test_localfs_mock_bucketfs_location.py | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index bb2fabe0..57b3f2d4 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -29,8 +29,6 @@ def generate_bucket_udf_path( if path_in_bucket is not None: path_in_bucket = bucketfs_utils.\ make_path_relative(path_in_bucket) - path_in_bucket = bucketfs_utils.\ - correct_path_in_bucket_for_archives(path_in_bucket) else: path_in_bucket = "" path = PurePosixPath(self.base_path, path_in_bucket) diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index 18244f34..14e7e3cf 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -8,11 +8,7 @@ @pytest.mark.parametrize("path_in_bucket", [ "/path/in/bucket/file.txt", - "path/in/bucket/file.txt", - "path/in/bucket/file.txt.tar.gz", - "path/in/bucket/file.txt.zip", - "path/in/bucket/file.txt.tgz", - "path/in/bucket/file.txt.tar"]) + "path/in/bucket/file.txt"]) def test_generate_bucket_udf_path(path_in_bucket): with tempfile.TemporaryDirectory() as tmpdir_name: bucketfs_location = LocalFSMockBucketFSLocation(tmpdir_name) From ad5e14d0b90790baadb0fbb183ab8e5658c0ae6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 14:45:37 +0200 Subject: [PATCH 5/9] Updated port --- tests/test_bucketfs_location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bucketfs_location.py b/tests/test_bucketfs_location.py index 72d64326..0a361f65 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/test_bucketfs_location.py @@ -70,7 +70,7 @@ def test_generate_bucket_udf_path_with_db( bucketfs_name = "bfsdefault" def get_bucket_config(): connection_config = BucketFSConnectionConfig(host="localhost", - port=6666, + port=6583, user="r", pwd="read", is_https=False) bucketfs_config = BucketFSConfig(bucketfs_name, connection_config=connection_config) From e22da7ca933c9febbae8a2cc215fd8418b167a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 16:01:28 +0200 Subject: [PATCH 6/9] Updated test --- tests/test_bucketfs_location.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_bucketfs_location.py b/tests/test_bucketfs_location.py index 0a361f65..222809d3 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/test_bucketfs_location.py @@ -64,30 +64,25 @@ def test_generate_bucket_udf_path_with_db( from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation - from pathlib import PurePosixPath + from pathlib import PurePosixPath, Path bucket_name = "default" bucketfs_name = "bfsdefault" def get_bucket_config(): connection_config = BucketFSConnectionConfig(host="localhost", - port=6583, + port=6666, user="r", pwd="read", is_https=False) bucketfs_config = BucketFSConfig(bucketfs_name, connection_config=connection_config) return BucketConfig(bucket_name, bucketfs_config) - def get_bucket_path(): - return PurePosixPath("/buckets", bucketfs_name, bucket_name) - def run(ctx): path_in_bucket = ctx.path_in_bucket bucket_config = get_bucket_config() - bucket_path = get_bucket_path() bucketfs_location = BucketFSLocation(bucket_config, "") - relative_file_path = bucketfs_location.generate_bucket_udf_path( - path_in_bucket).relative_to(bucket_path) - listed_files = bucketfs_location.list_files_in_bucketfs(".") - return str(relative_file_path) in listed_files + file_path = bucketfs_location.generate_bucket_udf_path(path_in_bucket) + + return Path(file_path).exists() """) pyexasol_connection.execute(udf_sql) result = pyexasol_connection.execute( From 72eb6d65c52273d7bafbe912eeffc275aa73f5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 16:25:50 +0200 Subject: [PATCH 7/9] Reorganized tests --- tests/fixtures/prepare_bucket_fixture.py | 2 +- .../upload_language_container_fixture.py | 2 +- tests/integration_tests/__init__.py | 0 tests/integration_tests/with_db/__init__.py | 0 .../with_db}/test_bucketfs_location.py | 66 +--------------- .../with_db}/test_load_fs_file_from_udf.py | 0 .../integration_tests/without_db/__init__.py | 0 .../without_db/test_bucketfs_location.py | 75 +++++++++++++++++++ .../without_db}/test_delete_file.py | 2 +- ...t_github_release_file_bucketfs_uploader.py | 0 .../without_db}/test_upload_download.py | 0 .../without_db}/test_upload_list.py | 0 tests/unit_tests/__init__.py | 0 tests/{ => unit_tests}/test_bucket_config.py | 0 .../{ => unit_tests}/test_bucketfs_config.py | 0 .../test_bucketfs_connection_config.py | 0 .../{ => unit_tests}/test_bucketfs_factory.py | 0 tests/{ => unit_tests}/test_bucketfs_utils.py | 0 .../test_localfs_mock_bucketfs_location.py | 0 19 files changed, 79 insertions(+), 68 deletions(-) create mode 100644 tests/integration_tests/__init__.py create mode 100644 tests/integration_tests/with_db/__init__.py rename tests/{ => integration_tests/with_db}/test_bucketfs_location.py (84%) rename tests/{ => integration_tests/with_db}/test_load_fs_file_from_udf.py (100%) create mode 100644 tests/integration_tests/without_db/__init__.py create mode 100644 tests/integration_tests/without_db/test_bucketfs_location.py rename tests/{ => integration_tests/without_db}/test_delete_file.py (95%) rename tests/{ => integration_tests/without_db}/test_github_release_file_bucketfs_uploader.py (100%) rename tests/{ => integration_tests/without_db}/test_upload_download.py (100%) rename tests/{ => integration_tests/without_db}/test_upload_list.py (100%) create mode 100644 tests/unit_tests/__init__.py rename tests/{ => unit_tests}/test_bucket_config.py (100%) rename tests/{ => unit_tests}/test_bucketfs_config.py (100%) rename tests/{ => unit_tests}/test_bucketfs_connection_config.py (100%) rename tests/{ => unit_tests}/test_bucketfs_factory.py (100%) rename tests/{ => unit_tests}/test_bucketfs_utils.py (100%) rename tests/{ => unit_tests}/test_localfs_mock_bucketfs_location.py (100%) diff --git a/tests/fixtures/prepare_bucket_fixture.py b/tests/fixtures/prepare_bucket_fixture.py index c26240fd..43d336ae 100644 --- a/tests/fixtures/prepare_bucket_fixture.py +++ b/tests/fixtures/prepare_bucket_fixture.py @@ -4,7 +4,7 @@ from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig from exasol_bucketfs_utils_python.bucketfs_connection_config import \ BucketFSConnectionConfig -from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs +from tests.integration_tests.with_db.test_load_fs_file_from_udf import delete_testfile_from_bucketfs @pytest.fixture(scope="module") diff --git a/tests/fixtures/upload_language_container_fixture.py b/tests/fixtures/upload_language_container_fixture.py index b6d6495b..4474872f 100644 --- a/tests/fixtures/upload_language_container_fixture.py +++ b/tests/fixtures/upload_language_container_fixture.py @@ -9,7 +9,7 @@ @pytest.fixture(scope="session") def db_connection(): - db_connection = Connection(address=f"localhost:8888", user="sys", password="exasol") + db_connection = Connection(address=f"localhost:9563", user="sys", password="exasol") return db_connection diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration_tests/with_db/__init__.py b/tests/integration_tests/with_db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_bucketfs_location.py b/tests/integration_tests/with_db/test_bucketfs_location.py similarity index 84% rename from tests/test_bucketfs_location.py rename to tests/integration_tests/with_db/test_bucketfs_location.py index 222809d3..67f8927d 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/integration_tests/with_db/test_bucketfs_location.py @@ -5,32 +5,10 @@ from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation import pytest import textwrap -from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs, upload_testfile_to_bucketfs +from tests.integration_tests.with_db.test_load_fs_file_from_udf import delete_testfile_from_bucketfs, upload_testfile_to_bucketfs # TODO replace upload_testfile_to_BucketFS once missing funcs in BucketFSLocation are implemented -@pytest.mark.parametrize("path_in_bucket", [ - "/path/in/bucket/file.txt", - "path/in/bucket/file.txt", - "path/in/bucket/file.txt.tar.gz", - "path/in/bucket/file.txt.zip", - "path/in/bucket/file.txt.tgz", - "path/in/bucket/file.txt.tar"]) -def test_generate_bucket_udf_path(path_in_bucket): - connection_config = BucketFSConnectionConfig( - host="localhost", port=6666, user="w", pwd="write", is_https=False) - bucketfs_config = BucketFSConfig( - connection_config=connection_config, bucketfs_name="bfsdefault") - bucket_config = BucketConfig( - bucket_name="default", bucketfs_config=bucketfs_config) - bucketfs_location = BucketFSLocation(bucket_config, "") - - udf_path = bucketfs_location.generate_bucket_udf_path( - path_in_bucket=path_in_bucket) - - assert str(udf_path) == "/buckets/bfsdefault/default/" \ - "path/in/bucket/file.txt" - def test_generate_bucket_udf_path_with_db( upload_language_container, pyexasol_connection): @@ -95,48 +73,6 @@ def run(ctx): pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {target_schema} CASCADE;") -def test_upload_download_string_from_different_instance(): - connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) - bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) - bucket_config = BucketConfig(bucket_name="default", bucketfs_config=bucketfs_config) - bucket_base_path = PurePosixPath("test_up_down_str") - bucketfs_location_upload = BucketFSLocation(bucket_config, bucket_base_path) - bucketfs_location_download = BucketFSLocation(bucket_config, bucket_base_path) - bucket_file_path = "test_file.txt" - test_string = "test_string" - bucketfs_location_upload.upload_string_to_bucketfs(bucket_file_path, test_string) - result = bucketfs_location_download.download_from_bucketfs_to_string(bucket_file_path) - assert result == test_string - delete_testfile_from_bucketfs(file_path=str(bucket_base_path) + "/" + bucket_file_path, - bucket_config=bucketfs_location_upload.bucket_config) - - -class TestValue: - __test__ = False - - def __init__(self, value: str): - self.value = value - - def __eq__(self, other): - return self.value == self.value - - -def test_upload_download_obj_from_different_instance(): - connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) - bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) - bucket_config = BucketConfig(bucket_name="default", bucketfs_config=bucketfs_config) - bucket_base_path = PurePosixPath("test_up_down_obj") - bucketfs_location_upload = BucketFSLocation(bucket_config, bucket_base_path) - bucketfs_location_download = BucketFSLocation(bucket_config, bucket_base_path) - bucket_file_path = "test_file.txt" - test_value = TestValue("test_string") - bucketfs_location_upload.upload_object_to_bucketfs_via_joblib(test_value, bucket_file_path) - result = bucketfs_location_download.download_object_from_bucketfs_via_joblib(bucket_file_path) - assert result == test_value - delete_testfile_from_bucketfs(file_path=str(bucket_base_path) + "/" + bucket_file_path, - bucket_config=bucketfs_location_upload.bucket_config) - - @pytest.mark.usefixtures("upload_language_container", "pyexasol_connection") def test_read_files_to_str_from_bucketfs_inside_udf(upload_language_container, pyexasol_connection): diff --git a/tests/test_load_fs_file_from_udf.py b/tests/integration_tests/with_db/test_load_fs_file_from_udf.py similarity index 100% rename from tests/test_load_fs_file_from_udf.py rename to tests/integration_tests/with_db/test_load_fs_file_from_udf.py diff --git a/tests/integration_tests/without_db/__init__.py b/tests/integration_tests/without_db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration_tests/without_db/test_bucketfs_location.py b/tests/integration_tests/without_db/test_bucketfs_location.py new file mode 100644 index 00000000..774e5601 --- /dev/null +++ b/tests/integration_tests/without_db/test_bucketfs_location.py @@ -0,0 +1,75 @@ +from pathlib import PurePosixPath +from exasol_bucketfs_utils_python.bucket_config import BucketConfig +from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +import pytest +from tests.integration_tests.with_db.test_load_fs_file_from_udf import delete_testfile_from_bucketfs + + +# TODO replace upload_testfile_to_BucketFS once missing funcs in BucketFSLocation are implemented + + +@pytest.mark.parametrize("path_in_bucket", [ + "/path/in/bucket/file.txt", + "path/in/bucket/file.txt", + "path/in/bucket/file.txt.tar.gz", + "path/in/bucket/file.txt.zip", + "path/in/bucket/file.txt.tgz", + "path/in/bucket/file.txt.tar"]) +def test_generate_bucket_udf_path(path_in_bucket): + connection_config = BucketFSConnectionConfig( + host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig( + connection_config=connection_config, bucketfs_name="bfsdefault") + bucket_config = BucketConfig( + bucket_name="default", bucketfs_config=bucketfs_config) + bucketfs_location = BucketFSLocation(bucket_config, "") + + udf_path = bucketfs_location.generate_bucket_udf_path( + path_in_bucket=path_in_bucket) + + assert str(udf_path) == "/buckets/bfsdefault/default/" \ + "path/in/bucket/file.txt" + + +def test_upload_download_string_from_different_instance(): + connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) + bucket_config = BucketConfig(bucket_name="default", bucketfs_config=bucketfs_config) + bucket_base_path = PurePosixPath("test_up_down_str") + bucketfs_location_upload = BucketFSLocation(bucket_config, bucket_base_path) + bucketfs_location_download = BucketFSLocation(bucket_config, bucket_base_path) + bucket_file_path = "test_file.txt" + test_string = "test_string" + bucketfs_location_upload.upload_string_to_bucketfs(bucket_file_path, test_string) + result = bucketfs_location_download.download_from_bucketfs_to_string(bucket_file_path) + assert result == test_string + delete_testfile_from_bucketfs(file_path=str(bucket_base_path) + "/" + bucket_file_path, + bucket_config=bucketfs_location_upload.bucket_config) + + +class TestValue: + __test__ = False + + def __init__(self, value: str): + self.value = value + + def __eq__(self, other): + return self.value == self.value + + +def test_upload_download_obj_from_different_instance(): + connection_config = BucketFSConnectionConfig(host="localhost", port=6666, user="w", pwd="write", is_https=False) + bucketfs_config = BucketFSConfig("bfsdefault", connection_config=connection_config) + bucket_config = BucketConfig(bucket_name="default", bucketfs_config=bucketfs_config) + bucket_base_path = PurePosixPath("test_up_down_obj") + bucketfs_location_upload = BucketFSLocation(bucket_config, bucket_base_path) + bucketfs_location_download = BucketFSLocation(bucket_config, bucket_base_path) + bucket_file_path = "test_file.txt" + test_value = TestValue("test_string") + bucketfs_location_upload.upload_object_to_bucketfs_via_joblib(test_value, bucket_file_path) + result = bucketfs_location_download.download_object_from_bucketfs_via_joblib(bucket_file_path) + assert result == test_value + delete_testfile_from_bucketfs(file_path=str(bucket_base_path) + "/" + bucket_file_path, + bucket_config=bucketfs_location_upload.bucket_config) diff --git a/tests/test_delete_file.py b/tests/integration_tests/without_db/test_delete_file.py similarity index 95% rename from tests/test_delete_file.py rename to tests/integration_tests/without_db/test_delete_file.py index 1df3cb53..ef3f547b 100644 --- a/tests/test_delete_file.py +++ b/tests/integration_tests/without_db/test_delete_file.py @@ -4,7 +4,7 @@ from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig -from tests.test_load_fs_file_from_udf import delete_testfile_from_bucketfs +from tests.integration_tests.with_db.test_load_fs_file_from_udf import delete_testfile_from_bucketfs def test_delete_files(): diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/integration_tests/without_db/test_github_release_file_bucketfs_uploader.py similarity index 100% rename from tests/test_github_release_file_bucketfs_uploader.py rename to tests/integration_tests/without_db/test_github_release_file_bucketfs_uploader.py diff --git a/tests/test_upload_download.py b/tests/integration_tests/without_db/test_upload_download.py similarity index 100% rename from tests/test_upload_download.py rename to tests/integration_tests/without_db/test_upload_download.py diff --git a/tests/test_upload_list.py b/tests/integration_tests/without_db/test_upload_list.py similarity index 100% rename from tests/test_upload_list.py rename to tests/integration_tests/without_db/test_upload_list.py diff --git a/tests/unit_tests/__init__.py b/tests/unit_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_bucket_config.py b/tests/unit_tests/test_bucket_config.py similarity index 100% rename from tests/test_bucket_config.py rename to tests/unit_tests/test_bucket_config.py diff --git a/tests/test_bucketfs_config.py b/tests/unit_tests/test_bucketfs_config.py similarity index 100% rename from tests/test_bucketfs_config.py rename to tests/unit_tests/test_bucketfs_config.py diff --git a/tests/test_bucketfs_connection_config.py b/tests/unit_tests/test_bucketfs_connection_config.py similarity index 100% rename from tests/test_bucketfs_connection_config.py rename to tests/unit_tests/test_bucketfs_connection_config.py diff --git a/tests/test_bucketfs_factory.py b/tests/unit_tests/test_bucketfs_factory.py similarity index 100% rename from tests/test_bucketfs_factory.py rename to tests/unit_tests/test_bucketfs_factory.py diff --git a/tests/test_bucketfs_utils.py b/tests/unit_tests/test_bucketfs_utils.py similarity index 100% rename from tests/test_bucketfs_utils.py rename to tests/unit_tests/test_bucketfs_utils.py diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/unit_tests/test_localfs_mock_bucketfs_location.py similarity index 100% rename from tests/test_localfs_mock_bucketfs_location.py rename to tests/unit_tests/test_localfs_mock_bucketfs_location.py From 80fa48ffc45b8b790c64c76801be79dd03480db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 16:27:39 +0200 Subject: [PATCH 8/9] Updated changelog --- doc/changes/changes_0.3.0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/changes/changes_0.3.0.md b/doc/changes/changes_0.3.0.md index 5f3b5d51..504fa5d6 100644 --- a/doc/changes/changes_0.3.0.md +++ b/doc/changes/changes_0.3.0.md @@ -5,7 +5,8 @@ ## Features / Enhancements - - / + + - #72: Added generate bucket udf path method to BucketFSLocation ## Bug Fixes From 1cbf03c97404a3a9ce65dd7be924a01c313e7974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 19 Jul 2022 16:31:38 +0200 Subject: [PATCH 9/9] Corrected db port --- tests/fixtures/upload_language_container_fixture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/upload_language_container_fixture.py b/tests/fixtures/upload_language_container_fixture.py index 4474872f..b6d6495b 100644 --- a/tests/fixtures/upload_language_container_fixture.py +++ b/tests/fixtures/upload_language_container_fixture.py @@ -9,7 +9,7 @@ @pytest.fixture(scope="session") def db_connection(): - db_connection = Connection(address=f"localhost:9563", user="sys", password="exasol") + db_connection = Connection(address=f"localhost:8888", user="sys", password="exasol") return db_connection