From 975a626bacd2258ee375a44b8e0be134e9c376a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 29 Mar 2022 11:37:42 +0200 Subject: [PATCH 1/4] Removed the conversion --- .gitignore | 3 +++ doc/changes/changes_0.2.0.md | 2 ++ tests/fixtures/upload_language_container_fixture.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 696495ba..d5fffc9e 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ poetry.lock # Sphinx doc/_build doc/api + +# Language container +.build_output diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md index cb8c1914..832e27c4 100644 --- a/doc/changes/changes_0.2.0.md +++ b/doc/changes/changes_0.2.0.md @@ -8,6 +8,8 @@ Code name: t.b.d ## Bug Fixes + - #54: Removed PosixPath conversion from alter session string + ## Documentation ## Refactoring diff --git a/tests/fixtures/upload_language_container_fixture.py b/tests/fixtures/upload_language_container_fixture.py index b778acad..b6d6495b 100644 --- a/tests/fixtures/upload_language_container_fixture.py +++ b/tests/fixtures/upload_language_container_fixture.py @@ -31,7 +31,7 @@ def upload_language_container(pyexasol_connection, language_container): pwd=container_connection.password, base_path=None) container_path = Path(language_container["container_path"]) - alter_session = Path(language_container["alter_session"]) + alter_session = language_container["alter_session"] pyexasol_connection.execute(f"ALTER SESSION SET SCRIPT_LANGUAGES='{alter_session}'") with open(container_path, "rb") as container_file: container_bucketfs_location.upload_fileobj_to_bucketfs(container_file, "ml.tar") From 1f1d7ccd28a4276d30813f8dc25b9496590ab9cf Mon Sep 17 00:00:00 2001 From: Umit Cavus Buyuksahin Date: Wed, 30 Mar 2022 13:16:18 +0200 Subject: [PATCH 2/4] #58: Added type hints (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added type hints * Apply suggestions from code review Co-authored-by: Marlene Kreß Co-authored-by: Marlene Kreß --- .../abstract_bucketfs_location.py | 47 +++++++---- .../bucketfs_config.py | 10 ++- .../bucketfs_connection_config.py | 6 +- .../bucketfs_factory.py | 54 ++++++++----- .../bucketfs_location.py | 81 ++++++++++++------- .../bucketfs_utils.py | 50 +++++++----- exasol_bucketfs_utils_python/download.py | 28 ++++--- .../github_release_file_bucketfs_uploader.py | 24 +++--- .../load_file_from_local_fs.py | 20 +++-- .../localfs_mock_bucketfs_location.py | 48 ++++++----- .../release_link_extractor.py | 20 +++-- exasol_bucketfs_utils_python/upload.py | 33 +++++--- tests/test_bucketfs_location.py | 4 +- tests/test_localfs_mock_bucketfs_location.py | 4 +- tests/test_upload_download.py | 2 + 15 files changed, 273 insertions(+), 158 deletions(-) diff --git a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py index 24f573a8..9b31f6f2 100644 --- a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py @@ -1,52 +1,67 @@ -import typing from abc import ABC, abstractmethod -from typing import Any -from pathlib import Path +from typing import Any, Tuple, IO +from pathlib import PurePosixPath, Path +from urllib.parse import ParseResult class AbstractBucketFSLocation(ABC): """ - Abstract class for a BucketFSLocation for uploading and downloading strings, fileobjects and joblib objects. - Also able to read files from the BucketFS directly, if called from inside a UDF. + Abstract class for a BucketFSLocation for uploading and downloading strings, + fileobjects and joblib objects. Also able to read files from the BucketFS + directly, if called from inside a UDF. """ @abstractmethod - def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: + def download_from_bucketfs_to_string(self, + bucket_file_path: str) -> str: pass @abstractmethod - def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any: + def download_object_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: pass @abstractmethod - def upload_string_to_bucketfs(self, bucket_file_path: str, string: str): + def upload_string_to_bucketfs(self, + bucket_file_path: str, + string: str) -> \ + Tuple[ParseResult, PurePosixPath]: pass @abstractmethod - def upload_object_to_bucketfs_via_joblib(self, object: Any, + def upload_object_to_bucketfs_via_joblib(self, + object: Any, bucket_file_path: str, - **kwargs): + **kwargs) -> \ + Tuple[ParseResult, PurePosixPath]: pass @abstractmethod def upload_fileobj_to_bucketfs(self, - fileobj: typing.IO, - bucket_file_path: str): + fileobj: IO, + bucket_file_path: str) -> \ + Tuple[ParseResult, PurePosixPath]: pass # TODO add missing upload/download functions @abstractmethod - def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str: + def read_file_from_bucketfs_to_string(self, + bucket_file_path: str) -> str: pass @abstractmethod - def read_file_from_bucketfs_to_file(self, bucket_file_path: str, local_file_path: Path): + def read_file_from_bucketfs_to_file(self, + bucket_file_path: str, + local_file_path: Path) -> None: pass @abstractmethod - def read_file_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: typing.IO): + def read_file_from_bucketfs_to_fileobj(self, + bucket_file_path: str, + fileobj: IO) -> None: pass @abstractmethod - def read_file_from_bucketfs_via_joblib(self, bucket_file_path: str) -> typing.Any: + def read_file_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: pass diff --git a/exasol_bucketfs_utils_python/bucketfs_config.py b/exasol_bucketfs_utils_python/bucketfs_config.py index 95876cea..b0cda736 100644 --- a/exasol_bucketfs_utils_python/bucketfs_config.py +++ b/exasol_bucketfs_utils_python/bucketfs_config.py @@ -1,8 +1,7 @@ from typing import Union - from typeguard import typechecked - -from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import \ + BucketFSConnectionConfig class BucketFSConfig: @@ -14,7 +13,10 @@ class BucketFSConfig: """ @typechecked(always=True) - def __init__(self, bucketfs_name: str, connection_config: Union[BucketFSConnectionConfig, None] = None): + def __init__( + self, + bucketfs_name: str, + connection_config: Union[BucketFSConnectionConfig, None] = None): self._connection_config = connection_config if bucketfs_name == "": raise ValueError("BucketFS name can't be an empty string") diff --git a/exasol_bucketfs_utils_python/bucketfs_connection_config.py b/exasol_bucketfs_utils_python/bucketfs_connection_config.py index 1b679f3f..d03e407e 100644 --- a/exasol_bucketfs_utils_python/bucketfs_connection_config.py +++ b/exasol_bucketfs_utils_python/bucketfs_connection_config.py @@ -8,14 +8,16 @@ class BucketFSConnectionConfig: """ @typechecked(always=True) - def __init__(self, host: str, port: int, user: str, pwd: str, is_https=False): + def __init__(self, host: str, port: int, + user: str, pwd: str, is_https: bool = False): self._is_https = is_https if host == "": raise ValueError("Host can't be an empty string") self._host = host self._port = port if user not in ["w", "r"]: # The BucketFs currently supports only these two users - raise ValueError(f"User can only be, 'w' (read-write access) or 'r' (read-only access), but got {user}") + raise ValueError(f"User can only be, 'w' (read-write access) or " + f"'r' (read-only access), but got {user}") self._user = user if pwd == "": raise ValueError("Password can't be an empty string") diff --git a/exasol_bucketfs_utils_python/bucketfs_factory.py b/exasol_bucketfs_utils_python/bucketfs_factory.py index 09b8ffbc..455498d0 100644 --- a/exasol_bucketfs_utils_python/bucketfs_factory.py +++ b/exasol_bucketfs_utils_python/bucketfs_factory.py @@ -1,25 +1,30 @@ import urllib.parse from pathlib import PurePosixPath from typing import Optional - from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig - +from exasol_bucketfs_utils_python.bucketfs_connection_config import \ + BucketFSConnectionConfig from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation -from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import LocalFSMockBucketFSLocation +from exasol_bucketfs_utils_python.localfs_mock_bucketfs_location import \ + LocalFSMockBucketFSLocation class BucketFSFactory: """ Creates a BucketFSLocation given an url. """ - def create_bucketfs_location(self, url: str, user: str, pwd: str, base_path: Optional[PurePosixPath] = None): + def create_bucketfs_location(self, url: str, user: str, pwd: str, + base_path: Optional[PurePosixPath] = None) -> \ + BucketFSLocation: """ - Create BucketFSLocation from the the url given. If the url has the schema http:// or https://, - this function creates a real BucketFSLocation for a url scheme file:/// we create a LocalFSMockBucketFSLocation. - For url with http:// or https:// schema you also need to provide the bucketfs-name via a url parameter. - A url would look like the following: http[s]://://; + Create BucketFSLocation from the url given. + If the url has the schema http:// or https://, this function creates a + real BucketFSLocation for a url scheme file:/// we create a + LocalFSMockBucketFSLocation. For url with http:// or https:// schema + you also need to provide the bucketfs-name via an url parameter. An url + would look like the following: + http[s]://://; :param url: :param user: :param pwd: @@ -29,26 +34,35 @@ def create_bucketfs_location(self, url: str, user: str, pwd: str, base_path: Opt parsed_url = urllib.parse.urlparse(url) if parsed_url.scheme == "http" or parsed_url.scheme == "https": is_https = parsed_url.scheme == "https" - connection_config = BucketFSConnectionConfig(host=parsed_url.hostname, - port=parsed_url.port, - user=user, pwd=pwd, - is_https=is_https) + connection_config = BucketFSConnectionConfig( + host=parsed_url.hostname, + port=parsed_url.port, + user=user, + pwd=pwd, + is_https=is_https) url_path = PurePosixPath(parsed_url.path) bucket_name = url_path.parts[1] - base_path_in_bucket = PurePosixPath(url_path.parts[2]).joinpath(*url_path.parts[3:]) + base_path_in_bucket = PurePosixPath( + url_path.parts[2]).joinpath(*url_path.parts[3:]) if base_path is not None: - base_path_in_bucket = PurePosixPath(base_path_in_bucket, base_path) + base_path_in_bucket = PurePosixPath( + base_path_in_bucket, base_path) bucketfs_name = parsed_url.params - bucketfs_config = BucketFSConfig(bucketfs_name, connection_config=connection_config) - bucket_config = BucketConfig(bucket_name=bucket_name, bucketfs_config=bucketfs_config) - bucketfs_location = BucketFSLocation(bucket_config, base_path_in_bucket) + bucketfs_config = BucketFSConfig( + bucketfs_name, connection_config=connection_config) + bucket_config = BucketConfig( + bucket_name=bucket_name, bucketfs_config=bucketfs_config) + bucketfs_location = BucketFSLocation( + bucket_config, base_path_in_bucket) return bucketfs_location elif parsed_url.scheme == "file": if parsed_url.netloc != '': - raise ValueError(f"URL '{url}' with file:// schema and netloc not support.") + raise ValueError(f"URL '{url}' with file:// schema " + f"and netloc not support.") base_path_in_bucket = PurePosixPath(parsed_url.path) if base_path is not None: - base_path_in_bucket = PurePosixPath(base_path_in_bucket, base_path) + base_path_in_bucket = PurePosixPath( + base_path_in_bucket, base_path) bucketfs_location = LocalFSMockBucketFSLocation(base_path_in_bucket) return bucketfs_location else: diff --git a/exasol_bucketfs_utils_python/bucketfs_location.py b/exasol_bucketfs_utils_python/bucketfs_location.py index 8eb98d81..305bba17 100644 --- a/exasol_bucketfs_utils_python/bucketfs_location.py +++ b/exasol_bucketfs_utils_python/bucketfs_location.py @@ -1,92 +1,113 @@ -import typing +from typing import Any, Tuple, IO from pathlib import PurePosixPath, Path -from typing import Any - +from urllib.parse import ParseResult from exasol_bucketfs_utils_python import download, upload from exasol_bucketfs_utils_python import load_file_from_local_fs as from_BFS from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.abstract_bucketfs_location import AbstractBucketFSLocation +from exasol_bucketfs_utils_python.abstract_bucketfs_location import \ + AbstractBucketFSLocation class BucketFSLocation(AbstractBucketFSLocation): """ BucketFSLocation implements AbstractBucketFSLocation. - BucketFSLocation is used to upload fileobjects, strings or joblib objects to the BucketFS given a path and the object, - or to download objects into strings, fileobjects or joblib objects from the BucketFS given a file path. - Also able to read files from the BucketFS directly, if called from inside of an UDF. - If reading an object via joblib inside of an UDF, make sure the object type is known inside the UDF. + BucketFSLocation is used to upload fileobjects, strings or joblib objects to + the BucketFS given a path and the object, or to download objects into + strings, fileobjects or joblib objects from the BucketFS given a file path. + Also able to read files from the BucketFS directly, if called from inside of + an UDF. If reading an object via joblib inside of an UDF, make sure the + object type is known inside the UDF. """ def __init__(self, bucket_config: BucketConfig, base_path: PurePosixPath): self.base_path = base_path self.bucket_config = bucket_config - def get_complete_file_path_in_bucket(self, bucket_file_path: str) -> str: + def get_complete_file_path_in_bucket(self, + bucket_file_path: str) -> str: return str(PurePosixPath(self.base_path, bucket_file_path)) - def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: + def download_from_bucketfs_to_string(self, + bucket_file_path: str) -> str: result = download.download_from_bucketfs_to_string( self.bucket_config, - self.get_complete_file_path_in_bucket(bucket_file_path)) + self.get_complete_file_path_in_bucket(bucket_file_path) + ) return result - def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any: + def download_object_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: result = download.download_object_from_bucketfs_via_joblib( self.bucket_config, - self.get_complete_file_path_in_bucket(bucket_file_path)) + self.get_complete_file_path_in_bucket(bucket_file_path) + ) return result - def upload_string_to_bucketfs(self, bucket_file_path: str, string: str): + def upload_string_to_bucketfs(self, + bucket_file_path: str, + string: str) -> \ + Tuple[ParseResult, PurePosixPath]: result = upload.upload_string_to_bucketfs( self.bucket_config, self.get_complete_file_path_in_bucket(bucket_file_path), - string) + string + ) return result def upload_object_to_bucketfs_via_joblib(self, object: Any, bucket_file_path: str, - **kwargs): + **kwargs) -> \ + Tuple[ParseResult, PurePosixPath]: result = upload.upload_object_to_bucketfs_via_joblib( object, self.bucket_config, self.get_complete_file_path_in_bucket(bucket_file_path), - **kwargs) + **kwargs + ) return result def upload_fileobj_to_bucketfs(self, - fileobj: typing.IO, - bucket_file_path: str): + fileobj: IO, + bucket_file_path: str) -> \ + Tuple[ParseResult, PurePosixPath]: result = upload.upload_fileobj_to_bucketfs( self.bucket_config, self.get_complete_file_path_in_bucket(bucket_file_path), - fileobj) - return result - - def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str: - result = from_BFS.read_file_from_bucketfs_to_string( - self.get_complete_file_path_in_bucket(bucket_file_path), - self.bucket_config + fileobj ) return result - def read_file_from_bucketfs_via_joblib(self, bucket_file_path: str) -> typing.Any: - result = from_BFS.read_file_from_bucketfs_via_joblib( + def read_file_from_bucketfs_to_string(self, + bucket_file_path: str) -> str: + result = from_BFS.read_file_from_bucketfs_to_string( self.get_complete_file_path_in_bucket(bucket_file_path), self.bucket_config ) return result - def read_file_from_bucketfs_to_file(self, bucket_file_path: str, local_file_path: Path) -> None: + def read_file_from_bucketfs_to_file(self, + bucket_file_path: str, + local_file_path: Path) -> None: from_BFS.read_file_from_bucketfs_to_file( self.get_complete_file_path_in_bucket(bucket_file_path), self.bucket_config, local_file_path ) - def read_file_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: typing.IO) -> None: + def read_file_from_bucketfs_to_fileobj(self, + bucket_file_path: str, + fileobj: IO) -> None: from_BFS.read_file_from_bucketfs_to_fileobj( self.get_complete_file_path_in_bucket(bucket_file_path), self.bucket_config, fileobj ) + + def read_file_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: + result = from_BFS.read_file_from_bucketfs_via_joblib( + self.get_complete_file_path_in_bucket(bucket_file_path), + self.bucket_config + ) + return result diff --git a/exasol_bucketfs_utils_python/bucketfs_utils.py b/exasol_bucketfs_utils_python/bucketfs_utils.py index b3910245..7c6f71e5 100644 --- a/exasol_bucketfs_utils_python/bucketfs_utils.py +++ b/exasol_bucketfs_utils_python/bucketfs_utils.py @@ -1,10 +1,8 @@ import urllib.parse from pathlib import PurePosixPath from typing import Union - from requests.auth import HTTPBasicAuth from typeguard import typechecked - from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig @@ -16,16 +14,18 @@ def _encode_url_part(part: str) -> str: return urlencoded -def _correct_path_in_bucket_for_archives(path_in_bucket: PurePosixPath) -> PurePosixPath: +def _correct_path_in_bucket_for_archives(path_in_bucket: PurePosixPath) \ + -> PurePosixPath: for extension in ARCHIVE_EXTENSIONS: if path_in_bucket.name.endswith(extension): - path_in_bucket = PurePosixPath(path_in_bucket.parent, - path_in_bucket.name[:-len(extension)]) + path_in_bucket = PurePosixPath( + path_in_bucket.parent, path_in_bucket.name[:-len(extension)]) break return path_in_bucket -def _make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) -> PurePosixPath: +def _make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) \ + -> PurePosixPath: path_in_bucket = PurePosixPath(path_in_bucket) if path_in_bucket.is_absolute(): path_in_bucket = path_in_bucket.relative_to(PurePosixPath("/")) @@ -33,9 +33,11 @@ def _make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) -> Pure @typechecked(always=True) -def generate_bucketfs_udf_path(bucketfs_config: BucketFSConfig) -> PurePosixPath: +def generate_bucketfs_udf_path(bucketfs_config: BucketFSConfig) \ + -> PurePosixPath: """ - This function generates the path where UDFs can access the content of a BucketFS in their file system + This function generates the path where UDFs can access the content of a + BucketFS in their file system :param bucketfs_config: Config of the BucketFS, the BucketFSConnectionConfig in the BucketFSConfig can be None :return: Path of the given BucketFS in the file system of the UDFs @@ -45,11 +47,12 @@ def generate_bucketfs_udf_path(bucketfs_config: BucketFSConfig) -> PurePosixPath @typechecked(always=True) -def generate_bucket_udf_path(bucket_config: BucketConfig, - path_in_bucket: Union[None, str, PurePosixPath]) -> PurePosixPath: +def generate_bucket_udf_path( + bucket_config: BucketConfig, + path_in_bucket: Union[None, str, PurePosixPath]) -> PurePosixPath: """ - This function generates the path where UDFs can access the content of a bucket or - the given path in a bucket in their file system + This function generates the path where UDFs can access the content of a + bucket or the given path in a bucket in their file system :param bucket_config: Config of the Bucket, the BucketFSConnectionConfig in the BucketFSConfig can be None :param path_in_bucket: If not None, path_in_bucket gets concatenated to the path of the bucket @@ -68,8 +71,9 @@ def generate_bucket_udf_path(bucket_config: BucketConfig, @typechecked(always=True) -def generate_bucketfs_http_url(bucketfs_config: BucketFSConfig, - with_credentials: bool = False) -> urllib.parse.ParseResult: +def generate_bucketfs_http_url( + bucketfs_config: BucketFSConfig, + with_credentials: bool = False) -> urllib.parse.ParseResult: """ This function generates an HTTP[s] url for the given BucketFSConfig with or without basic authentication (a template: http[s]://user:password@host:port) @@ -79,9 +83,11 @@ def generate_bucketfs_http_url(bucketfs_config: BucketFSConfig, :return: HTTP[S] URL of the BucketFS """ if bucketfs_config.connection_config is None: - raise ValueError("bucket_config.bucketfs_config.connection_config can't be None for this operation") + raise ValueError( + "bucket_config.bucketfs_config.connection_config can't be None for this operation") if with_credentials: - encoded_password = _encode_url_part(bucketfs_config.connection_config.pwd) + encoded_password = _encode_url_part( + bucketfs_config.connection_config.pwd) encoded_user = _encode_url_part(bucketfs_config.connection_config.user) credentials = f"{encoded_user}:{encoded_password}@" else: @@ -98,8 +104,10 @@ def generate_bucketfs_http_url(bucketfs_config: BucketFSConfig, @typechecked(always=True) -def generate_bucket_http_url(bucket_config: BucketConfig, path_in_bucket: Union[None, str, PurePosixPath], - with_credentials: bool = False) -> urllib.parse.ParseResult: +def generate_bucket_http_url( + bucket_config: BucketConfig, + path_in_bucket: Union[None, str, PurePosixPath], + with_credentials: bool = False) -> urllib.parse.ParseResult: """ This function generates an HTTP[s] url for the given bucket or the path in the bucket with or without basic authentication (a template: http[s]://user:password@host:port) @@ -109,7 +117,8 @@ def generate_bucket_http_url(bucket_config: BucketConfig, path_in_bucket: Union[ :param with_credentials: If True, this function generates a url with basic authentication, default False :return: HTTP[S] URL of the bucket or the path in the bucket """ - url = generate_bucketfs_http_url(bucket_config.bucketfs_config, with_credentials) + url = generate_bucketfs_http_url(bucket_config.bucketfs_config, + with_credentials) if path_in_bucket is not None: path_in_bucket = _make_path_relative(path_in_bucket) else: @@ -127,7 +136,8 @@ def generate_bucket_http_url(bucket_config: BucketConfig, path_in_bucket: Union[ @typechecked(always=True) def create_auth_object(bucket_config: BucketConfig) -> HTTPBasicAuth: if bucket_config.bucketfs_config.connection_config is None: - raise TypeError("bucket_config.bucketfs_config.connection_config can't be None for this operation") + raise TypeError("bucket_config.bucketfs_config.connection_config " + "can't be None for this operation") auth = HTTPBasicAuth( bucket_config.bucketfs_config.connection_config.user, bucket_config.bucketfs_config.connection_config.pwd) diff --git a/exasol_bucketfs_utils_python/download.py b/exasol_bucketfs_utils_python/download.py index 9f8ac122..7c43fc0b 100644 --- a/exasol_bucketfs_utils_python/download.py +++ b/exasol_bucketfs_utils_python/download.py @@ -1,18 +1,19 @@ -import typing +from typing import IO, Any from pathlib import Path from tempfile import NamedTemporaryFile - import joblib import requests - from exasol_bucketfs_utils_python import bucketfs_utils from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_utils import generate_bucket_http_url -def download_from_bucketfs_to_file(bucket_config: BucketConfig, bucket_file_path: str, local_file_path: Path): +def download_from_bucketfs_to_file(bucket_config: BucketConfig, + bucket_file_path: str, + local_file_path: Path) -> None: """ - Download a file from the specified path in the bucket in the BucketFs and save as a local file + Download a file from the specified path in the bucket in the BucketFs + and save as a local file. :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from @@ -23,10 +24,12 @@ def download_from_bucketfs_to_file(bucket_config: BucketConfig, bucket_file_path download_from_bucketfs_to_fileobj(bucket_config, bucket_file_path, f) -def download_from_bucketfs_to_fileobj(bucket_config: BucketConfig, bucket_file_path: str, fileobj: typing.IO): +def download_from_bucketfs_to_fileobj(bucket_config: BucketConfig, + bucket_file_path: str, + fileobj: IO) -> None: """ - Download a file from the specified path in the bucket in the BucketFs into a given - `file object `_ + Download a file from the specified path in the BucketFs into a given file + object `_ :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from @@ -43,7 +46,8 @@ def download_from_bucketfs_to_fileobj(bucket_config: BucketConfig, bucket_file_p fileobj.write(chunk) -def download_from_bucketfs_to_string(bucket_config: BucketConfig, bucket_file_path: str) -> str: +def download_from_bucketfs_to_string(bucket_config: BucketConfig, + bucket_file_path: str) -> str: """ Download a file from the specified path in the bucket in the BucketFs into a string @@ -60,7 +64,8 @@ def download_from_bucketfs_to_string(bucket_config: BucketConfig, bucket_file_pa return response.text -def download_object_from_bucketfs_via_joblib(bucket_config: BucketConfig, bucket_file_path: str) -> typing.Any: +def download_object_from_bucketfs_via_joblib(bucket_config: BucketConfig, + bucket_file_path: str) -> Any: """ Download a file from the specified path in the bucket in the BucketFs and deserialize it via `joblib.load `_ @@ -70,7 +75,8 @@ def download_object_from_bucketfs_via_joblib(bucket_config: BucketConfig, bucket :return: The deserialized object which was downloaded from the BucketFS """ with NamedTemporaryFile() as temp_file: - download_from_bucketfs_to_fileobj(bucket_config, bucket_file_path, temp_file) + download_from_bucketfs_to_fileobj( + bucket_config, bucket_file_path, temp_file) temp_file.flush() temp_file.seek(0) obj = joblib.load(temp_file) diff --git a/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py index 530156bd..403179eb 100644 --- a/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py +++ b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py @@ -5,14 +5,16 @@ class GithubReleaseFileBucketFSUploader: - def __init__(self, file_to_download_name, github_user, repository_name, release_name, path_inside_bucket): + def __init__( + self, file_to_download_name: str, github_user: str, + repository_name: str, release_name: str, path_inside_bucket: str): self.file_to_download_name = file_to_download_name self.github_user = github_user self.repository_name = repository_name self.release_name = release_name self.path_inside_bucket = path_inside_bucket - def upload(self, address, username, password): + def upload(self, address: str, username: str, password: str) -> None: """ This method uploads the GitHub release into a selected Exasol bucket. @@ -24,20 +26,24 @@ def upload(self, address, username, password): download_url = self.__extract_download_url() r_download = requests.get(download_url, stream=True) upload_url = self.__build_upload_url(address) - requests.put(upload_url, data=r_download.iter_content(10 * 1024), auth=HTTPBasicAuth(username, password)) + requests.put( + upload_url, + data=r_download.iter_content(10 * 1024), + auth=HTTPBasicAuth(username, password)) - def __build_upload_url(self, address): + def __build_upload_url(self, address: str) -> str: if self.path_inside_bucket: address += self.path_inside_bucket address += self.file_to_download_name return address - def __extract_download_url(self): + def __extract_download_url(self) -> str: github_api_link = self.__build_github_api_link() release_link_extractor = ReleaseLinkExtractor(github_api_link) - download_url = release_link_extractor.get_link_by_release_name(self.file_to_download_name) + download_url = release_link_extractor.get_link_by_release_name( + self.file_to_download_name) return download_url - def __build_github_api_link(self): - return "https://api.github.com/repos/{github_user}/{repository_name}/releases/{release_name}".format( - github_user=self.github_user, repository_name=self.repository_name, release_name=self.release_name) + def __build_github_api_link(self) -> str: + return f"https://api.github.com/repos/{self.github_user}/" \ + f"{self.repository_name}/releases/{self.release_name}" diff --git a/exasol_bucketfs_utils_python/load_file_from_local_fs.py b/exasol_bucketfs_utils_python/load_file_from_local_fs.py index 66d2387d..ef4ea46c 100644 --- a/exasol_bucketfs_utils_python/load_file_from_local_fs.py +++ b/exasol_bucketfs_utils_python/load_file_from_local_fs.py @@ -1,12 +1,13 @@ from exasol_bucketfs_utils_python.bucket_config import BucketConfig from exasol_bucketfs_utils_python.bucketfs_utils import generate_bucket_udf_path -import typing +from typing import Any, IO from pathlib import Path from tempfile import NamedTemporaryFile import joblib -def read_file_from_bucketfs_to_string(bucket_file_path: str, bucket_config: BucketConfig) -> str: +def read_file_from_bucketfs_to_string(bucket_file_path: str, + bucket_config: BucketConfig) -> str: """ Read a file from the specified path in the bucket in the BucketFs into a string. Can be used inside of an UDF. @@ -23,7 +24,9 @@ def read_file_from_bucketfs_to_string(bucket_file_path: str, bucket_config: Buck return text_as_string -def read_file_from_bucketfs_to_file(bucket_file_path: str, bucket_config: BucketConfig, local_file_path: Path) -> None: +def read_file_from_bucketfs_to_file(bucket_file_path: str, + bucket_config: BucketConfig, + local_file_path: Path) -> None: """ Read a file from the specified path in the bucket in the BucketFs and save as a local file Can be used inside of an UDF. @@ -34,10 +37,13 @@ def read_file_from_bucketfs_to_file(bucket_file_path: str, bucket_config: Bucket :return: None """ with local_file_path.open("wb") as f: - read_file_from_bucketfs_to_fileobj(bucket_file_path, bucket_config, fileobj=f) + read_file_from_bucketfs_to_fileobj( + bucket_file_path, bucket_config, fileobj=f) -def read_file_from_bucketfs_to_fileobj(bucket_file_path: str, bucket_config: BucketConfig, fileobj: typing.IO) -> None: +def read_file_from_bucketfs_to_fileobj(bucket_file_path: str, + bucket_config: BucketConfig, + fileobj: IO) -> None: """ Download a file from the specified path in the bucket in the BucketFs into a given `file object `_ @@ -56,7 +62,9 @@ def read_file_from_bucketfs_to_fileobj(bucket_file_path: str, bucket_config: Buc fileobj.write(file.read()) -def read_file_from_bucketfs_via_joblib(bucket_file_path: str, bucket_config: BucketConfig) -> typing.Any: +def read_file_from_bucketfs_via_joblib( + bucket_file_path: str, + bucket_config: BucketConfig) -> Any: """ Download a file from the specified path in the bucket in the BucketFs and deserialize it via `joblib.load `_ diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index 76cd54c6..4d0e2a6b 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -1,10 +1,9 @@ -import typing +from typing import Any, IO from pathlib import PurePosixPath, Path from typing import Any - import joblib - -from exasol_bucketfs_utils_python.abstract_bucketfs_location import AbstractBucketFSLocation +from exasol_bucketfs_utils_python.abstract_bucketfs_location import \ + AbstractBucketFSLocation class LocalFSMockBucketFSLocation(AbstractBucketFSLocation): @@ -22,50 +21,63 @@ def get_complete_file_path_in_bucket(self, bucket_file_path) -> str: return str(PurePosixPath(self.base_path, bucket_file_path)) def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: - with open(self.get_complete_file_path_in_bucket(bucket_file_path), "rt") as f: + with open(self.get_complete_file_path_in_bucket( + bucket_file_path), "rt") as f: result = f.read() return result - def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any: - result = joblib.load(self.get_complete_file_path_in_bucket(bucket_file_path)) + def download_object_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: + result = joblib.load( + self.get_complete_file_path_in_bucket(bucket_file_path)) return result - def upload_string_to_bucketfs(self, bucket_file_path: str, string: str): + def upload_string_to_bucketfs(self, + bucket_file_path: str, + string: str) -> None: path = self.get_complete_file_path_in_bucket(bucket_file_path) Path(path).parent.mkdir(parents=True, exist_ok=True) with open(path, "wt") as f: f.write(string) - def upload_object_to_bucketfs_via_joblib(self, object: Any, + def upload_object_to_bucketfs_via_joblib(self, + object_: Any, bucket_file_path: str, - **kwargs): + **kwargs) -> None: path = self.get_complete_file_path_in_bucket(bucket_file_path) Path(path).parent.mkdir(parents=True, exist_ok=True) - joblib.dump(object, path, **kwargs) + joblib.dump(object_, path, **kwargs) def upload_fileobj_to_bucketfs(self, - fileobj: typing.IO, - bucket_file_path: str): + fileobj: IO, + bucket_file_path: str) -> None: path = self.get_complete_file_path_in_bucket(bucket_file_path) Path(path).parent.mkdir(parents=True, exist_ok=True) with open(path, "wb") as f: for chunk in iter(lambda: fileobj.read(10000), ''): f.write(chunk) - def read_file_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: typing.IO): + def read_file_from_bucketfs_to_fileobj(self, + bucket_file_path: str, + fileobj: IO) -> None: bucket_path = self.get_complete_file_path_in_bucket(bucket_file_path) with open(bucket_path, "rb") as read_file: read_file.seek(0) fileobj.write(read_file.read()) - def read_file_from_bucketfs_to_file(self, bucket_file_path: str, local_file_path: Path): + def read_file_from_bucketfs_to_file(self, + bucket_file_path: str, + local_file_path: Path) -> None: with open(local_file_path, "wb") as fileobj: self.read_file_from_bucketfs_to_fileobj(bucket_file_path, fileobj) - def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str: + def read_file_from_bucketfs_to_string(self, + bucket_file_path: str) -> str: result = self.download_from_bucketfs_to_string(bucket_file_path) return result - def read_file_from_bucketfs_via_joblib(self, bucket_file_path: str) -> typing.Any: - result = joblib.load(self.get_complete_file_path_in_bucket(bucket_file_path)) + def read_file_from_bucketfs_via_joblib(self, + bucket_file_path: str) -> Any: + result = joblib.load( + self.get_complete_file_path_in_bucket(bucket_file_path)) return result diff --git a/exasol_bucketfs_utils_python/release_link_extractor.py b/exasol_bucketfs_utils_python/release_link_extractor.py index ac2076c3..39505207 100644 --- a/exasol_bucketfs_utils_python/release_link_extractor.py +++ b/exasol_bucketfs_utils_python/release_link_extractor.py @@ -2,30 +2,36 @@ class ReleaseLinkExtractor: - def __init__(self, repository_api_link): + def __init__(self, repository_api_link: str): """ Create a new instance of ReleaseLinkExtractor class. :param repository_api_link: Link to the GitHub API page with the latest release. """ self.repository_api_link = repository_api_link - def get_link_by_release_name(self, file_to_download_name): + def get_link_by_release_name(self, file_to_download_name: str) -> str: """ - This method extracts a link from the GitHub API page searching by a release name. + This method extracts a link from the GitHub API page searching + by a release name. + :param file_to_download_name: the name of the file :return: a link in a string format """ response = requests.get(self.repository_api_link) json_release_page = response.json() list_of_available_releases = json_release_page["assets"] - result_link = self.__find_link(list_of_available_releases, file_to_download_name) + result_link = self.__find_link( + list_of_available_releases, file_to_download_name) if result_link is not None: return result_link else: - raise ValueError( - 'Release with the name ' + file_to_download_name + ' was not found. Please check the name or select another release') + raise ValueError(f'Release with the name {file_to_download_name} ' + f'was not found. Please check the name or ' + f'select another release') - def __find_link(self, list_of_available_releases, release_name): + def __find_link(self, + list_of_available_releases: list, + release_name: str) -> str: for release in list_of_available_releases: if release_name in release["name"]: return release["browser_download_url"] diff --git a/exasol_bucketfs_utils_python/upload.py b/exasol_bucketfs_utils_python/upload.py index e8ca518a..b674dedf 100644 --- a/exasol_bucketfs_utils_python/upload.py +++ b/exasol_bucketfs_utils_python/upload.py @@ -2,17 +2,18 @@ from tempfile import NamedTemporaryFile from typing import Tuple, IO, Any from urllib.parse import ParseResult - import joblib import requests - from exasol_bucketfs_utils_python import bucketfs_utils from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.bucketfs_utils import generate_bucket_http_url, generate_bucket_udf_path +from exasol_bucketfs_utils_python.bucketfs_utils import \ + generate_bucket_http_url, generate_bucket_udf_path -def upload_file_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str, local_file_path: Path) \ - -> Tuple[ParseResult, PurePosixPath]: +def upload_file_to_bucketfs(bucket_config: BucketConfig, + bucket_file_path: str, + local_file_path: Path) -> \ + Tuple[ParseResult, PurePosixPath]: """ This function uploads a file to the specified path in a bucket of the BucketFS. @@ -25,8 +26,10 @@ def upload_file_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str, return upload_fileobj_to_bucketfs(bucket_config, bucket_file_path, f) -def upload_fileobj_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str, fileobj: IO) \ - -> Tuple[ParseResult, PurePosixPath]: +def upload_fileobj_to_bucketfs(bucket_config: BucketConfig, + bucket_file_path: str, + fileobj: IO) -> \ + Tuple[ParseResult, PurePosixPath]: """ This function uploads a `file object `_ to the specified path in a bucket of the BucketFS. @@ -46,8 +49,10 @@ def upload_fileobj_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: st return url, path -def upload_string_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str, string: str) \ - -> Tuple[ParseResult, PurePosixPath]: +def upload_string_to_bucketfs(bucket_config: BucketConfig, + bucket_file_path: str, + string: str) -> \ + Tuple[ParseResult, PurePosixPath]: """ This function uploads a string to the specified path in a bucket of the BucketFS. @@ -67,9 +72,10 @@ def upload_string_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str def upload_object_to_bucketfs_via_joblib(object: Any, - bucket_config: BucketConfig, bucket_file_path: str, - **kwargs) \ - -> Tuple[ParseResult, PurePosixPath]: + bucket_config: BucketConfig, + bucket_file_path: str, + **kwargs) -> \ + Tuple[ParseResult, PurePosixPath]: """ This function serializes a python object with `joblib.dump `_ @@ -85,4 +91,5 @@ def upload_object_to_bucketfs_via_joblib(object: Any, joblib.dump(object, temp_file.name, **kwargs) temp_file.flush() temp_file.seek(0) - return upload_fileobj_to_bucketfs(bucket_config, bucket_file_path, temp_file) + return upload_fileobj_to_bucketfs( + bucket_config, bucket_file_path, temp_file) diff --git a/tests/test_bucketfs_location.py b/tests/test_bucketfs_location.py index f1ea0d40..5edcd6ea 100644 --- a/tests/test_bucketfs_location.py +++ b/tests/test_bucketfs_location.py @@ -27,7 +27,9 @@ def test_upload_download_string_from_different_instance(): bucket_config=bucketfs_location_upload.bucket_config) -class TestValue(): +class TestValue: + __test__ = False + def __init__(self, value: str): self.value = value diff --git a/tests/test_localfs_mock_bucketfs_location.py b/tests/test_localfs_mock_bucketfs_location.py index ee335f19..402bbcbe 100644 --- a/tests/test_localfs_mock_bucketfs_location.py +++ b/tests/test_localfs_mock_bucketfs_location.py @@ -14,7 +14,9 @@ def test_upload_download_string_from_different_instance(): assert result == test_string -class TestValue(): +class TestValue: + __test__ = False + def __init__(self, value: str): self.value = value diff --git a/tests/test_upload_download.py b/tests/test_upload_download.py index 89deb9c7..e96bc654 100644 --- a/tests/test_upload_download.py +++ b/tests/test_upload_download.py @@ -80,6 +80,8 @@ def test_string_upload_download(): class TestClass: + __test__ = False + def __init__(self, attribute: str): self.attribute = attribute From 2d697ad16591ae21ef0a56188627a6e606e36635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Tue, 29 Mar 2022 11:37:42 +0200 Subject: [PATCH 3/4] Removed the conversion --- .gitignore | 3 +++ doc/changes/changes_0.2.0.md | 2 ++ tests/fixtures/upload_language_container_fixture.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 696495ba..d5fffc9e 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ poetry.lock # Sphinx doc/_build doc/api + +# Language container +.build_output diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md index cb8c1914..832e27c4 100644 --- a/doc/changes/changes_0.2.0.md +++ b/doc/changes/changes_0.2.0.md @@ -8,6 +8,8 @@ Code name: t.b.d ## Bug Fixes + - #54: Removed PosixPath conversion from alter session string + ## Documentation ## Refactoring diff --git a/tests/fixtures/upload_language_container_fixture.py b/tests/fixtures/upload_language_container_fixture.py index b778acad..b6d6495b 100644 --- a/tests/fixtures/upload_language_container_fixture.py +++ b/tests/fixtures/upload_language_container_fixture.py @@ -31,7 +31,7 @@ def upload_language_container(pyexasol_connection, language_container): pwd=container_connection.password, base_path=None) container_path = Path(language_container["container_path"]) - alter_session = Path(language_container["alter_session"]) + alter_session = language_container["alter_session"] pyexasol_connection.execute(f"ALTER SESSION SET SCRIPT_LANGUAGES='{alter_session}'") with open(container_path, "rb") as container_file: container_bucketfs_location.upload_fileobj_to_bucketfs(container_file, "ml.tar") From 2342c9fd26b9e7e91bc8b5243cbaf22ac52e6e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cumitbuyuksahin=E2=80=9D?= Date: Wed, 30 Mar 2022 13:33:23 +0200 Subject: [PATCH 4/4] Updated changelog --- doc/changes/changes_0.2.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md index 832e27c4..c22c8db7 100644 --- a/doc/changes/changes_0.2.0.md +++ b/doc/changes/changes_0.2.0.md @@ -14,6 +14,8 @@ Code name: t.b.d ## Refactoring + - #58: Added Python type hints + ## Security - #51: Added fixed numpy version build from source because of Buffer Overflow vulnerability in NumPy