From 83918ffd6f9e5b597b5693635cd7634d0ab25f8a Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 26 Oct 2020 11:51:55 +0100 Subject: [PATCH 01/20] #1: Added initial implementation. --- .github/workflows/check_setup_py.yaml | 28 ++++ .gitignore | 135 ++++++++++++++++++ README.md | 2 + doc/changes/changelog.md | 3 + doc/changes/changes_0.1.0.md | 7 + .../GithubReleaseFileBucketFSUploader.py | 43 ++++++ extension_downloading/ReleaseLinkExtractor.py | 31 ++++ githooks/install.sh | 49 +++++++ githooks/pre-commit | 11 ++ githooks/pre-push | 43 ++++++ githooks/prohibit_commit_to_master.sh | 8 ++ githooks/update_setup_py.sh | 22 +++ pyproject.toml | 29 ++++ 13 files changed, 411 insertions(+) create mode 100644 .github/workflows/check_setup_py.yaml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 doc/changes/changelog.md create mode 100644 doc/changes/changes_0.1.0.md create mode 100644 extension_downloading/GithubReleaseFileBucketFSUploader.py create mode 100644 extension_downloading/ReleaseLinkExtractor.py create mode 100755 githooks/install.sh create mode 100755 githooks/pre-commit create mode 100755 githooks/pre-push create mode 100755 githooks/prohibit_commit_to_master.sh create mode 100755 githooks/update_setup_py.sh create mode 100644 pyproject.toml diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml new file mode 100644 index 00000000..18727668 --- /dev/null +++ b/.github/workflows/check_setup_py.yaml @@ -0,0 +1,28 @@ +name: Check if setup.py is up to date + +on: [push, pull_request] + +jobs: + check_setup_py: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run convert + uses: dephell/dephell_action@master + with: + dephell-env: convert + - name: Show changes on working copy + run: git status --porcelain=v1 -uno + - name: Show diff on working copy + run: git diff + - name: Check if setup.py changed + run: | + [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3428480c --- /dev/null +++ b/.gitignore @@ -0,0 +1,135 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Poetry +poetry.lock + +# PyCharm +.idea \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..5cf45ba8 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# BucketFS Utils Python + diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md new file mode 100644 index 00000000..387930a1 --- /dev/null +++ b/doc/changes/changelog.md @@ -0,0 +1,3 @@ +# Changelog + +* [0.1.0](changes_0.1.0.md) \ No newline at end of file diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md new file mode 100644 index 00000000..d8ef7086 --- /dev/null +++ b/doc/changes/changes_0.1.0.md @@ -0,0 +1,7 @@ +# BucketFS Utils Python 0.1.0, released 2020-??-?? + +Code name: + +## Features + +#1: Added initial implementation. \ No newline at end of file diff --git a/extension_downloading/GithubReleaseFileBucketFSUploader.py b/extension_downloading/GithubReleaseFileBucketFSUploader.py new file mode 100644 index 00000000..55e1790f --- /dev/null +++ b/extension_downloading/GithubReleaseFileBucketFSUploader.py @@ -0,0 +1,43 @@ +import requests +from ReleaseLinkExtractor import ReleaseLinkExtractor + + +class GithubReleaseFileBucketFSUploader: + def __init__(self, file_to_download_name, github_user, repository_name, release_name, path_inside_bucket): + self.file_to_download_name = file_to_download_name + self.github_user = github_user + self.repository_name = repository_name + self.release_name = release_name + self.path_inside_bucket = path_inside_bucket + + def upload(self, address, username, password): + """ + This method uploads the GitHub release into a selected Exasol bucket. + :param address: address in the format 'http://:/' + :param username: bucket writing username + :param password: bucket writing password + """ + download_url = self.__extract_download_url() + r_download = requests.get(download_url, stream=True) + upload_url = self.__build_upload_url(address, username, password) + requests.put(upload_url, data=r_download.iter_content(10 * 1024)) + + def __build_upload_url(self, address, username, password): + connection_first_part = 'http://' + split_url = address.split(connection_first_part, 1) + upload_url = "{connection_first_part}{username}:{password}@{url}/".format( + connection_first_part=connection_first_part, username=username, password=password, url=split_url[1]) + if self.path_inside_bucket: + upload_url += self.path_inside_bucket + upload_url += self.file_to_download_name + return upload_url + + def __extract_download_url(self): + github_api_link = self.__build_github_api_link() + release_link_extractor = ReleaseLinkExtractor(github_api_link) + download_url = release_link_extractor.get_link_by_release_name(self.file_to_download_name) + return download_url + + def __build_github_api_link(self): + return "https://api.github.com/repos/{github_user}/{repository_name}/releases/{release_name}".format( + github_user=self.github_user, repository_name=self.repository_name, release_name=self.release_name) diff --git a/extension_downloading/ReleaseLinkExtractor.py b/extension_downloading/ReleaseLinkExtractor.py new file mode 100644 index 00000000..a5d6d0fc --- /dev/null +++ b/extension_downloading/ReleaseLinkExtractor.py @@ -0,0 +1,31 @@ +import requests + + +class ReleaseLinkExtractor: + def __init__(self, repository_api_link): + """ + Create a new instance of ReleaseLinkExtractor class. + :param repository_api_link: Link to the GitHub API page with the latest release. + """ + self.repository_api_link = repository_api_link + + def get_link_by_release_name(self, file_to_download_name): + """ + This method extracts a link from the GitHub API page searching by a release name. + :param file_to_download_name: the name of the file + :return: a link in a string format + """ + response = requests.get(self.repository_api_link) + json_release_page = response.json() + list_of_available_releases = json_release_page["assets"] + result_link = self.__find_link(list_of_available_releases, file_to_download_name) + if result_link is not None: + return result_link + else: + raise ValueError( + 'Release with the name ' + file_to_download_name + ' was not found. Please check the name or select another release') + + def __find_link(self, list_of_available_releases, release_name): + for release in list_of_available_releases: + if release_name in release["name"]: + return release["browser_download_url"] \ No newline at end of file diff --git a/githooks/install.sh b/githooks/install.sh new file mode 100755 index 00000000..1a873788 --- /dev/null +++ b/githooks/install.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +REPO_DIR=$(git rev-parse --show-toplevel) +REPO_DIR="$(readlink -f "${REPO_DIR}")" +GIT_DIR="$REPO_DIR/.git" +GIT_DIR="$(readlink -f "${GIT_DIR}")" + +if [[ ! -d "$GIT_DIR" ]]; then + if [[ -d "$REPO_DIR/../.git" ]]; then + GIT_DIR="$REPO_DIR/../.git" + GITHOOKS_PATH="$GIT_DIR/modules/bucketfs-utils-python/hooks" + else + echo "$GIT_DIR is not a git directory." >&2 + exit 1 + fi +else + GITHOOKS_PATH="$GIT_DIR/hooks" +fi + +GITHOOKS_PATH="$(readlink -f "${GITHOOKS_PATH}")" + +copy_hook() { + local SCRIPT_PATH="$SCRIPT_DIR/$1" + local GITHOOK_PATH="$GITHOOKS_PATH/$2" + local RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH") + pushd "$GITHOOKS_PATH" > /dev/null + if [ -e "$GITHOOK_PATH" ] || [ -L "$GITHOOK_PATH" ] + then + echo + echo "Going to delete old hook $GITHOOK_PATH" + rm "$GITHOOK_PATH" > /dev/null + fi + echo + echo "Link hook to script" >&2 + echo "Hook-Path: $GITHOOK_PATH" >&2 + echo "Script-path: $SCRIPT_PATH" >&2 + echo + ln -s "$RELATIVE_PATH" "$2" > /dev/null + chmod +x "$SCRIPT_PATH" > /dev/null + popd > /dev/null +} + +copy_hook pre-commit pre-commit +copy_hook pre-commit post-rewrite +copy_hook pre-push pre-push diff --git a/githooks/pre-commit b/githooks/pre-commit new file mode 100755 index 00000000..2c8000ff --- /dev/null +++ b/githooks/pre-commit @@ -0,0 +1,11 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +REPO_DIR=$(git rev-parse --show-toplevel) +GITHOOKS_PATH="$REPO_DIR/githooks" +pushd "$REPO_DIR" +bash "$GITHOOKS_PATH/prohibit_commit_to_master.sh" +bash "$GITHOOKS_PATH/update_setup_py.sh" +popd diff --git a/githooks/pre-push b/githooks/pre-push new file mode 100755 index 00000000..5ae1a28a --- /dev/null +++ b/githooks/pre-push @@ -0,0 +1,43 @@ +#!/bin/bash +protected_branches=( master ) +for i in "${protected_branches[@]}" +do + + protected_branch=$i + + policy='[Policy] Never push, force push or delete the '$protected_branch' branch! (Prevented with pre-push hook.)' + + current_branch=$(git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,') + + push_command=$(ps -ocommand= -p $PPID) + + is_destructive='force|delete|\-f' + + will_remove_protected_branch=':'$protected_branch + + do_exit(){ + echo $policy + exit 1 + } + + if [[ $push_command =~ $is_destructive ]] && [ $current_branch = $protected_branch ]; then + do_exit + fi + + if [[ $push_command =~ $is_destructive ]] && [[ $push_command =~ $protected_branch ]]; then + do_exit + fi + + if [[ $push_command =~ $will_remove_protected_branch ]]; then + do_exit + fi + + if [[ $protected_branch == $current_branch ]]; then + do_exit + fi + +done + +unset do_exit + +exit 0 diff --git a/githooks/prohibit_commit_to_master.sh b/githooks/prohibit_commit_to_master.sh new file mode 100755 index 00000000..632d10e7 --- /dev/null +++ b/githooks/prohibit_commit_to_master.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +branch="$(git rev-parse --abbrev-ref HEAD)" + +if [ "$branch" = "master" ]; then + echo "You can't commit directly to master branch" + exit 1 +fi diff --git a/githooks/update_setup_py.sh b/githooks/update_setup_py.sh new file mode 100755 index 00000000..d3ac28e2 --- /dev/null +++ b/githooks/update_setup_py.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# define colors for use in output +green='\033[0;32m' +no_color='\033[0m' +grey='\033[0;90m' + +echo -e "Update setup.py with dephell convert ${grey}(pre-commit hook)${no_color} " + +# Jump to the current project's root directory (the one containing +# .git/) +ROOT_DIR=$(git rev-parse --show-cdup) + +pushd "$ROOT_DIR" > /dev/null + +dephell venv run --env convert +git add setup.py README.rst + +popd > /dev/null diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..09ee21f9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.poetry] +name = "bucketfs-utils-python" +version = "0.1.0" +description = "BucketFS utilities for the Python programming language" + +license = "MIT" + +authors = [ + "Torsten Kilias ", + "Anastasiia Sergienko " +] + +readme = 'README.md' + +repository = "https://github.com/exasol/bucketfs-utils-python" +homepage = "https://github.com/exasol/bucketfs-utils-python" + +keywords = ['exasol', 'bucketfs'] + +[tool.poetry.dependencies] +python = ">=3.6.1" + +[tool.dephell.main] +from = {format = "poetry", path = "pyproject.toml"} +to = {format = "setuppy", path = "setup.py"} + +[tool.dephell.convert] +from = {format = "poetry", path = "pyproject.toml"} +command = "dephell deps convert" \ No newline at end of file From 79c010c2fa7ab4da7e1cf8b8e5e9a5e6e0394b0e Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 26 Oct 2020 11:59:00 +0100 Subject: [PATCH 02/20] #1: Added initial implementation. --- .github/workflows/check_setup_py.yaml | 28 --------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/check_setup_py.yaml diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml deleted file mode 100644 index 18727668..00000000 --- a/.github/workflows/check_setup_py.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check if setup.py is up to date - -on: [push, pull_request] - -jobs: - check_setup_py: - strategy: - fail-fast: false - matrix: - python-version: [3.6, 3.7, 3.8] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Run convert - uses: dephell/dephell_action@master - with: - dephell-env: convert - - name: Show changes on working copy - run: git status --porcelain=v1 -uno - - name: Show diff on working copy - run: git diff - - name: Check if setup.py changed - run: | - [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] \ No newline at end of file From 13d1ddaa852e8131b893fd409a09875a1f992ba3 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 26 Oct 2020 12:02:47 +0100 Subject: [PATCH 03/20] #1: Added workflows --- .github/workflows/check_setup_py.yaml | 28 +++++++++++++++++++++++++++ .github/workflows/pytest.yaml | 21 ++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/workflows/check_setup_py.yaml create mode 100644 .github/workflows/pytest.yaml diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml new file mode 100644 index 00000000..27fe805d --- /dev/null +++ b/.github/workflows/check_setup_py.yaml @@ -0,0 +1,28 @@ +name: Check if setup.py is up to date + +on: [push, pull_request] + +jobs: + check_setup_py: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run convert + uses: dephell/dephell_action@master + with: + dephell-env: convert + - name: Show changes on working copy + run: git status --porcelain=v1 -uno + - name: Show diff on working copy + run: git diff + - name: Check if setup.py changed + run: | + [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 00000000..319251c0 --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,21 @@ +name: Run pytest + +on: [push, pull_request] + +jobs: + pytest: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run pytest + uses: dephell/dephell_action@master + with: + dephell-env: pytest From 67cafd4faea3602bce79562c9c7ffe911bcd615f Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 26 Oct 2020 12:13:14 +0100 Subject: [PATCH 04/20] #1: Removed test workflow --- .github/workflows/pytest.yaml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/pytest.yaml diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml deleted file mode 100644 index 319251c0..00000000 --- a/.github/workflows/pytest.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Run pytest - -on: [push, pull_request] - -jobs: - pytest: - strategy: - fail-fast: false - matrix: - python-version: [3.6, 3.7, 3.8] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Run pytest - uses: dephell/dephell_action@master - with: - dephell-env: pytest From 0f3c71437e2a35e76671c7d33bfc661934cd81a3 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Fri, 6 Nov 2020 16:26:59 +0100 Subject: [PATCH 05/20] #1: Added a draft for integration test --- .github/workflows/pytest.yaml | 30 +++++ .../GithubReleaseFileBucketFSUploader.py | 3 +- extension_downloading/ReleaseLinkExtractor.py | 2 +- .../TestGithubReleaseFileBucketFSUploader.py | 103 ++++++++++++++++++ tests/bucketfs_utils.py | 57 ++++++++++ 5 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pytest.yaml create mode 100644 tests/TestGithubReleaseFileBucketFSUploader.py create mode 100644 tests/bucketfs_utils.py diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 00000000..d576158d --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,30 @@ +name: Run pytest tests + +on: [push, pull_request] + +jobs: + integration_tests: + strategy: + fail-fast: false + matrix: + python-version: [3.6] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: abatilo/actions-poetry@v2.0.0 + - name: Poetry install + run: poetry install + + - name: Checkout test environment + run: git clone https://github.com/exasol/integration-test-docker-environment.git + working-directory: .. + - name: Spawn EXASOL environemnt + run: ./start-test-env spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666 --db-mem-size 4GB + working-directory: ../integration-test-docker-environment + + - name: Poetry run pytest integration tests + run: poetry run pytest tests \ No newline at end of file diff --git a/extension_downloading/GithubReleaseFileBucketFSUploader.py b/extension_downloading/GithubReleaseFileBucketFSUploader.py index 55e1790f..902611ad 100644 --- a/extension_downloading/GithubReleaseFileBucketFSUploader.py +++ b/extension_downloading/GithubReleaseFileBucketFSUploader.py @@ -1,5 +1,6 @@ import requests -from ReleaseLinkExtractor import ReleaseLinkExtractor + +from extension_downloading.ReleaseLinkExtractor import ReleaseLinkExtractor class GithubReleaseFileBucketFSUploader: diff --git a/extension_downloading/ReleaseLinkExtractor.py b/extension_downloading/ReleaseLinkExtractor.py index a5d6d0fc..ac2076c3 100644 --- a/extension_downloading/ReleaseLinkExtractor.py +++ b/extension_downloading/ReleaseLinkExtractor.py @@ -28,4 +28,4 @@ def get_link_by_release_name(self, file_to_download_name): def __find_link(self, list_of_available_releases, release_name): for release in list_of_available_releases: if release_name in release["name"]: - return release["browser_download_url"] \ No newline at end of file + return release["browser_download_url"] diff --git a/tests/TestGithubReleaseFileBucketFSUploader.py b/tests/TestGithubReleaseFileBucketFSUploader.py new file mode 100644 index 00000000..97673942 --- /dev/null +++ b/tests/TestGithubReleaseFileBucketFSUploader.py @@ -0,0 +1,103 @@ +import os +import re +import subprocess +import textwrap +from pathlib import Path + +import pyexasol + +import extension_downloading + +from extension_downloading.GithubReleaseFileBucketFSUploader import GithubReleaseFileBucketFSUploader +from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, upload_file_to_bucketfs, BucketFSCredentials + + +class DatabaseCredentials: + def __init__(self, host="localhost", port=8888, user="sys", pwd="exasol"): + self.host = host + self.port = port + self.user = user + self.pwd = pwd + + +class DatabaseConfig: + def __init__(self, credentials: DatabaseCredentials, schema: str): + self.credentials = credentials + self.schema = schema + + +def find_project_base_directory(): + return Path(extension_downloading.__file__).parent.parent + + +def upload_file_to_bucket(bucketfs_config): + release_uploader = \ + GithubReleaseFileBucketFSUploader(file_to_download_name="virtual-schema-dist", + github_user="exasol", + repository_name="exasol-virtual-schema", + release_name="latest", + path_inside_bucket="virtualschemas/") + release_uploader.upload( + generate_bucketfs_url(bucketfs_config, file_name=None, with_credentials=False), + username=bucketfs_config.credentials.user, + password=bucketfs_config.credentials.pwd) + + +def upload_sdist(bucketfs_config: BucketFsConfig): + result = subprocess.run(["echo $PATH; poetry build"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + result.check_returncode() + project_base_directory = find_project_base_directory() + dist_directory = Path(project_base_directory, "dist") + tar_release_file_name = next(filter(lambda x: x.endswith(".tar.gz"), os.listdir(dist_directory))) + tar_release_file_path = Path(dist_directory, tar_release_file_name) + url, path = upload_file_to_bucketfs(bucketfs_config, tar_release_file_name, tar_release_file_path) + return url, path + + +def prepare_test(con, database_config): + bucketfs_config = BucketFsConfig(BucketFSCredentials()) + sdist_url, sdist_path = upload_sdist(bucketfs_config) + con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") + con.execute(f"OPEN SCHEMA {database_config.schema};") + + +def test_tensorflow_udf_train(): + database_config = DatabaseConfig(DatabaseCredentials(), "test_tensorflow_udf") + bucketfs_config = BucketFsConfig(BucketFSCredentials()) + + upload_file_to_bucket(bucketfs_config) + with pyexasol.connect( + dsn=f"{database_config.credentials.host}:{database_config.credentials.port}", + user=database_config.credentials.user, + password=database_config.credentials.pwd) as con: + try: + prepare_test(con, database_config) + con.execute(textwrap.dedent(f""" + CREATE OR REPLACE PYTHON SCALAR SCRIPT EXA_toolbox.bucketfs_ls(my_path VARCHAR(256)) + EMITS (files VARCHAR(256)) AS + import subprocess + + def run(c): + try: + p = subprocess.Popen('ls -F ' + c.my_path, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + close_fds = True, + shell = True) + out, err = p.communicate() + for line in out.strip().split('\n'): + c.emit(line) + finally: + if p is not None: + try: p.kill() + except: pass + / + """)) + result = con.execute("SELECT bucketfs_ls('/buckets/bfsdefault/default')").fetchall() + output = result[0][0] + print(output) + assert re.match( + r"""some text""", + output) + finally: + con.execute(f"DROP SCHEMA IF EXISTS {database_config.schema} CASCADE;") diff --git a/tests/bucketfs_utils.py b/tests/bucketfs_utils.py new file mode 100644 index 00000000..2b58f2ea --- /dev/null +++ b/tests/bucketfs_utils.py @@ -0,0 +1,57 @@ +from pathlib import Path + +import requests + + +class BucketFSCredentials: + def __init__(self, host="localhost", port=6666, user="w", pwd="write"): + self.host = host + self.port = port + self.user = user + self.pwd = pwd + + +class BucketFsConfig: + def __init__(self, credentials: BucketFSCredentials, bucket="default", bucketfs_name="bfsdefault"): + self.credentials = credentials + self.bucket = bucket + self.bucketfs_name = bucketfs_name + + +def upload_file_to_bucketfs(bucket_config: BucketFsConfig, file_name: str, file_path: Path): + with file_path.open("rb") as f: + url = generate_bucketfs_url(bucket_config, file_name) + response = requests.put(url, data=f) + response.raise_for_status() + path = get_bucketfs_path(bucket_config, file_name) + return url, path + + +def upload_string_to_bucketfs(bucketfs_config: BucketFsConfig, file_name: str, string: str): + url = generate_bucketfs_url(bucketfs_config, file_name) + response = requests.put(url, data=string.encode("UTF-8")) + response.raise_for_status() + path = get_bucketfs_path(bucketfs_config, file_name) + return url, path + + +def get_bucketfs_path(bucketfs_config: BucketFsConfig, file_name: str): + archive_extensions = [".tar.gz", ".tar.bz2", ".zip", ".tar"] + for extension in archive_extensions: + if file_name.endswith(extension): + file_name = file_name[:-len(extension)] + break + path = f"/buckets/{bucketfs_config.bucketfs_name}/{bucketfs_config.bucket}/{file_name}" + return path + + +def generate_bucketfs_url(bucketfs_config: BucketFsConfig, file_name: str, with_credentials: bool = True): + if with_credentials: + credentials = f"{bucketfs_config.credentials.user}:{bucketfs_config.credentials.pwd}@" + else: + credentials = "" + url = f"http://{credentials}" \ + f"{bucketfs_config.credentials.host}:{bucketfs_config.credentials.port}/{bucketfs_config.bucket}/" + if file_name is not None: + url += f"{file_name}" + return url From e58d32cf8fe31a76d8925b6a9433b5549275337d Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 9 Nov 2020 14:50:37 +0100 Subject: [PATCH 06/20] #1: Fixed the integration test --- .github/workflows/check_setup_py.yaml | 2 +- pyproject.toml | 11 +++++-- .../TestGithubReleaseFileBucketFSUploader.py | 29 +++---------------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml index 27fe805d..0b5a9731 100644 --- a/.github/workflows/check_setup_py.yaml +++ b/.github/workflows/check_setup_py.yaml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7] runs-on: ubuntu-latest steps: diff --git a/pyproject.toml b/pyproject.toml index 09ee21f9..083bbb50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,10 +20,15 @@ keywords = ['exasol', 'bucketfs'] [tool.poetry.dependencies] python = ">=3.6.1" +[tool.poetry.dev-dependencies] +pytest = "^6.1.1" +pytest-cov = "^2.10.1" +pyexasol = "^0.14.1" + [tool.dephell.main] -from = {format = "poetry", path = "pyproject.toml"} -to = {format = "setuppy", path = "setup.py"} +from = { format = "poetry", path = "pyproject.toml" } +to = { format = "setuppy", path = "setup.py" } [tool.dephell.convert] -from = {format = "poetry", path = "pyproject.toml"} +from = { format = "poetry", path = "pyproject.toml" } command = "dephell deps convert" \ No newline at end of file diff --git a/tests/TestGithubReleaseFileBucketFSUploader.py b/tests/TestGithubReleaseFileBucketFSUploader.py index 97673942..d924ef06 100644 --- a/tests/TestGithubReleaseFileBucketFSUploader.py +++ b/tests/TestGithubReleaseFileBucketFSUploader.py @@ -26,10 +26,6 @@ def __init__(self, credentials: DatabaseCredentials, schema: str): self.schema = schema -def find_project_base_directory(): - return Path(extension_downloading.__file__).parent.parent - - def upload_file_to_bucket(bucketfs_config): release_uploader = \ GithubReleaseFileBucketFSUploader(file_to_download_name="virtual-schema-dist", @@ -43,26 +39,8 @@ def upload_file_to_bucket(bucketfs_config): password=bucketfs_config.credentials.pwd) -def upload_sdist(bucketfs_config: BucketFsConfig): - result = subprocess.run(["echo $PATH; poetry build"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - result.check_returncode() - project_base_directory = find_project_base_directory() - dist_directory = Path(project_base_directory, "dist") - tar_release_file_name = next(filter(lambda x: x.endswith(".tar.gz"), os.listdir(dist_directory))) - tar_release_file_path = Path(dist_directory, tar_release_file_name) - url, path = upload_file_to_bucketfs(bucketfs_config, tar_release_file_name, tar_release_file_path) - return url, path - - -def prepare_test(con, database_config): - bucketfs_config = BucketFsConfig(BucketFSCredentials()) - sdist_url, sdist_path = upload_sdist(bucketfs_config) - con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") - con.execute(f"OPEN SCHEMA {database_config.schema};") - - -def test_tensorflow_udf_train(): - database_config = DatabaseConfig(DatabaseCredentials(), "test_tensorflow_udf") +def test_uploading_github_release_to_bucketfs(): + database_config = DatabaseConfig(DatabaseCredentials(), "test_schema") bucketfs_config = BucketFsConfig(BucketFSCredentials()) upload_file_to_bucket(bucketfs_config) @@ -71,7 +49,8 @@ def test_tensorflow_udf_train(): user=database_config.credentials.user, password=database_config.credentials.pwd) as con: try: - prepare_test(con, database_config) + con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") + con.execute(f"OPEN SCHEMA {database_config.schema};") con.execute(textwrap.dedent(f""" CREATE OR REPLACE PYTHON SCALAR SCRIPT EXA_toolbox.bucketfs_ls(my_path VARCHAR(256)) EMITS (files VARCHAR(256)) AS From 81acbd46446f830913701eb25828610aec221200 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 9 Nov 2020 15:10:19 +0100 Subject: [PATCH 07/20] #1: Renamed modules --- ...oader.py => github_release_file_bucketfs_uploader.py} | 2 +- ...ReleaseLinkExtractor.py => release_link_extractor.py} | 0 pyproject.toml | 1 + ....py => test_github_release_file_bucketfs_uploader.py} | 9 ++------- 4 files changed, 4 insertions(+), 8 deletions(-) rename extension_downloading/{GithubReleaseFileBucketFSUploader.py => github_release_file_bucketfs_uploader.py} (96%) rename extension_downloading/{ReleaseLinkExtractor.py => release_link_extractor.py} (100%) rename tests/{TestGithubReleaseFileBucketFSUploader.py => test_github_release_file_bucketfs_uploader.py} (92%) diff --git a/extension_downloading/GithubReleaseFileBucketFSUploader.py b/extension_downloading/github_release_file_bucketfs_uploader.py similarity index 96% rename from extension_downloading/GithubReleaseFileBucketFSUploader.py rename to extension_downloading/github_release_file_bucketfs_uploader.py index 902611ad..c6f4326b 100644 --- a/extension_downloading/GithubReleaseFileBucketFSUploader.py +++ b/extension_downloading/github_release_file_bucketfs_uploader.py @@ -1,6 +1,6 @@ import requests -from extension_downloading.ReleaseLinkExtractor import ReleaseLinkExtractor +from extension_downloading.release_link_extractor import ReleaseLinkExtractor class GithubReleaseFileBucketFSUploader: diff --git a/extension_downloading/ReleaseLinkExtractor.py b/extension_downloading/release_link_extractor.py similarity index 100% rename from extension_downloading/ReleaseLinkExtractor.py rename to extension_downloading/release_link_extractor.py diff --git a/pyproject.toml b/pyproject.toml index 083bbb50..79a9576f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ keywords = ['exasol', 'bucketfs'] [tool.poetry.dependencies] python = ">=3.6.1" +requests = "^2.24.0" [tool.poetry.dev-dependencies] pytest = "^6.1.1" diff --git a/tests/TestGithubReleaseFileBucketFSUploader.py b/tests/test_github_release_file_bucketfs_uploader.py similarity index 92% rename from tests/TestGithubReleaseFileBucketFSUploader.py rename to tests/test_github_release_file_bucketfs_uploader.py index d924ef06..30cce803 100644 --- a/tests/TestGithubReleaseFileBucketFSUploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -1,15 +1,10 @@ -import os import re -import subprocess import textwrap -from pathlib import Path import pyexasol -import extension_downloading - -from extension_downloading.GithubReleaseFileBucketFSUploader import GithubReleaseFileBucketFSUploader -from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, upload_file_to_bucketfs, BucketFSCredentials +from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader +from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, BucketFSCredentials class DatabaseCredentials: From 9d3f8df67c5a54a1d2b101d6cbad1f3d7aa91f6a Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 9 Nov 2020 15:43:48 +0100 Subject: [PATCH 08/20] #1: Added __init__.py --- extension_downloading/__init__.py | 0 tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 extension_downloading/__init__.py create mode 100644 tests/__init__.py diff --git a/extension_downloading/__init__.py b/extension_downloading/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 096d21231c3dfe2163cd42e6b2ad476ae555c630 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 9 Nov 2020 15:57:51 +0100 Subject: [PATCH 09/20] #1: Fixed a typo in the integration test --- tests/test_github_release_file_bucketfs_uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 30cce803..736047b6 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -47,7 +47,7 @@ def test_uploading_github_release_to_bucketfs(): con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") con.execute(f"OPEN SCHEMA {database_config.schema};") con.execute(textwrap.dedent(f""" - CREATE OR REPLACE PYTHON SCALAR SCRIPT EXA_toolbox.bucketfs_ls(my_path VARCHAR(256)) + CREATE OR REPLACE PYTHON SCALAR SCRIPT test_schema.bucketfs_ls(my_path VARCHAR(256)) EMITS (files VARCHAR(256)) AS import subprocess From 024d718e62d67136b347e5926360fe8586a300ac Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Mon, 9 Nov 2020 16:44:25 +0100 Subject: [PATCH 10/20] #1: Fixed SELECT query --- tests/test_github_release_file_bucketfs_uploader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 736047b6..7f753f32 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -45,7 +45,6 @@ def test_uploading_github_release_to_bucketfs(): password=database_config.credentials.pwd) as con: try: con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") - con.execute(f"OPEN SCHEMA {database_config.schema};") con.execute(textwrap.dedent(f""" CREATE OR REPLACE PYTHON SCALAR SCRIPT test_schema.bucketfs_ls(my_path VARCHAR(256)) EMITS (files VARCHAR(256)) AS @@ -67,7 +66,7 @@ def run(c): except: pass / """)) - result = con.execute("SELECT bucketfs_ls('/buckets/bfsdefault/default')").fetchall() + result = con.execute("SELECT test_schema.bucketfs_ls('/buckets/bfsdefault/default')").fetchall() output = result[0][0] print(output) assert re.match( From ba1501615883adbaf138518f022ea04c255ef442 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 9 Nov 2020 17:43:21 +0100 Subject: [PATCH 11/20] Fix indention and escaping error in test_github_release_file_bucketfs_uploader.py --- ...t_github_release_file_bucketfs_uploader.py | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 7f753f32..fb3b152e 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -46,31 +46,28 @@ def test_uploading_github_release_to_bucketfs(): try: con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") con.execute(textwrap.dedent(f""" - CREATE OR REPLACE PYTHON SCALAR SCRIPT test_schema.bucketfs_ls(my_path VARCHAR(256)) - EMITS (files VARCHAR(256)) AS - import subprocess +CREATE OR REPLACE PYTHON SCALAR SCRIPT test_schema.bucketfs_ls(my_path VARCHAR(256)) +EMITS (files VARCHAR(256)) AS +import subprocess - def run(c): - try: - p = subprocess.Popen('ls -F ' + c.my_path, - stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - close_fds = True, - shell = True) - out, err = p.communicate() - for line in out.strip().split('\n'): - c.emit(line) - finally: - if p is not None: - try: p.kill() - except: pass - / - """)) +def run(c): + try: + p = subprocess.Popen('ls -F ' + c.my_path, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + close_fds = True, + shell = True) + out, err = p.communicate() + for line in out.strip().split('\\n'): + c.emit(line) + finally: + if p is not None: + try: p.kill() + except: pass +/""")) result = con.execute("SELECT test_schema.bucketfs_ls('/buckets/bfsdefault/default')").fetchall() - output = result[0][0] + output = result print(output) - assert re.match( - r"""some text""", - output) + assert ("""virtualschemas/""",) in output finally: con.execute(f"DROP SCHEMA IF EXISTS {database_config.schema} CASCADE;") From 1e9721d261a3358f53dc4ae066921b562ca4aab5 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 15:16:07 +0100 Subject: [PATCH 12/20] #1: Adjusted an integration test --- ...t_github_release_file_bucketfs_uploader.py | 41 ++----------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index fb3b152e..f25366f8 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -1,7 +1,4 @@ -import re -import textwrap - -import pyexasol +import requests from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, BucketFSCredentials @@ -37,37 +34,7 @@ def upload_file_to_bucket(bucketfs_config): def test_uploading_github_release_to_bucketfs(): database_config = DatabaseConfig(DatabaseCredentials(), "test_schema") bucketfs_config = BucketFsConfig(BucketFSCredentials()) - upload_file_to_bucket(bucketfs_config) - with pyexasol.connect( - dsn=f"{database_config.credentials.host}:{database_config.credentials.port}", - user=database_config.credentials.user, - password=database_config.credentials.pwd) as con: - try: - con.execute(f"CREATE SCHEMA IF NOT EXISTS {database_config.schema};") - con.execute(textwrap.dedent(f""" -CREATE OR REPLACE PYTHON SCALAR SCRIPT test_schema.bucketfs_ls(my_path VARCHAR(256)) -EMITS (files VARCHAR(256)) AS -import subprocess - -def run(c): - try: - p = subprocess.Popen('ls -F ' + c.my_path, - stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - close_fds = True, - shell = True) - out, err = p.communicate() - for line in out.strip().split('\\n'): - c.emit(line) - finally: - if p is not None: - try: p.kill() - except: pass -/""")) - result = con.execute("SELECT test_schema.bucketfs_ls('/buckets/bfsdefault/default')").fetchall() - output = result - print(output) - assert ("""virtualschemas/""",) in output - finally: - con.execute(f"DROP SCHEMA IF EXISTS {database_config.schema} CASCADE;") + response = requests.get("http://{}:{}/virtualschemas/".format(database_config.credentials.host, + database_config.credentials.port)) + assert "virtual-schema-dist" in response.text From 8d1a6dc35d28b630d3cd4e90de519255f91d14bb Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 15:37:40 +0100 Subject: [PATCH 13/20] #1: Adjusted an integration test --- tests/bucketfs_utils.py | 32 ------------------- ...t_github_release_file_bucketfs_uploader.py | 19 ++--------- 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/tests/bucketfs_utils.py b/tests/bucketfs_utils.py index 2b58f2ea..94bc0451 100644 --- a/tests/bucketfs_utils.py +++ b/tests/bucketfs_utils.py @@ -1,8 +1,3 @@ -from pathlib import Path - -import requests - - class BucketFSCredentials: def __init__(self, host="localhost", port=6666, user="w", pwd="write"): self.host = host @@ -18,33 +13,6 @@ def __init__(self, credentials: BucketFSCredentials, bucket="default", bucketfs_ self.bucketfs_name = bucketfs_name -def upload_file_to_bucketfs(bucket_config: BucketFsConfig, file_name: str, file_path: Path): - with file_path.open("rb") as f: - url = generate_bucketfs_url(bucket_config, file_name) - response = requests.put(url, data=f) - response.raise_for_status() - path = get_bucketfs_path(bucket_config, file_name) - return url, path - - -def upload_string_to_bucketfs(bucketfs_config: BucketFsConfig, file_name: str, string: str): - url = generate_bucketfs_url(bucketfs_config, file_name) - response = requests.put(url, data=string.encode("UTF-8")) - response.raise_for_status() - path = get_bucketfs_path(bucketfs_config, file_name) - return url, path - - -def get_bucketfs_path(bucketfs_config: BucketFsConfig, file_name: str): - archive_extensions = [".tar.gz", ".tar.bz2", ".zip", ".tar"] - for extension in archive_extensions: - if file_name.endswith(extension): - file_name = file_name[:-len(extension)] - break - path = f"/buckets/{bucketfs_config.bucketfs_name}/{bucketfs_config.bucket}/{file_name}" - return path - - def generate_bucketfs_url(bucketfs_config: BucketFsConfig, file_name: str, with_credentials: bool = True): if with_credentials: credentials = f"{bucketfs_config.credentials.user}:{bucketfs_config.credentials.pwd}@" diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index f25366f8..5b2a3392 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -4,20 +4,6 @@ from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, BucketFSCredentials -class DatabaseCredentials: - def __init__(self, host="localhost", port=8888, user="sys", pwd="exasol"): - self.host = host - self.port = port - self.user = user - self.pwd = pwd - - -class DatabaseConfig: - def __init__(self, credentials: DatabaseCredentials, schema: str): - self.credentials = credentials - self.schema = schema - - def upload_file_to_bucket(bucketfs_config): release_uploader = \ GithubReleaseFileBucketFSUploader(file_to_download_name="virtual-schema-dist", @@ -32,9 +18,8 @@ def upload_file_to_bucket(bucketfs_config): def test_uploading_github_release_to_bucketfs(): - database_config = DatabaseConfig(DatabaseCredentials(), "test_schema") bucketfs_config = BucketFsConfig(BucketFSCredentials()) upload_file_to_bucket(bucketfs_config) - response = requests.get("http://{}:{}/virtualschemas/".format(database_config.credentials.host, - database_config.credentials.port)) + response = requests.get("http://{}:{}/virtualschemas/".format(bucketfs_config.credentials.host, + bucketfs_config.credentials.port)) assert "virtual-schema-dist" in response.text From 71e89df859c115e9b35b98f63174e8333fcf837f Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 15:56:35 +0100 Subject: [PATCH 14/20] #1: Adjusted an integration test --- tests/test_github_release_file_bucketfs_uploader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 5b2a3392..b2d232a5 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -20,6 +20,7 @@ def upload_file_to_bucket(bucketfs_config): def test_uploading_github_release_to_bucketfs(): bucketfs_config = BucketFsConfig(BucketFSCredentials()) upload_file_to_bucket(bucketfs_config) - response = requests.get("http://{}:{}/virtualschemas/".format(bucketfs_config.credentials.host, - bucketfs_config.credentials.port)) + response = requests.get("http://{}:{}/".format(bucketfs_config.credentials.host, + bucketfs_config.credentials.port)) + print(response.text) assert "virtual-schema-dist" in response.text From 440de21e73411104ab9662d4724e739192562277 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 16:07:08 +0100 Subject: [PATCH 15/20] #1: Adjusted an integration test --- tests/test_github_release_file_bucketfs_uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index b2d232a5..d6225848 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -20,7 +20,7 @@ def upload_file_to_bucket(bucketfs_config): def test_uploading_github_release_to_bucketfs(): bucketfs_config = BucketFsConfig(BucketFSCredentials()) upload_file_to_bucket(bucketfs_config) - response = requests.get("http://{}:{}/".format(bucketfs_config.credentials.host, + response = requests.get("http://{}:{}/default/virtualschemas/".format(bucketfs_config.credentials.host, bucketfs_config.credentials.port)) print(response.text) assert "virtual-schema-dist" in response.text From b88f4bc3e2105e927281905b94a2a484b51a6c60 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 16:23:46 +0100 Subject: [PATCH 16/20] #1: Adjusted an integration test --- tests/test_github_release_file_bucketfs_uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index d6225848..8f51dba1 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -20,7 +20,7 @@ def upload_file_to_bucket(bucketfs_config): def test_uploading_github_release_to_bucketfs(): bucketfs_config = BucketFsConfig(BucketFSCredentials()) upload_file_to_bucket(bucketfs_config) - response = requests.get("http://{}:{}/default/virtualschemas/".format(bucketfs_config.credentials.host, + response = requests.get("http://{}:{}/default/".format(bucketfs_config.credentials.host, bucketfs_config.credentials.port)) print(response.text) assert "virtual-schema-dist" in response.text From 2b005a70bcfe206dcc4311ce29035ead44176962 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Tue, 10 Nov 2020 16:59:13 +0100 Subject: [PATCH 17/20] #1: Adjusted an integration test --- tests/bucketfs_utils.py | 25 ------------------- ...t_github_release_file_bucketfs_uploader.py | 18 +++---------- 2 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 tests/bucketfs_utils.py diff --git a/tests/bucketfs_utils.py b/tests/bucketfs_utils.py deleted file mode 100644 index 94bc0451..00000000 --- a/tests/bucketfs_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -class BucketFSCredentials: - def __init__(self, host="localhost", port=6666, user="w", pwd="write"): - self.host = host - self.port = port - self.user = user - self.pwd = pwd - - -class BucketFsConfig: - def __init__(self, credentials: BucketFSCredentials, bucket="default", bucketfs_name="bfsdefault"): - self.credentials = credentials - self.bucket = bucket - self.bucketfs_name = bucketfs_name - - -def generate_bucketfs_url(bucketfs_config: BucketFsConfig, file_name: str, with_credentials: bool = True): - if with_credentials: - credentials = f"{bucketfs_config.credentials.user}:{bucketfs_config.credentials.pwd}@" - else: - credentials = "" - url = f"http://{credentials}" \ - f"{bucketfs_config.credentials.host}:{bucketfs_config.credentials.port}/{bucketfs_config.bucket}/" - if file_name is not None: - url += f"{file_name}" - return url diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 8f51dba1..58adec4c 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -1,26 +1,16 @@ import requests from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader -from tests.bucketfs_utils import BucketFsConfig, generate_bucketfs_url, BucketFSCredentials -def upload_file_to_bucket(bucketfs_config): +def test_uploading_github_release_to_bucketfs(): + bucketfs_url = "http://localhost:6666/default/" release_uploader = \ GithubReleaseFileBucketFSUploader(file_to_download_name="virtual-schema-dist", github_user="exasol", repository_name="exasol-virtual-schema", release_name="latest", path_inside_bucket="virtualschemas/") - release_uploader.upload( - generate_bucketfs_url(bucketfs_config, file_name=None, with_credentials=False), - username=bucketfs_config.credentials.user, - password=bucketfs_config.credentials.pwd) - - -def test_uploading_github_release_to_bucketfs(): - bucketfs_config = BucketFsConfig(BucketFSCredentials()) - upload_file_to_bucket(bucketfs_config) - response = requests.get("http://{}:{}/default/".format(bucketfs_config.credentials.host, - bucketfs_config.credentials.port)) - print(response.text) + release_uploader.upload(bucketfs_url, "w", "write") + response = requests.get(bucketfs_url) assert "virtual-schema-dist" in response.text From 30f9f4ba1717769fa80e4a1f92fa6397f5272e85 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Wed, 11 Nov 2020 13:11:51 +0100 Subject: [PATCH 18/20] #1: Fixed review findings --- README.md | 52 +++++++++++++++++++ doc/changes/changes_0.1.0.md | 4 +- .../github_release_file_bucketfs_uploader.py | 17 +++--- pyproject.toml | 3 +- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5cf45ba8..09b6185e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ # BucketFS Utils Python +This project provides a python library for the Exasol BucketFS system. + +Features: + +* [Uploading a GitHub release to a bucket](#uploading-github-release-to-bucket) + +## How to Use It + +Install the package from Github via `pip`: + +``` +pip install -e git://github.com/exasol/bucketfs-utils-python.git@{ tag name }#egg=exasol-bucketfs-utils-python +``` + +## Uploading GitHub Release to Bucket + +Example: + +```python +from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader + +release_uploader = \ + GithubReleaseFileBucketFSUploader(file_to_download_name="file", + github_user="user", + repository_name="repository", + release_name="latest", + path_inside_bucket="some/path/") +release_uploader.upload("http://://", "user", "password") +``` + +### Run Time Dependencies + +| Dependency | Purpose | License | +|-------------------------------|----------------------------------|--------------------| +| [Python 3.6][python-3-6-1] | Python version 3.6.1 | PSF | +| [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 | + + +### Test Dependencies + +| Dependency | Purpose | License | +|-------------------------------|-----------------------------------|-------------------| +| [Pytest][pytest] | Testing framework | MIT | +| [Pytest Coverage][pytest-cov] | Tests coverage | MIT | + + + +[python-3-6-1]: https://docs.python.org/release/3.6.1/ +[requests]: https://pypi.org/project/requests/ + +[pytest]: https://docs.pytest.org/en/stable/ +[pytest-cov]: https://pypi.org/project/pytest-cov/ diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index d8ef7086..3046288b 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -1,6 +1,6 @@ -# BucketFS Utils Python 0.1.0, released 2020-??-?? +# BucketFS Utils Python 0.1.0, released 2020-11-?? -Code name: +Code name: Initial implementation ## Features diff --git a/extension_downloading/github_release_file_bucketfs_uploader.py b/extension_downloading/github_release_file_bucketfs_uploader.py index c6f4326b..24b654a6 100644 --- a/extension_downloading/github_release_file_bucketfs_uploader.py +++ b/extension_downloading/github_release_file_bucketfs_uploader.py @@ -1,4 +1,5 @@ import requests +from requests.auth import HTTPBasicAuth from extension_downloading.release_link_extractor import ReleaseLinkExtractor @@ -20,18 +21,14 @@ def upload(self, address, username, password): """ download_url = self.__extract_download_url() r_download = requests.get(download_url, stream=True) - upload_url = self.__build_upload_url(address, username, password) - requests.put(upload_url, data=r_download.iter_content(10 * 1024)) + upload_url = self.__build_upload_url(address) + requests.put(upload_url, data=r_download.iter_content(10 * 1024), auth=HTTPBasicAuth(username, password)) - def __build_upload_url(self, address, username, password): - connection_first_part = 'http://' - split_url = address.split(connection_first_part, 1) - upload_url = "{connection_first_part}{username}:{password}@{url}/".format( - connection_first_part=connection_first_part, username=username, password=password, url=split_url[1]) + def __build_upload_url(self, address): if self.path_inside_bucket: - upload_url += self.path_inside_bucket - upload_url += self.file_to_download_name - return upload_url + address += self.path_inside_bucket + address += self.file_to_download_name + return address def __extract_download_url(self): github_api_link = self.__build_github_api_link() diff --git a/pyproject.toml b/pyproject.toml index 79a9576f..095236e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "bucketfs-utils-python" +name = "exasol-bucketfs-utils-python" version = "0.1.0" description = "BucketFS utilities for the Python programming language" @@ -24,7 +24,6 @@ requests = "^2.24.0" [tool.poetry.dev-dependencies] pytest = "^6.1.1" pytest-cov = "^2.10.1" -pyexasol = "^0.14.1" [tool.dephell.main] from = { format = "poetry", path = "pyproject.toml" } From c44c7ff7ff5d948a0371b06765a751421f7f5eb5 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko <46891819+AnastasiiaSergienko@users.noreply.github.com> Date: Wed, 11 Nov 2020 14:01:09 +0100 Subject: [PATCH 19/20] Update .github/workflows/pytest.yaml Co-authored-by: Torsten Kilias --- .github/workflows/pytest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d576158d..a748fb5f 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6] + python-version: [3.6, 3.7, 3.8] runs-on: ubuntu-latest steps: @@ -27,4 +27,4 @@ jobs: working-directory: ../integration-test-docker-environment - name: Poetry run pytest integration tests - run: poetry run pytest tests \ No newline at end of file + run: poetry run pytest tests From a1a152ca846c76ef32e4437ef0d763ddd83ef5e2 Mon Sep 17 00:00:00 2001 From: Anastasiia Sergienko Date: Wed, 11 Nov 2020 14:05:53 +0100 Subject: [PATCH 20/20] #1: Fixed review findings --- .github/workflows/pytest.yaml | 36 ++++++++++--------- README.md | 6 ++-- .../__init__.py | 0 .../github_release_file_bucketfs_uploader.py | 2 +- .../release_link_extractor.py | 0 ...t_github_release_file_bucketfs_uploader.py | 2 +- 6 files changed, 24 insertions(+), 22 deletions(-) rename {extension_downloading => exasol_bucketfs_utils_python}/__init__.py (100%) rename {extension_downloading => exasol_bucketfs_utils_python}/github_release_file_bucketfs_uploader.py (95%) rename {extension_downloading => exasol_bucketfs_utils_python}/release_link_extractor.py (100%) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d576158d..ffb2d7a0 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -1,30 +1,32 @@ name: Run pytest tests -on: [push, pull_request] +on: [ push, pull_request ] jobs: integration_tests: strategy: fail-fast: false matrix: - python-version: [3.6] + python-version: [3.6, 3.7, 3.8] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: abatilo/actions-poetry@v2.0.0 - - name: Poetry install - run: poetry install + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: abatilo/actions-poetry@v2.0.0 + - name: Poetry install + run: poetry install + - name: Poetry build + run: poetry build - - name: Checkout test environment - run: git clone https://github.com/exasol/integration-test-docker-environment.git - working-directory: .. - - name: Spawn EXASOL environemnt - run: ./start-test-env spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666 --db-mem-size 4GB - working-directory: ../integration-test-docker-environment + - name: Checkout test environment + run: git clone https://github.com/exasol/integration-test-docker-environment.git + working-directory: .. + - name: Spawn EXASOL environemnt + run: ./start-test-env spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666 --db-mem-size 4GB + working-directory: ../integration-test-docker-environment - - name: Poetry run pytest integration tests - run: poetry run pytest tests \ No newline at end of file + - name: Poetry run pytest integration tests + run: poetry run pytest tests \ No newline at end of file diff --git a/README.md b/README.md index 09b6185e..c0af00b2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ pip install -e git://github.com/exasol/bucketfs-utils-python.git@{ tag name }#eg Example: ```python -from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader +from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader release_uploader = \ GithubReleaseFileBucketFSUploader(file_to_download_name="file", @@ -34,7 +34,7 @@ release_uploader.upload("http://://", "user", "password") | Dependency | Purpose | License | |-------------------------------|----------------------------------|--------------------| -| [Python 3.6][python-3-6-1] | Python version 3.6.1 | PSF | +| [Python 3][python] | Python version 3.6.1 and above | PSF | | [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 | @@ -47,7 +47,7 @@ release_uploader.upload("http://://", "user", "password") -[python-3-6-1]: https://docs.python.org/release/3.6.1/ +[python]: https://docs.python.org [requests]: https://pypi.org/project/requests/ [pytest]: https://docs.pytest.org/en/stable/ diff --git a/extension_downloading/__init__.py b/exasol_bucketfs_utils_python/__init__.py similarity index 100% rename from extension_downloading/__init__.py rename to exasol_bucketfs_utils_python/__init__.py diff --git a/extension_downloading/github_release_file_bucketfs_uploader.py b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py similarity index 95% rename from extension_downloading/github_release_file_bucketfs_uploader.py rename to exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py index 24b654a6..f5d94b7b 100644 --- a/extension_downloading/github_release_file_bucketfs_uploader.py +++ b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py @@ -1,7 +1,7 @@ import requests from requests.auth import HTTPBasicAuth -from extension_downloading.release_link_extractor import ReleaseLinkExtractor +from exasol_bucketfs_utils_python.release_link_extractor import ReleaseLinkExtractor class GithubReleaseFileBucketFSUploader: diff --git a/extension_downloading/release_link_extractor.py b/exasol_bucketfs_utils_python/release_link_extractor.py similarity index 100% rename from extension_downloading/release_link_extractor.py rename to exasol_bucketfs_utils_python/release_link_extractor.py diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py index 58adec4c..f4e75665 100644 --- a/tests/test_github_release_file_bucketfs_uploader.py +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -1,6 +1,6 @@ import requests -from extension_downloading.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader +from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader def test_uploading_github_release_to_bucketfs():