diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml new file mode 100644 index 00000000..0b5a9731 --- /dev/null +++ b/.github/workflows/check_setup_py.yaml @@ -0,0 +1,28 @@ +name: Check if setup.py is up to date + +on: [push, pull_request] + +jobs: + check_setup_py: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run convert + uses: dephell/dephell_action@master + with: + dephell-env: convert + - name: Show changes on working copy + run: git status --porcelain=v1 -uno + - name: Show diff on working copy + run: git diff + - name: Check if setup.py changed + run: | + [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 00000000..ffb2d7a0 --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,32 @@ +name: Run pytest tests + +on: [ push, pull_request ] + +jobs: + integration_tests: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: abatilo/actions-poetry@v2.0.0 + - name: Poetry install + run: poetry install + - name: Poetry build + run: poetry build + + - name: Checkout test environment + run: git clone https://github.com/exasol/integration-test-docker-environment.git + working-directory: .. + - name: Spawn EXASOL environemnt + run: ./start-test-env spawn-test-environment --environment-name test --database-port-forward 8888 --bucketfs-port-forward 6666 --db-mem-size 4GB + working-directory: ../integration-test-docker-environment + + - name: Poetry run pytest integration tests + run: poetry run pytest tests \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3428480c --- /dev/null +++ b/.gitignore @@ -0,0 +1,135 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Poetry +poetry.lock + +# PyCharm +.idea \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..c0af00b2 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# BucketFS Utils Python + +This project provides a python library for the Exasol BucketFS system. + +Features: + +* [Uploading a GitHub release to a bucket](#uploading-github-release-to-bucket) + +## How to Use It + +Install the package from Github via `pip`: + +``` +pip install -e git://github.com/exasol/bucketfs-utils-python.git@{ tag name }#egg=exasol-bucketfs-utils-python +``` + +## Uploading GitHub Release to Bucket + +Example: + +```python +from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader + +release_uploader = \ + GithubReleaseFileBucketFSUploader(file_to_download_name="file", + github_user="user", + repository_name="repository", + release_name="latest", + path_inside_bucket="some/path/") +release_uploader.upload("http://://", "user", "password") +``` + +### Run Time Dependencies + +| Dependency | Purpose | License | +|-------------------------------|----------------------------------|--------------------| +| [Python 3][python] | Python version 3.6.1 and above | PSF | +| [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 | + + +### Test Dependencies + +| Dependency | Purpose | License | +|-------------------------------|-----------------------------------|-------------------| +| [Pytest][pytest] | Testing framework | MIT | +| [Pytest Coverage][pytest-cov] | Tests coverage | MIT | + + + +[python]: https://docs.python.org +[requests]: https://pypi.org/project/requests/ + +[pytest]: https://docs.pytest.org/en/stable/ +[pytest-cov]: https://pypi.org/project/pytest-cov/ diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md new file mode 100644 index 00000000..387930a1 --- /dev/null +++ b/doc/changes/changelog.md @@ -0,0 +1,3 @@ +# Changelog + +* [0.1.0](changes_0.1.0.md) \ No newline at end of file diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md new file mode 100644 index 00000000..3046288b --- /dev/null +++ b/doc/changes/changes_0.1.0.md @@ -0,0 +1,7 @@ +# BucketFS Utils Python 0.1.0, released 2020-11-?? + +Code name: Initial implementation + +## Features + +#1: Added initial implementation. \ No newline at end of file diff --git a/exasol_bucketfs_utils_python/__init__.py b/exasol_bucketfs_utils_python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py new file mode 100644 index 00000000..f5d94b7b --- /dev/null +++ b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py @@ -0,0 +1,41 @@ +import requests +from requests.auth import HTTPBasicAuth + +from exasol_bucketfs_utils_python.release_link_extractor import ReleaseLinkExtractor + + +class GithubReleaseFileBucketFSUploader: + def __init__(self, file_to_download_name, github_user, repository_name, release_name, path_inside_bucket): + self.file_to_download_name = file_to_download_name + self.github_user = github_user + self.repository_name = repository_name + self.release_name = release_name + self.path_inside_bucket = path_inside_bucket + + def upload(self, address, username, password): + """ + This method uploads the GitHub release into a selected Exasol bucket. + :param address: address in the format 'http://:/' + :param username: bucket writing username + :param password: bucket writing password + """ + download_url = self.__extract_download_url() + r_download = requests.get(download_url, stream=True) + upload_url = self.__build_upload_url(address) + requests.put(upload_url, data=r_download.iter_content(10 * 1024), auth=HTTPBasicAuth(username, password)) + + def __build_upload_url(self, address): + if self.path_inside_bucket: + address += self.path_inside_bucket + address += self.file_to_download_name + return address + + def __extract_download_url(self): + github_api_link = self.__build_github_api_link() + release_link_extractor = ReleaseLinkExtractor(github_api_link) + download_url = release_link_extractor.get_link_by_release_name(self.file_to_download_name) + return download_url + + def __build_github_api_link(self): + return "https://api.github.com/repos/{github_user}/{repository_name}/releases/{release_name}".format( + github_user=self.github_user, repository_name=self.repository_name, release_name=self.release_name) diff --git a/exasol_bucketfs_utils_python/release_link_extractor.py b/exasol_bucketfs_utils_python/release_link_extractor.py new file mode 100644 index 00000000..ac2076c3 --- /dev/null +++ b/exasol_bucketfs_utils_python/release_link_extractor.py @@ -0,0 +1,31 @@ +import requests + + +class ReleaseLinkExtractor: + def __init__(self, repository_api_link): + """ + Create a new instance of ReleaseLinkExtractor class. + :param repository_api_link: Link to the GitHub API page with the latest release. + """ + self.repository_api_link = repository_api_link + + def get_link_by_release_name(self, file_to_download_name): + """ + This method extracts a link from the GitHub API page searching by a release name. + :param file_to_download_name: the name of the file + :return: a link in a string format + """ + response = requests.get(self.repository_api_link) + json_release_page = response.json() + list_of_available_releases = json_release_page["assets"] + result_link = self.__find_link(list_of_available_releases, file_to_download_name) + if result_link is not None: + return result_link + else: + raise ValueError( + 'Release with the name ' + file_to_download_name + ' was not found. Please check the name or select another release') + + def __find_link(self, list_of_available_releases, release_name): + for release in list_of_available_releases: + if release_name in release["name"]: + return release["browser_download_url"] diff --git a/githooks/install.sh b/githooks/install.sh new file mode 100755 index 00000000..1a873788 --- /dev/null +++ b/githooks/install.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +REPO_DIR=$(git rev-parse --show-toplevel) +REPO_DIR="$(readlink -f "${REPO_DIR}")" +GIT_DIR="$REPO_DIR/.git" +GIT_DIR="$(readlink -f "${GIT_DIR}")" + +if [[ ! -d "$GIT_DIR" ]]; then + if [[ -d "$REPO_DIR/../.git" ]]; then + GIT_DIR="$REPO_DIR/../.git" + GITHOOKS_PATH="$GIT_DIR/modules/bucketfs-utils-python/hooks" + else + echo "$GIT_DIR is not a git directory." >&2 + exit 1 + fi +else + GITHOOKS_PATH="$GIT_DIR/hooks" +fi + +GITHOOKS_PATH="$(readlink -f "${GITHOOKS_PATH}")" + +copy_hook() { + local SCRIPT_PATH="$SCRIPT_DIR/$1" + local GITHOOK_PATH="$GITHOOKS_PATH/$2" + local RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH") + pushd "$GITHOOKS_PATH" > /dev/null + if [ -e "$GITHOOK_PATH" ] || [ -L "$GITHOOK_PATH" ] + then + echo + echo "Going to delete old hook $GITHOOK_PATH" + rm "$GITHOOK_PATH" > /dev/null + fi + echo + echo "Link hook to script" >&2 + echo "Hook-Path: $GITHOOK_PATH" >&2 + echo "Script-path: $SCRIPT_PATH" >&2 + echo + ln -s "$RELATIVE_PATH" "$2" > /dev/null + chmod +x "$SCRIPT_PATH" > /dev/null + popd > /dev/null +} + +copy_hook pre-commit pre-commit +copy_hook pre-commit post-rewrite +copy_hook pre-push pre-push diff --git a/githooks/pre-commit b/githooks/pre-commit new file mode 100755 index 00000000..2c8000ff --- /dev/null +++ b/githooks/pre-commit @@ -0,0 +1,11 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +REPO_DIR=$(git rev-parse --show-toplevel) +GITHOOKS_PATH="$REPO_DIR/githooks" +pushd "$REPO_DIR" +bash "$GITHOOKS_PATH/prohibit_commit_to_master.sh" +bash "$GITHOOKS_PATH/update_setup_py.sh" +popd diff --git a/githooks/pre-push b/githooks/pre-push new file mode 100755 index 00000000..5ae1a28a --- /dev/null +++ b/githooks/pre-push @@ -0,0 +1,43 @@ +#!/bin/bash +protected_branches=( master ) +for i in "${protected_branches[@]}" +do + + protected_branch=$i + + policy='[Policy] Never push, force push or delete the '$protected_branch' branch! (Prevented with pre-push hook.)' + + current_branch=$(git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,') + + push_command=$(ps -ocommand= -p $PPID) + + is_destructive='force|delete|\-f' + + will_remove_protected_branch=':'$protected_branch + + do_exit(){ + echo $policy + exit 1 + } + + if [[ $push_command =~ $is_destructive ]] && [ $current_branch = $protected_branch ]; then + do_exit + fi + + if [[ $push_command =~ $is_destructive ]] && [[ $push_command =~ $protected_branch ]]; then + do_exit + fi + + if [[ $push_command =~ $will_remove_protected_branch ]]; then + do_exit + fi + + if [[ $protected_branch == $current_branch ]]; then + do_exit + fi + +done + +unset do_exit + +exit 0 diff --git a/githooks/prohibit_commit_to_master.sh b/githooks/prohibit_commit_to_master.sh new file mode 100755 index 00000000..632d10e7 --- /dev/null +++ b/githooks/prohibit_commit_to_master.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +branch="$(git rev-parse --abbrev-ref HEAD)" + +if [ "$branch" = "master" ]; then + echo "You can't commit directly to master branch" + exit 1 +fi diff --git a/githooks/update_setup_py.sh b/githooks/update_setup_py.sh new file mode 100755 index 00000000..d3ac28e2 --- /dev/null +++ b/githooks/update_setup_py.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# define colors for use in output +green='\033[0;32m' +no_color='\033[0m' +grey='\033[0;90m' + +echo -e "Update setup.py with dephell convert ${grey}(pre-commit hook)${no_color} " + +# Jump to the current project's root directory (the one containing +# .git/) +ROOT_DIR=$(git rev-parse --show-cdup) + +pushd "$ROOT_DIR" > /dev/null + +dephell venv run --env convert +git add setup.py README.rst + +popd > /dev/null diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..095236e5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "exasol-bucketfs-utils-python" +version = "0.1.0" +description = "BucketFS utilities for the Python programming language" + +license = "MIT" + +authors = [ + "Torsten Kilias ", + "Anastasiia Sergienko " +] + +readme = 'README.md' + +repository = "https://github.com/exasol/bucketfs-utils-python" +homepage = "https://github.com/exasol/bucketfs-utils-python" + +keywords = ['exasol', 'bucketfs'] + +[tool.poetry.dependencies] +python = ">=3.6.1" +requests = "^2.24.0" + +[tool.poetry.dev-dependencies] +pytest = "^6.1.1" +pytest-cov = "^2.10.1" + +[tool.dephell.main] +from = { format = "poetry", path = "pyproject.toml" } +to = { format = "setuppy", path = "setup.py" } + +[tool.dephell.convert] +from = { format = "poetry", path = "pyproject.toml" } +command = "dephell deps convert" \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_github_release_file_bucketfs_uploader.py b/tests/test_github_release_file_bucketfs_uploader.py new file mode 100644 index 00000000..f4e75665 --- /dev/null +++ b/tests/test_github_release_file_bucketfs_uploader.py @@ -0,0 +1,16 @@ +import requests + +from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader + + +def test_uploading_github_release_to_bucketfs(): + bucketfs_url = "http://localhost:6666/default/" + release_uploader = \ + GithubReleaseFileBucketFSUploader(file_to_download_name="virtual-schema-dist", + github_user="exasol", + repository_name="exasol-virtual-schema", + release_name="latest", + path_inside_bucket="virtualschemas/") + release_uploader.upload(bucketfs_url, "w", "write") + response = requests.get(bucketfs_url) + assert "virtual-schema-dist" in response.text