diff --git a/.github/workflows/check_github_pages_docs_for_feature_branch.yaml b/.github/workflows/check_github_pages_docs_for_feature_branch.yaml new file mode 100644 index 00000000..1beb4a56 --- /dev/null +++ b/.github/workflows/check_github_pages_docs_for_feature_branch.yaml @@ -0,0 +1,38 @@ +name: Check documentation build and deployment for feature branch + +on: + push: + branches-ignore: + - main + - github-pages/* + +jobs: + create_github_pages_docs: + strategy: + fail-fast: false + matrix: + python-version: [3.6] + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: 1.1.4 + - name: Poetry install + run: poetry install + - name: Test documentation deployment for current feature branch + run: | + git config --local user.email "opensource@exasol.com" + git config --local user.name "GitHub Action" + git fetch + SOURCE_BRANCH="$(git rev-parse --abbrev-ref HEAD)" + TARGET_BRANCH="$(doc/get_target_branch_name.sh "$SOURCE_BRANCH")" + poetry run poe push-html-doc-to-github-pages-current || echo + git push origin --delete "$TARGET_BRANCH" diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml index eec2d710..1a930674 100644 --- a/.github/workflows/check_setup_py.yaml +++ b/.github/workflows/check_setup_py.yaml @@ -1,6 +1,9 @@ name: Check if setup.py is up to date -on: [push, pull_request] +on: + push: + branches-ignore: + - github-pages/* jobs: check_setup_py: @@ -16,6 +19,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: 1.1.4 + - name: Poetry install + run: poetry install - name: Run packaging update run: bash githooks/update_setup_py.sh - name: Show changes on working copy diff --git a/.github/workflows/deploy_github_pages_docs_for_main.yaml b/.github/workflows/deploy_github_pages_docs_for_main.yaml new file mode 100644 index 00000000..a98769a9 --- /dev/null +++ b/.github/workflows/deploy_github_pages_docs_for_main.yaml @@ -0,0 +1,33 @@ +name: Build and push documentation for main branch + +on: + push: + branches: + - main +jobs: + create_github_pages_docs: + strategy: + fail-fast: false + matrix: + python-version: [3.6] + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: 1.1.4 + - name: Poetry install + run: poetry install + - name: Deploy documentation to github-pages-main branch + run: | + git config --local user.email "opensource@exasol.com" + git config --local user.name "GitHub Action" + git fetch + poetry run poe push-html-doc-to-github-pages-main diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index b26fc374..3dd71c49 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -1,6 +1,9 @@ name: Run pytest tests -on: [ push, pull_request ] +on: + push: + branches-ignore: + - github-pages/* jobs: integration_tests: @@ -16,6 +19,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: 1.1.4 - name: Poetry install run: poetry install - name: Poetry build diff --git a/.gitignore b/.gitignore index 3428480c..284db3de 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,7 @@ dmypy.json poetry.lock # PyCharm -.idea \ No newline at end of file +.idea + +# Sphinx +doc/_build diff --git a/README.md b/README.md deleted file mode 100644 index c0af00b2..00000000 --- a/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# BucketFS Utils Python - -This project provides a python library for the Exasol BucketFS system. - -Features: - -* [Uploading a GitHub release to a bucket](#uploading-github-release-to-bucket) - -## How to Use It - -Install the package from Github via `pip`: - -``` -pip install -e git://github.com/exasol/bucketfs-utils-python.git@{ tag name }#egg=exasol-bucketfs-utils-python -``` - -## Uploading GitHub Release to Bucket - -Example: - -```python -from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader - -release_uploader = \ - GithubReleaseFileBucketFSUploader(file_to_download_name="file", - github_user="user", - repository_name="repository", - release_name="latest", - path_inside_bucket="some/path/") -release_uploader.upload("http://://", "user", "password") -``` - -### Run Time Dependencies - -| Dependency | Purpose | License | -|-------------------------------|----------------------------------|--------------------| -| [Python 3][python] | Python version 3.6.1 and above | PSF | -| [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 | - - -### Test Dependencies - -| Dependency | Purpose | License | -|-------------------------------|-----------------------------------|-------------------| -| [Pytest][pytest] | Testing framework | MIT | -| [Pytest Coverage][pytest-cov] | Tests coverage | MIT | - - - -[python]: https://docs.python.org -[requests]: https://pypi.org/project/requests/ - -[pytest]: https://docs.pytest.org/en/stable/ -[pytest-cov]: https://pypi.org/project/pytest-cov/ diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..d2cb205f --- /dev/null +++ b/README.rst @@ -0,0 +1,43 @@ +##################### +BucketFS Utils Python +##################### + +******** +Overview +******** + +This project provides a python library for accessing the Exasol BucketFS system. +It provides functions to upload and download files to and from the BucketFS. + +In a Nutshell +============= + +Prerequisites +------------- + +- Python 3.6+ + +Installation +------------- + +Install the package from Github via `pip`:: + + pip install -e git://github.com/exasol/bucketfs-utils-python.git@{tag name}#egg=exasol-bucketfs-utils-python + +Documentation +------------- + +`Documentation for the latest release text `_ is hosted on the Github Pages of this project. + +Features +======== + +* Download or upload files from/to the Exasol BucketFS +* Supported sources and targets for the uploads and downloads: + + * Files on the local Filesystem + * Python file objects + * Python Strings + * Python objects ((De-)Serialization with [Joblib](https://joblib.readthedocs.io/en/latest/persistence.html)) + +* Loading an artefact from a public Github Release into the BucketFS diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..2fc37c4c --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,30 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAPIDOC ?= sphinx-apidoc +SOURCEDIR = . +BUILDDIR = _build +MODULES_PATH = ../exasol_bucketfs_utils_python +APIDOC_OUTPUT = api + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +api-doc: + @$(SPHINXAPIDOC) -T -d 1 --separate -o "$(APIDOC_OUTPUT)" "$(MODULES_PATH)" + +clean-build: + [ ! -e "$(BUILDDIR)" ] || rm -r "$(BUILDDIR)" + [ ! -e "$(APIDOC_OUTPUT)" ] || rm -r "$(APIDOC_OUTPUT)" + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile api-doc + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/_static/.gitkeep b/doc/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/doc/_templates/.gitkeep b/doc/_templates/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/doc/api/exasol_bucketfs_utils_python.bucket_config.rst b/doc/api/exasol_bucketfs_utils_python.bucket_config.rst new file mode 100644 index 00000000..11e0aead --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.bucket_config.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.bucket\_config module +===================================================== + +.. automodule:: exasol_bucketfs_utils_python.bucket_config + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.bucketfs_config.rst b/doc/api/exasol_bucketfs_utils_python.bucketfs_config.rst new file mode 100644 index 00000000..863f3865 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.bucketfs_config.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.bucketfs\_config module +======================================================= + +.. automodule:: exasol_bucketfs_utils_python.bucketfs_config + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.bucketfs_connection_config.rst b/doc/api/exasol_bucketfs_utils_python.bucketfs_connection_config.rst new file mode 100644 index 00000000..bb01128d --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.bucketfs_connection_config.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.bucketfs\_connection\_config module +=================================================================== + +.. automodule:: exasol_bucketfs_utils_python.bucketfs_connection_config + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.bucketfs_utils.rst b/doc/api/exasol_bucketfs_utils_python.bucketfs_utils.rst new file mode 100644 index 00000000..23ab96f7 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.bucketfs_utils.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.bucketfs\_utils module +====================================================== + +.. automodule:: exasol_bucketfs_utils_python.bucketfs_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.download.rst b/doc/api/exasol_bucketfs_utils_python.download.rst new file mode 100644 index 00000000..40e6f557 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.download.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.download module +=============================================== + +.. automodule:: exasol_bucketfs_utils_python.download + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader.rst b/doc/api/exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader.rst new file mode 100644 index 00000000..73ad7c62 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.github\_release\_file\_bucketfs\_uploader module +================================================================================ + +.. automodule:: exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.release_link_extractor.rst b/doc/api/exasol_bucketfs_utils_python.release_link_extractor.rst new file mode 100644 index 00000000..68771d41 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.release_link_extractor.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.release\_link\_extractor module +=============================================================== + +.. automodule:: exasol_bucketfs_utils_python.release_link_extractor + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.rst b/doc/api/exasol_bucketfs_utils_python.rst new file mode 100644 index 00000000..10f5c468 --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.rst @@ -0,0 +1,25 @@ +exasol\_bucketfs\_utils\_python package +======================================= + +Submodules +---------- + +.. toctree:: + :maxdepth: 1 + + exasol_bucketfs_utils_python.bucket_config + exasol_bucketfs_utils_python.bucketfs_config + exasol_bucketfs_utils_python.bucketfs_connection_config + exasol_bucketfs_utils_python.bucketfs_utils + exasol_bucketfs_utils_python.download + exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader + exasol_bucketfs_utils_python.release_link_extractor + exasol_bucketfs_utils_python.upload + +Module contents +--------------- + +.. automodule:: exasol_bucketfs_utils_python + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/exasol_bucketfs_utils_python.upload.rst b/doc/api/exasol_bucketfs_utils_python.upload.rst new file mode 100644 index 00000000..6fd3755c --- /dev/null +++ b/doc/api/exasol_bucketfs_utils_python.upload.rst @@ -0,0 +1,7 @@ +exasol\_bucketfs\_utils\_python.upload module +============================================= + +.. automodule:: exasol_bucketfs_utils_python.upload + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md deleted file mode 100644 index 387930a1..00000000 --- a/doc/changes/changelog.md +++ /dev/null @@ -1,3 +0,0 @@ -# Changelog - -* [0.1.0](changes_0.1.0.md) \ No newline at end of file diff --git a/doc/changes/changelog.rst b/doc/changes/changelog.rst new file mode 100644 index 00000000..0519d9ca --- /dev/null +++ b/doc/changes/changelog.rst @@ -0,0 +1,8 @@ +***************** +Changelog +***************** + +.. toctree:: + :maxdepth: 1 + + changes_0.1.0.md diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 3046288b..e68a3fd1 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -1,7 +1,15 @@ -# BucketFS Utils Python 0.1.0, released 2020-11-?? - +# BucketFs Utily Python 0.1.0, released XYZ Code name: Initial implementation -## Features +## Summary + +### Features + + - #1: Added initial implementation. + - #6: Download file into fileobj, string or file + - #7: Upload file, string or fileboj + - #29: Add sphinx documentation + +### Refactorings -#1: Added initial implementation. \ No newline at end of file + - #15: Remove DepHell dependency, because it is not maintained anymore diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 00000000..1f794175 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,65 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'Exasol BucketFS Utils Python' +copyright = '2021, Exasol' +author = 'Exasol' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.intersphinx', + 'sphinx.ext.autosummary', + 'myst_parser', + 'sphinx.ext.autosectionlabel', +] +# Make sure the target is unique +autosectionlabel_prefix_document = True +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/doc/dependencies.md b/doc/dependencies.md new file mode 100644 index 00000000..4421031f --- /dev/null +++ b/doc/dependencies.md @@ -0,0 +1,24 @@ +# Dependencies + +## Run Time Dependencies + +| Dependency | Purpose | License | +|-------------------------------|----------------------------------|--------------------| +| [Python 3][python] | Python version 3.6.1 and above | PSF | +| [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 | + + +## Test Dependencies + +| Dependency | Purpose | License | +|-------------------------------|-----------------------------------|-------------------| +| [Pytest][pytest] | Testing framework | MIT | +| [Pytest Coverage][pytest-cov] | Tests coverage | MIT | + + + +[python]: https://docs.python.org +[requests]: https://pypi.org/project/requests/ + +[pytest]: https://docs.pytest.org/en/stable/ +[pytest-cov]: https://pypi.org/project/pytest-cov/ diff --git a/doc/deploy-to-github-pages-current.sh b/doc/deploy-to-github-pages-current.sh new file mode 100755 index 00000000..605df1b2 --- /dev/null +++ b/doc/deploy-to-github-pages-current.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +PUSH_ENABLED="$1" + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +SOURCE_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +if [ "$SOURCE_BRANCH" != "main" ]; then + if [ -n "$SOURCE_BRANCH" ]; then + TARGET_BRANCH="$("$SCRIPT_DIR/get_target_branch_name.sh" "$SOURCE_BRANCH")" + bash "$SCRIPT_DIR/deploy-to-github-pages.sh" "$TARGET_BRANCH" origin "$PUSH_ENABLED" "$SOURCE_BRANCH" + else + echo "ERROR: Could not acquire your current checked out branch. Please check the state of your " + fi +else + echo "ERROR: Source branch is 'main'." +fi diff --git a/doc/deploy-to-github-pages-main.sh b/doc/deploy-to-github-pages-main.sh new file mode 100755 index 00000000..347a3fc6 --- /dev/null +++ b/doc/deploy-to-github-pages-main.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +PUSH_ENABLED="$1" + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +SOURCE_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +if [ "$SOURCE_BRANCH" == "main" ]; then + TARGET_BRANCH="$("$SCRIPT_DIR/get_target_branch_name.sh" "main")" + bash "$SCRIPT_DIR/deploy-to-github-pages.sh" "$TARGET_BRANCH" origin "$PUSH_ENABLED" "$SOURCE_BRANCH" +else + echo "Abort. Source branch is not 'main', you are at branch '$SOURCE_BRANCH'" +fi diff --git a/doc/deploy-to-github-pages.sh b/doc/deploy-to-github-pages.sh new file mode 100755 index 00000000..f7ccc866 --- /dev/null +++ b/doc/deploy-to-github-pages.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +PUSH_ORIGIN="$2" +PUSH_ENABLED="$3" +SOURCE_BRANCH="$4" + +set -euo pipefail + +detect_or_verify_source_branch() { + CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" + if [ -z "$SOURCE_BRANCH" ]; then + if [ -z "$CURRENT_BRANCH" ]; then + echo "Abort. Could not detect current branch and no source branch given." + exit 1 + fi + SOURCE_BRANCH="$CURRENT_BRANCH" + fi + if [ "$SOURCE_BRANCH" != "$CURRENT_BRANCH" ]; then + echo "Abort. Specified Source Branch doesn't correspond to the currently checked out branch $CURRENT_BRANCH." + exit 1 + fi +} + +cleanup_trap() { + if [ -e "$WORKTREE" ]; then + echo "Cleanup git worktree $WORKTREE" + git worktree remove --force "$WORKTREE" || echo "Removing worktree $WORKTREE failed" + fi + echo "Cleanup temporary directory $TMP" + rm -rf "$TMP" +} + +checkout_target_branch_as_worktree() { + TARGET_BRANCH_EXISTS="$(git show-ref "refs/heads/$TARGET_BRANCH" || echo)" + if [ -n "$TARGET_BRANCH_EXISTS" ]; then + echo "Checkout existing branch $TARGET_BRANCH" + git worktree add "$WORKTREE" "$TARGET_BRANCH" + else + echo "Checkout new branch $TARGET_BRANCH" + # We create the branch with git branch and not with the -b option of git worktree, + # because the -b option seems to doesn't work in all cases + git branch "$TARGET_BRANCH" + # We need to create the worktree directly with the TARGET_BRANCH, + # because every other branch could be already checked out + git worktree add "$WORKTREE" "$TARGET_BRANCH" + pushd "$WORKTREE" + # We need to set the TARGET_BRANCH to the default branch + # The default branch from github for pages is gh-pages, but you can change that. + # Not using the default branch actually has benefits, because the branch gh-pages enforces some things. + # We use github-pages/main with separate history for Github Pages, + # because automated commits to the main branch can cause problems and + # we don't want to mix the generated documentation with sources. + # Furthermore, Github Pages expects a certain directory structure in the repository + # which we only can provide with a separate history. + GH_PAGES_MAIN_BRANCH=origin/github-pages/main + GH_PAGES_MAIN_BRANCH_EXISTS="$(git show-ref "refs/heads/$GH_PAGES_MAIN_BRANCH" || echo)" + if [ -n "$GH_PAGES_MAIN_BRANCH_EXISTS" ] + then + git reset --hard "$GH_PAGES_MAIN_BRANCH" + else + echo "Creating a new empty root commit for the Github Pages." + + git checkout --orphan "$GH_PAGES_MAIN_BRANCH" + git reset --hard + git commit --no-verify --allow-empty -m 'Initial empty commit for Github Pages' + fi + popd + fi +} + +build_and_copy_documentation() { + echo "Build with sphinx" + sphinx-build -M html "$SCRIPT_DIR" "$BUILD_DIR" -W + + echo "Generated HTML Output" + HTML_OUTPUT_DIR="$BUILD_DIR/html/" + ls -la "$HTML_OUTPUT_DIR" + + OUTPUT_DIR="${WORKTREE:?}/${SOURCE_BRANCH:?}" + if [ -e "${OUTPUT_DIR}" ]; then + echo "Removing existing output directory $OUTPUT_DIR" + rm -rf "${OUTPUT_DIR}" + fi + echo "Creating output directory $OUTPUT_DIR" + mkdir -p "$OUTPUT_DIR" + echo "Copying HTML output $HTML_OUTPUT_DIR to the output directory $OUTPUT_DIR" + find "$HTML_OUTPUT_DIR" -mindepth 1 -maxdepth 1 -exec mv -t "$OUTPUT_DIR" -- {} + + echo "Content of output directory $OUTPUT_DIR" + touch "$WORKTREE/.nojekyll" + ls -la "$OUTPUT_DIR" +} + +git_commit_and_push() { + pushd "$WORKTREE" + echo "Git commit" + echo "BRANCH=$SOURCE_BRANCH" >.source + echo "COMMIT_ID=$CURRENT_COMMIT_ID" >>.source + git add . + git diff-index --quiet HEAD || git commit --no-verify -m "Update documentation from source branch '$SOURCE_BRANCH' with commit id '$CURRENT_COMMIT_ID'" + if [ -n "$PUSH_ORIGIN" ] && [ "$PUSH_ENABLED" == "push" ]; then + echo "Git push $PUSH_ORIGIN $TARGET_BRANCH" + git push "$PUSH_ORIGIN" "$TARGET_BRANCH" + fi + popd +} + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +TMP="$(mktemp -d)" +WORKTREE="$TMP/worktree" +BUILD_DIR="$TMP/build" +trap 'cleanup_trap' EXIT + +TARGET_BRANCH="$1" +CURRENT_COMMIT_ID="$(git rev-parse HEAD)" + +detect_or_verify_source_branch +checkout_target_branch_as_worktree +build_and_copy_documentation +git_commit_and_push diff --git a/doc/developer_guide/building_documentation.rst b/doc/developer_guide/building_documentation.rst new file mode 100644 index 00000000..564a305b --- /dev/null +++ b/doc/developer_guide/building_documentation.rst @@ -0,0 +1,60 @@ +************************** +Building the Documentation +************************** + +We are using Sphinx for generating the documentation of this project, +because it is the default documentation tool for Python projects. +Sphinx supports API documentation generation for Python and with plugins also for other languages. +Furthermore, it supports reStructuredText with proper cross-document references. +We using the MyST-Parser to also integrate markdown files into the documentation. + +###################################################### +Building the Documentation interactivily during coding +###################################################### + +We defined several commands in the project.toml in poethepoet +which allow you to build and view the documentation during coding:: + + poetry run poe build-html-doc # Builds the documentation + poetry run poe open-html-doc # Opens the currently build documentation in the browser + poetry run poe build-and-open-html-doc # Builds and opens the documentation + +All three build commands use the generated documentation located in /doc/_build/ +which is excluded in gitignore. If you want to build the documentation for other formats than HTML, +you find a Makefile in /doc which allows you to run the sphinx build with other goals. + +#################################### +Building the Documentation in the CI +#################################### + +Building the documentation in the CI is a bit different to the steps you can use during coding, +because it also contains the preparations for publishing. At the moment, we publish +the documentation on Github Pages. + +To publish it there, we need to build the HTML from the documentation source and commit it. +However, Github Pages expects a specific directory structure to find the HTML. +Our usual directory structure doesn't fit these requirements, so we decided to create +a new Git root commit and initially set github-pages/main branch to this commit. +We then add new commits to this branch to update existing or add new versions of the documentation. + +This has the additional benefit, that we don't have automatic commits to the source branch. +For each branch or tag for which we build the documentation in the CI +we add a directory to the root directory of the github-pages/main branch. + +With each merge into the main branch the CI updates the documentation for the main branch in github-pages/main. +For feature branches the CI checks this deployment process by creating a branch github-pages/. +but it removes the branch directly after pushing it. However, you can run this also locally for testing purposes or +checking the branch with Github Pages in a fork of the main repostory. +The scripts which are responsible for the deployment are:: + + deploy-to-github-pages-current # creates or updates github-pages/ + deploy-to-github-pages-main.sh # only applicable for the main branch and creates or updates github-pages/main + + +We also provide a few shortcuts defined in our project.toml for poethepoet:: + + poetry run poe commit-html-doc-to-github-pages-main # creates or updates github-pages/main locally + poetry run poe push-html-doc-to-github-pages-main # creates or updates github-pages/main and pushes it to origin + poetry run poe commit-html-doc-to-github-pages-current # creates or updates github-pages/ locally + poetry run poe push-html-doc-to-github-pages-current # creates or updates github-pages/ and pushes it to origin + diff --git a/doc/developer_guide/developer_guide.rst b/doc/developer_guide/developer_guide.rst new file mode 100644 index 00000000..e91bad0c --- /dev/null +++ b/doc/developer_guide/developer_guide.rst @@ -0,0 +1,10 @@ +*************** +Developer Guide +*************** + +In this developer guide we explain how you can build this project. + +.. toctree:: + :maxdepth: 1 + + building_documentation diff --git a/doc/get_target_branch_name.sh b/doc/get_target_branch_name.sh new file mode 100755 index 00000000..eff883c2 --- /dev/null +++ b/doc/get_target_branch_name.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -euo pipefail + +BRANCH=$1 +echo "github-pages/$BRANCH" \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 00000000..5a112c1d --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,29 @@ +.. include:: ../README.rst + +***************** +Table of Contents +***************** + +.. toctree:: + :maxdepth: 1 + + user_guide/user_guide + developer_guide/developer_guide + dependencies + changes/changelog + +************* +API Reference +************* + +.. autosummary:: + :toctree: api + + exasol_bucketfs_utils_python + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/user_guide/upload_github_release_to_bucket.py b/doc/user_guide/upload_github_release_to_bucket.py new file mode 100644 index 00000000..362c491e --- /dev/null +++ b/doc/user_guide/upload_github_release_to_bucket.py @@ -0,0 +1,9 @@ +from exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader + +release_uploader = \ + GithubReleaseFileBucketFSUploader(file_to_download_name="file", + github_user="user", + repository_name="repository", + release_name="latest", + path_inside_bucket="some/path/") +release_uploader.upload("http://://", "user", "password") diff --git a/doc/user_guide/upload_github_release_to_bucket.rst b/doc/user_guide/upload_github_release_to_bucket.rst new file mode 100644 index 00000000..4c6fde0a --- /dev/null +++ b/doc/user_guide/upload_github_release_to_bucket.rst @@ -0,0 +1,11 @@ +******************************************************************** +Uploading an artefact from a public Github Release into the BucketFS +******************************************************************** + +With this function you can upload an artefact from a public Github Release into the BucketFS. +This can be useful, if you want to upload a released artefact from one of `Exasol's open source repositories `_. + +Example: + +.. literalinclude:: upload_github_release_to_bucket.py + :language: python3 \ No newline at end of file diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst new file mode 100644 index 00000000..93dd825a --- /dev/null +++ b/doc/user_guide/user_guide.rst @@ -0,0 +1,11 @@ +********** +User Guide +********** + +This user guide provides you with usage examples for this repository. +For a detailed explanation of the API, please refer to our :doc:`API Documentation ` + +.. toctree:: + :maxdepth: 1 + + upload_github_release_to_bucket diff --git a/exasol_bucketfs_utils_python/bucketfs_utils.py b/exasol_bucketfs_utils_python/bucketfs_utils.py index 81e73a1b..b3910245 100644 --- a/exasol_bucketfs_utils_python/bucketfs_utils.py +++ b/exasol_bucketfs_utils_python/bucketfs_utils.py @@ -36,6 +36,7 @@ def _make_path_relative(path_in_bucket: Union[None, str, PurePosixPath]) -> Pure def generate_bucketfs_udf_path(bucketfs_config: BucketFSConfig) -> PurePosixPath: """ This function generates the path where UDFs can access the content of a BucketFS in their file system + :param bucketfs_config: Config of the BucketFS, the BucketFSConnectionConfig in the BucketFSConfig can be None :return: Path of the given BucketFS in the file system of the UDFs """ @@ -49,6 +50,7 @@ def generate_bucket_udf_path(bucket_config: BucketConfig, """ This function generates the path where UDFs can access the content of a bucket or the given path in a bucket in their file system + :param bucket_config: Config of the Bucket, the BucketFSConnectionConfig in the BucketFSConfig can be None :param path_in_bucket: If not None, path_in_bucket gets concatenated to the path of the bucket :return: Path of the bucket or the file in the Bucket in the file system of UDFs @@ -71,6 +73,7 @@ def generate_bucketfs_http_url(bucketfs_config: BucketFSConfig, """ This function generates an HTTP[s] url for the given BucketFSConfig with or without basic authentication (a template: http[s]://user:password@host:port) + :param bucketfs_config: A BucketFSConfig with a non None BucketFSConnectionConfig :param with_credentials: If True, this function generates a url with basic authentication, default False :return: HTTP[S] URL of the BucketFS @@ -100,6 +103,7 @@ def generate_bucket_http_url(bucket_config: BucketConfig, path_in_bucket: Union[ """ This function generates an HTTP[s] url for the given bucket or the path in the bucket with or without basic authentication (a template: http[s]://user:password@host:port) + :param bucket_config: Config of the Bucket, the BucketFSConnectionConfig in the BucketFSConfig must be not None :param path_in_bucket: If not None, path_in_bucket gets concatenated to the path of the bucket :param with_credentials: If True, this function generates a url with basic authentication, default False diff --git a/exasol_bucketfs_utils_python/download.py b/exasol_bucketfs_utils_python/download.py index 98de4b2f..e260fe15 100644 --- a/exasol_bucketfs_utils_python/download.py +++ b/exasol_bucketfs_utils_python/download.py @@ -13,6 +13,7 @@ def download_from_bucketfs_to_file(bucket_config: BucketConfig, bucket_file_path: str, local_file_path: Path): """ Download a file from the specified path in the bucket in the BucketFs and save as a local file + :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from :param local_file_path: File path to the local file to store the downloaded data @@ -26,6 +27,7 @@ def download_from_bucketfs_to_fileobj(bucket_config: BucketConfig, bucket_file_p """ Download a file from the specified path in the bucket in the BucketFs into a given `file object `_ + :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from :param fileobj: File object where the data of the file in the BucketFS is downloaded to @@ -44,6 +46,7 @@ def download_from_bucketfs_to_fileobj(bucket_config: BucketConfig, bucket_file_p def download_from_bucketfs_to_string(bucket_config: BucketConfig, bucket_file_path: str) -> str: """ Download a file from the specified path in the bucket in the BucketFs into a string + :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from :return: The content of the file in the BucketFS as string @@ -61,6 +64,7 @@ def download_object_from_bucketfs_via_joblib(bucket_config: BucketConfig, bucket """ Download a file from the specified path in the bucket in the BucketFs and deserialize it via `joblib.load `_ + :param bucket_config: BucketConfig for the bucket to download from :param bucket_file_path: Path in the bucket to download the file from :return: The deserialized object which was downloaded from the BucketFS diff --git a/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py index f5d94b7b..530156bd 100644 --- a/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py +++ b/exasol_bucketfs_utils_python/github_release_file_bucketfs_uploader.py @@ -15,9 +15,11 @@ def __init__(self, file_to_download_name, github_user, repository_name, release_ def upload(self, address, username, password): """ This method uploads the GitHub release into a selected Exasol bucket. - :param address: address in the format 'http://:/' + + :param address: address in the format :samp:`http://{host}:{port}/{bucket name}` :param username: bucket writing username :param password: bucket writing password + :return: none """ download_url = self.__extract_download_url() r_download = requests.get(download_url, stream=True) diff --git a/exasol_bucketfs_utils_python/upload.py b/exasol_bucketfs_utils_python/upload.py index fcf7a504..e8ca518a 100644 --- a/exasol_bucketfs_utils_python/upload.py +++ b/exasol_bucketfs_utils_python/upload.py @@ -15,6 +15,7 @@ def upload_file_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str, -> Tuple[ParseResult, PurePosixPath]: """ This function uploads a file to the specified path in a bucket of the BucketFS. + :param bucket_config: BucketConfig for the destination bucket :param bucket_file_path: Path in the bucket to upload the file to :param local_file_path: File path to the local file @@ -29,6 +30,7 @@ def upload_fileobj_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: st """ This function uploads a `file object `_ to the specified path in a bucket of the BucketFS. + :param bucket_config: BucketConfig for the destination bucket :param bucket_file_path: Path in the bucket to upload the file to :param fileobj: File object which should be uploaded @@ -48,6 +50,7 @@ def upload_string_to_bucketfs(bucket_config: BucketConfig, bucket_file_path: str -> Tuple[ParseResult, PurePosixPath]: """ This function uploads a string to the specified path in a bucket of the BucketFS. + :param bucket_config: BucketConfig for the destination bucket :param bucket_file_path: Path in the bucket to upload the file to :param string: String which should be uploaded @@ -71,6 +74,7 @@ def upload_object_to_bucketfs_via_joblib(object: Any, This function serializes a python object with `joblib.dump `_ and uploads it to the specified path in a bucket of the BucketFS. + :param object: Object which gets serialized and uploaded via joblib.dump :param bucket_config: BucketConfig for the destination bucket :param bucket_file_path: Path in the bucket to upload the file to diff --git a/pyproject.toml b/pyproject.toml index 382e9766..57a6f910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ "Anastasiia Sergienko " ] -readme = 'README.md' +readme = 'README.rst' repository = "https://github.com/exasol/bucketfs-utils-python" homepage = "https://github.com/exasol/bucketfs-utils-python" @@ -18,7 +18,7 @@ homepage = "https://github.com/exasol/bucketfs-utils-python" keywords = ['exasol', 'bucketfs'] [tool.poetry.dependencies] -python = ">=3.6.1" +python = ">=3.6.1,<4.0" requests = "^2.24.0" joblib="^1.0.1" typeguard = "^2.11.1" @@ -26,3 +26,23 @@ typeguard = "^2.11.1" [tool.poetry.dev-dependencies] pytest = "^6.1.1" pytest-cov = "^2.10.1" +Sphinx = "^3.5.3" +poethepoet = "^0.10.0" +myst-parser = "^0.14.0" + +[tool.poe.tasks] +build-html-doc = {shell = """ + cd "$(git rev-parse --show-toplevel)/doc"; + make clean-build; make html + """ } +open-html-doc = { shell = """ + cd "$(git rev-parse --show-toplevel)/doc"; + xdg-open _build/html/index.html""" } +build-and-open-html-doc = [ "build-html-doc", "open-html-doc" ] + +commit-html-doc-to-github-pages-main = { shell = "cd $(git rev-parse --show-toplevel)/doc; bash deploy-to-github-pages-main.sh commit" } +push-html-doc-to-github-pages-main = { shell = "cd $(git rev-parse --show-toplevel)/doc; bash deploy-to-github-pages-main.sh push" } +commit-html-doc-to-github-pages-current = { shell = "cd $(git rev-parse --show-toplevel)/doc; bash deploy-to-github-pages-current.sh commit" } +push-html-doc-to-github-pages-current = { shell = "cd $(git rev-parse --show-toplevel)/doc; bash deploy-to-github-pages-current.sh push" } + +test = "pytest tests" diff --git a/setup.py b/setup.py index cbd1a2c9..1fa02345 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ 'name': 'exasol-bucketfs-utils-python', 'version': '0.1.0', 'description': 'BucketFS utilities for the Python programming language', - 'long_description': '# BucketFS Utils Python\n\nThis project provides a python library for the Exasol BucketFS system.\n\nFeatures:\n\n* [Uploading a GitHub release to a bucket](#uploading-github-release-to-bucket)\n\n## How to Use It\n\nInstall the package from Github via `pip`:\n \n```\npip install -e git://github.com/exasol/bucketfs-utils-python.git@{ tag name }#egg=exasol-bucketfs-utils-python\n```\n\n## Uploading GitHub Release to Bucket\n\nExample:\n\n```python\nfrom exasol_bucketfs_utils_python.github_release_file_bucketfs_uploader import GithubReleaseFileBucketFSUploader\n\nrelease_uploader = \\\n GithubReleaseFileBucketFSUploader(file_to_download_name="file",\n github_user="user",\n repository_name="repository",\n release_name="latest",\n path_inside_bucket="some/path/")\nrelease_uploader.upload("http://://", "user", "password")\n```\n\n### Run Time Dependencies\n\n| Dependency | Purpose | License |\n|-------------------------------|----------------------------------|--------------------|\n| [Python 3][python] | Python version 3.6.1 and above | PSF |\n| [Requests][requests] | Allows to send HTTP/1.1 requests | Apache License 2.0 |\n\n\n### Test Dependencies\n\n| Dependency | Purpose | License |\n|-------------------------------|-----------------------------------|-------------------|\n| [Pytest][pytest] | Testing framework | MIT |\n| [Pytest Coverage][pytest-cov] | Tests coverage | MIT |\n\n\n\n[python]: https://docs.python.org\n[requests]: https://pypi.org/project/requests/\n\n[pytest]: https://docs.pytest.org/en/stable/\n[pytest-cov]: https://pypi.org/project/pytest-cov/\n', + 'long_description': '#####################\nBucketFS Utils Python\n#####################\n\n********\nOverview\n********\n\nThis project provides a python library for accessing the Exasol BucketFS system.\nIt provides functions to upload and download files to and from the BucketFS.\n\nIn a Nutshell\n=============\n\nPrerequisites\n-------------\n\n- Python 3.6+\n\nInstallation\n-------------\n\nInstall the package from Github via `pip`::\n\n pip install -e git://github.com/exasol/bucketfs-utils-python.git@{tag name}#egg=exasol-bucketfs-utils-python\n\nDocumentation\n-------------\n\n`Documentation for the latest release text `_ is hosted on the Github Pages of this project.\n\nFeatures\n========\n\n* Download or upload files from/to the Exasol BucketFS\n* Supported sources and targets for the uploads and downloads:\n\n * Files on the local Filesystem\n * Python file objects\n * Python Strings\n * Python objects ((De-)Serialization with [Joblib](https://joblib.readthedocs.io/en/latest/persistence.html))\n\n* Loading an artefact from a public Github Release into the BucketFS\n', 'author': 'Torsten Kilias', 'author_email': 'torsten.kilias@exasol.com', 'maintainer': None, @@ -23,7 +23,7 @@ 'packages': packages, 'package_data': package_data, 'install_requires': install_requires, - 'python_requires': '>=3.6.1', + 'python_requires': '>=3.6.1,<4.0', }