From b32cb882f3ec9f48b899aa6e82750762fb55437a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:15:46 -0300 Subject: [PATCH 1/7] #31: Setup shellcheck for the starter scripts --- .github/workflows/shellcheck.yaml | 11 +++++++++++ githooks/create_checksums.sh | 2 +- githooks/install.sh | 3 ++- githooks/update_packaging.sh | 5 ++--- .../checksums/exaslct_install_template.sh.sha512sum | 2 +- installer/exaslct_install_template.sh | 2 +- scripts/build/build_release.sh | 3 ++- scripts/build/check_release.sh | 3 ++- scripts/build/shellcheck.sh | 11 +++++++++++ scripts/test/ci_tests/get_ci_envs.sh | 3 ++- scripts/test/ci_tests/run_ci_test.sh | 2 +- scripts/test/ci_tests/run_ci_test_within_docker.sh | 2 +- scripts/test/get_all_tests.sh | 2 +- scripts/test/run_test.sh | 1 + .../checksums/exaslct_with_poetry.sh.sha512sum | 2 +- .../exaslct_within_docker_container.sh.sha512sum | 2 +- .../exaslct_within_docker_container_slim.sh.sha512sum | 2 +- .../checksums/mount_point_parsing.sh.sha512sum | 2 +- .../checksums/push_docker_runner_image.sh.sha512sum | 2 +- starter_scripts/exaslct_with_poetry.sh | 2 ++ starter_scripts/exaslct_within_docker_container.sh | 1 + .../exaslct_within_docker_container_slim.sh | 3 +-- starter_scripts/mount_point_parsing.sh | 1 + starter_scripts/push_docker_runner_image.sh | 2 +- starter_scripts/test/test_scripts.sh | 2 +- 25 files changed, 51 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/shellcheck.yaml create mode 100755 scripts/build/shellcheck.sh diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 00000000..0ff99b28 --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,11 @@ +name: Check bash scripts + +on: [push, pullrequest] + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run shellcheck + run: ./scripts/build/shellcheck.sh diff --git a/githooks/create_checksums.sh b/githooks/create_checksums.sh index b9c0cb35..409f7af3 100644 --- a/githooks/create_checksums.sh +++ b/githooks/create_checksums.sh @@ -6,4 +6,4 @@ CHECKSUM_DIRECTORY="$PWD/checksums" if [ ! -e "$CHECKSUM_DIRECTORY" ]; then mkdir "$CHECKSUM_DIRECTORY" fi -find $PWD -maxdepth 1 -type f -printf "%f\\0" | xargs --null -n1 -I{} bash -c "sha512sum {} > '$CHECKSUM_DIRECTORY/{}.sha512sum'" +find "$PWD" -maxdepth 1 -type f -printf "%f\\0" | xargs --null -n1 -I{} bash -c "sha512sum {} > '$CHECKSUM_DIRECTORY/{}.sha512sum'" diff --git a/githooks/install.sh b/githooks/install.sh index 757a917f..81264309 100755 --- a/githooks/install.sh +++ b/githooks/install.sh @@ -26,7 +26,8 @@ GITHOOKS_PATH="$(readlink -f "${GITHOOKS_PATH}")" copy_hook() { local SCRIPT_PATH="$SCRIPT_DIR/$1" local GITHOOK_PATH="$GITHOOKS_PATH/$2" - local RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH") + local RELATIVE_PATH="" + RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH") pushd "$GITHOOKS_PATH" > /dev/null if [ -e "$GITHOOK_PATH" ] || [ -L "$GITHOOK_PATH" ] then diff --git a/githooks/update_packaging.sh b/githooks/update_packaging.sh index 4b0738df..f45e06bc 100755 --- a/githooks/update_packaging.sh +++ b/githooks/update_packaging.sh @@ -6,7 +6,6 @@ set -o pipefail SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" # define colors for use in output -green='\033[0;32m' no_color='\033[0m' grey='\033[0;90m' @@ -31,9 +30,9 @@ then fi poetry build > /dev/null pushd dist > /dev/null -tar_file=$(ls *.tar.gz) +tar_file=$(ls -- *.tar.gz) extracted_dir=${tar_file%.tar.gz} -tar -xf $tar_file +tar -xf "$tar_file" cp "$extracted_dir/setup.py" ../setup.py rm -r "$extracted_dir" popd > /dev/null diff --git a/installer/checksums/exaslct_install_template.sh.sha512sum b/installer/checksums/exaslct_install_template.sh.sha512sum index 12cb7e09..c3d6815a 100644 --- a/installer/checksums/exaslct_install_template.sh.sha512sum +++ b/installer/checksums/exaslct_install_template.sh.sha512sum @@ -1 +1 @@ -088853eea60a69592c4e5e0f3c811f60353dfebb4172313bdd01727cbc36f9ac8b6d555986d0cac0b32f1073cea3f0b6ff5ea26e0cf08052069b8e57467f5586 exaslct_install_template.sh +d0a4303e68cff1d94ceb1568a167d978a3e3b6753334f15d1e5ba6788c97fb40ac0d3673de0b6f2b401c9c3656cc1ca35e586f0a4216c36200d45a21d1bdee22 exaslct_install_template.sh diff --git a/installer/exaslct_install_template.sh b/installer/exaslct_install_template.sh index 06a6df35..3995a0f2 100755 --- a/installer/exaslct_install_template.sh +++ b/installer/exaslct_install_template.sh @@ -21,5 +21,5 @@ SCRIPT_DIR="$(dirname "$($rl -f "${BASH_SOURCE[0]}")")" EXASLCT_GIT_REF="<<<>>>" RUNNER_IMAGE_NAME="$(bash "$SCRIPT_DIR/construct_docker_runner_image_name.sh" "$EXASLCT_GIT_REF")" -bash $SCRIPT_DIR/exaslct_within_docker_container_without_container_build.sh "$RUNNER_IMAGE_NAME" "${@}" +bash "$SCRIPT_DIR/exaslct_within_docker_container_without_container_build.sh" "$RUNNER_IMAGE_NAME" "${@}" diff --git a/scripts/build/build_release.sh b/scripts/build/build_release.sh index c7fb4a78..b7e018f6 100755 --- a/scripts/build/build_release.sh +++ b/scripts/build/build_release.sh @@ -2,7 +2,8 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" - + +#shellcheck source=./starter_scripts/poetry_utils.sh source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh" check_requirements diff --git a/scripts/build/check_release.sh b/scripts/build/check_release.sh index cde6b887..1175273d 100755 --- a/scripts/build/check_release.sh +++ b/scripts/build/check_release.sh @@ -2,6 +2,7 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +#shellcheck source=./starter_scripts/poetry_utils.sh source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh" check_requirements @@ -12,7 +13,7 @@ init_poetry if [ -n "$POETRY_BIN" ] then - PYTHONPATH=$SCRIPT_DIR/../.. $POETRY_BIN run python3 -u $SCRIPT_DIR/check_release.py + PYTHONPATH="$SCRIPT_DIR/../.." $POETRY_BIN run python3 -u "$SCRIPT_DIR/check_release.py" else echo "Could not find poetry!" exit 1 diff --git a/scripts/build/shellcheck.sh b/scripts/build/shellcheck.sh new file mode 100755 index 00000000..089acc59 --- /dev/null +++ b/scripts/build/shellcheck.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +interesting_paths=("scripts" "starter_scripts" "githooks" "installer" "project_management_scripts") + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +for path in "${interesting_paths[@]}"; do + find "$SCRIPT_DIR/../../$path" -name '*.sh' -type f -print0 | xargs -0 -n1 shellcheck -x +done diff --git a/scripts/test/ci_tests/get_ci_envs.sh b/scripts/test/ci_tests/get_ci_envs.sh index 682918c6..d7c47f74 100755 --- a/scripts/test/ci_tests/get_ci_envs.sh +++ b/scripts/test/ci_tests/get_ci_envs.sh @@ -3,5 +3,6 @@ set -e SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# shellcheck disable=SC2046 CI_ENVS=$(basename -a $(ls -d "${SCRIPT_DIR}"/*/)) -printf '%s\n' ${CI_ENVS[@]} | jq -R . | jq -cs . \ No newline at end of file +printf '%s\n' "${CI_ENVS[@]}" | jq -R . | jq -cs . \ No newline at end of file diff --git a/scripts/test/ci_tests/run_ci_test.sh b/scripts/test/ci_tests/run_ci_test.sh index d3fdecb8..cd2c67d3 100755 --- a/scripts/test/ci_tests/run_ci_test.sh +++ b/scripts/test/ci_tests/run_ci_test.sh @@ -17,4 +17,4 @@ docker container rm -f -v ci_env_test || true #Need to mount docker socket as we use docker within the CI environment #Also need to mount root directory of script-languages-container-tool because we want to test this commit PROJ_DIR="$(realpath "$SCRIPT_DIR/../../..")" -docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -v "${PROJ_DIR}":"${PROJ_DIR}" --name ci_env_test -t $IMAGE_NAME "${PROJ_DIR}" \ No newline at end of file +docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -v "${PROJ_DIR}":"${PROJ_DIR}" --name ci_env_test -t "$IMAGE_NAME" "${PROJ_DIR}" \ No newline at end of file diff --git a/scripts/test/ci_tests/run_ci_test_within_docker.sh b/scripts/test/ci_tests/run_ci_test_within_docker.sh index 7b4d9882..42a02e45 100755 --- a/scripts/test/ci_tests/run_ci_test_within_docker.sh +++ b/scripts/test/ci_tests/run_ci_test_within_docker.sh @@ -2,7 +2,7 @@ set -e -cd $1 +cd "$1" echo "************* RUN BASH TEST **********" ./starter_scripts/test/test_scripts.sh diff --git a/scripts/test/get_all_tests.sh b/scripts/test/get_all_tests.sh index 7a7953dc..dfed1a74 100755 --- a/scripts/test/get_all_tests.sh +++ b/scripts/test/get_all_tests.sh @@ -2,5 +2,5 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -declare -a test_array=( $(ls $SCRIPT_DIR/../../test/test_*.py)) +declare -a test_array=( $(ls "$SCRIPT_DIR/../../test/test_*.py")) printf '%s\n' "${test_array[@]}" | jq -R . | jq -cs . \ No newline at end of file diff --git a/scripts/test/run_test.sh b/scripts/test/run_test.sh index ef235a24..d4668290 100755 --- a/scripts/test/run_test.sh +++ b/scripts/test/run_test.sh @@ -2,6 +2,7 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +#shellcheck source=./starter_scripts/poetry_utils.sh source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh" check_requirements diff --git a/starter_scripts/checksums/exaslct_with_poetry.sh.sha512sum b/starter_scripts/checksums/exaslct_with_poetry.sh.sha512sum index aa8ec9f1..e27d390b 100644 --- a/starter_scripts/checksums/exaslct_with_poetry.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_with_poetry.sh.sha512sum @@ -1 +1 @@ -285eb790ad7675bd32ab883a81050578bbc263cd346d905192c7b639117592b9b12ba2ad3c32a74439add315ce455a99b8f54e24663f6e9aa486c00f7e7a2845 exaslct_with_poetry.sh +603939626632a7a558e9014d4d8b7efecfedf540e7cb488373bfb776f8ac16d1935cd8416e0704458b5cd096801499bf7f3d20aa8c30297bd61276430f0d3acf exaslct_with_poetry.sh diff --git a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum index 5450d33a..16768c21 100644 --- a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum @@ -1 +1 @@ -26f584f6dc72b01fcf37e6666534580415c1bec73899f7fa2959acd6dc7def7b965a1649e580af7ab503d0a139c02dca1e23183332ac236e0ae0d0dd8029c53a exaslct_within_docker_container.sh +663d2b7b0db2a12bf0c6aa5d2919b784fd82a3b27370a0db4f9f7473cb1d79858b7b2897434d1a7d6e55ade0c03b9845c88309cbc55c76efa7fb65404050bf47 exaslct_within_docker_container.sh diff --git a/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum b/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum index 8c2e4503..9e096043 100644 --- a/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum @@ -1 +1 @@ -fbe0b3fafce7aa34eca0018d8cbfe0f61a57674065a3c270564e8334c94292af0faa6f3f628e278b7e0c4f79cf12deaeeeb114ae91e66065dfcdc712f1b9ba44 exaslct_within_docker_container_slim.sh +8c81b04d13ea1e8b176663597ef9c4cc9b8806e61d5e613d3d6360847d7f04d8b6616167fe2a8106777a2b0189eee8e36645c3cfdfaa8bb00eb974e0a849a305 exaslct_within_docker_container_slim.sh diff --git a/starter_scripts/checksums/mount_point_parsing.sh.sha512sum b/starter_scripts/checksums/mount_point_parsing.sh.sha512sum index f6c70606..18832934 100644 --- a/starter_scripts/checksums/mount_point_parsing.sh.sha512sum +++ b/starter_scripts/checksums/mount_point_parsing.sh.sha512sum @@ -1 +1 @@ -9481d697fa3a9143a3571171c69b7b5bcb13902aebc83d879f34e874c1878b1e7d74d7217dc18ba77c7b502f753cb505f94afc141e5153458fc7b61a6ac28ee6 mount_point_parsing.sh +5ab6c49d7a91d4d169bdae947e876ae1ab12de747ec369d73e4e411751d641431b657d6eb7e506805e4a0e23cae1eedb4cbe8bd277a93358fe0c002b459ca16b mount_point_parsing.sh diff --git a/starter_scripts/checksums/push_docker_runner_image.sh.sha512sum b/starter_scripts/checksums/push_docker_runner_image.sh.sha512sum index 9088cc49..693f65f1 100644 --- a/starter_scripts/checksums/push_docker_runner_image.sh.sha512sum +++ b/starter_scripts/checksums/push_docker_runner_image.sh.sha512sum @@ -1 +1 @@ -a779c2e4c65051d7baa865c24248bbd23faf4998d3d62638181930b81f2bbb209e2a26defbe473f203c15154c15eda5e2e44cfad0533fe8630d619dbbc38699b push_docker_runner_image.sh +68f6746d14673ebb413588cdf04ec6c71168aa5676766cee02297a7e859948e55ecf9b4f297bdbf57ee7eb7f11a0b7faef90935632891bcf5313f5474540250c push_docker_runner_image.sh diff --git a/starter_scripts/exaslct_with_poetry.sh b/starter_scripts/exaslct_with_poetry.sh index e9b69928..28139f1c 100755 --- a/starter_scripts/exaslct_with_poetry.sh +++ b/starter_scripts/exaslct_with_poetry.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash + SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +#shellcheck source=./starter_scripts/poetry_utils.sh source "$SCRIPT_DIR/poetry_utils.sh" check_requirements diff --git a/starter_scripts/exaslct_within_docker_container.sh b/starter_scripts/exaslct_within_docker_container.sh index be85e331..baeca4da 100755 --- a/starter_scripts/exaslct_within_docker_container.sh +++ b/starter_scripts/exaslct_within_docker_container.sh @@ -93,6 +93,7 @@ tmpfile_env=$(mktemp) trap 'rm -f -- "$tmpfile_env"' INT TERM HUP EXIT create_env_file_debug_protected "$tmpfile_env" +# shellcheck disable=SC2068 docker run --network host --env-file "$tmpfile_env" --rm $terminal_parameter -v "$PWD:$PWD" -v "$DOCKER_SOCKET_MOUNT" -w "$PWD" ${mount_point_parameter[@]} "$RUNNER_IMAGE_NAME" bash -c "$RUN_COMMAND" umask "$old_umask" diff --git a/starter_scripts/exaslct_within_docker_container_slim.sh b/starter_scripts/exaslct_within_docker_container_slim.sh index 77bc8cd0..000f744a 100755 --- a/starter_scripts/exaslct_within_docker_container_slim.sh +++ b/starter_scripts/exaslct_within_docker_container_slim.sh @@ -18,6 +18,7 @@ else fi RED='\033[0;31m' +#shellcheck disable=SC2059 printf "${RED} Using slim version of exaslct. Please note that all input/output arguments (input flavor directories, output, log, etc.) needs to point to directories within the current directory.\n" rl=readlink @@ -30,8 +31,6 @@ if [[ ! "$(command -v $rl)" ]]; then exit 1 fi -SCRIPT_DIR="$(dirname "$($rl -f "${BASH_SOURCE[0]}")")" - quoted_arguments='' for argument in "${@}"; do argument="${argument//\\/\\\\}" diff --git a/starter_scripts/mount_point_parsing.sh b/starter_scripts/mount_point_parsing.sh index 6980b744..8198cc67 100755 --- a/starter_scripts/mount_point_parsing.sh +++ b/starter_scripts/mount_point_parsing.sh @@ -124,6 +124,7 @@ function _format_parameters() { declare -a mount_point_paths formatted_params=$(_format_parameters "${@}") +# shellcheck disable=SC2086 _get_mount_point_paths $formatted_params echo "${mount_point_paths[@]}" \ No newline at end of file diff --git a/starter_scripts/push_docker_runner_image.sh b/starter_scripts/push_docker_runner_image.sh index 70505db3..6ee828f2 100755 --- a/starter_scripts/push_docker_runner_image.sh +++ b/starter_scripts/push_docker_runner_image.sh @@ -7,7 +7,7 @@ set -euo pipefail if [ -n "${1-}" ]; then image_suffixes=("$@") else - image_suffixes="latest" + image_suffixes=("latest") fi SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" diff --git a/starter_scripts/test/test_scripts.sh b/starter_scripts/test/test_scripts.sh index 9928aac9..dc182f7e 100755 --- a/starter_scripts/test/test_scripts.sh +++ b/starter_scripts/test/test_scripts.sh @@ -23,7 +23,7 @@ flavorDirA="$SCRIPT_DIR/test/abc=def" exportDirA="$SCRIPT_DIR/test/exportdir=xyz" mkdir "$flavorDirA" || true -trap "rm -rf $flavorDirA $exportDirA" EXIT +trap 'rm -rf "$flavorDirA" "$exportDirA"' EXIT testStr=$(bash "$SCRIPT_DIR/mount_point_parsing.sh" --flavor-path="$flavorDirA" --export-path "$exportDirA" dummy) From 5d800e6425255a83ba469851ce32140f95b51b44 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:17:15 -0300 Subject: [PATCH 2/7] Fix yaml file --- .github/workflows/shellcheck.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 0ff99b28..019ff3e3 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -1,6 +1,6 @@ name: Check bash scripts -on: [push, pullrequest] +on: [push, pull_request] jobs: shellcheck: From 9a315cd3542e3c6a14cb6bb3c1ee0be0362a3421 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:27:54 -0300 Subject: [PATCH 3/7] Added ignore tag to get_all_tests.sh --- scripts/test/get_all_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/get_all_tests.sh b/scripts/test/get_all_tests.sh index dfed1a74..5e44f94d 100755 --- a/scripts/test/get_all_tests.sh +++ b/scripts/test/get_all_tests.sh @@ -2,5 +2,6 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# shellcheck disable=SC2207 declare -a test_array=( $(ls "$SCRIPT_DIR/../../test/test_*.py")) printf '%s\n' "${test_array[@]}" | jq -R . | jq -cs . \ No newline at end of file From a96ebaca97d8804e5b084145cd9d03b23855866b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:38:13 -0300 Subject: [PATCH 4/7] 1/ Added ignore tag to exaslct_within_docker_container.sh and test_scripts.sh 2. Updated changelog --- doc/changes/changelog.md | 1 + doc/changes/changes_0.9.0.md | 19 +++++++++++++++++++ pyproject.toml | 2 +- setup.py | 2 +- ...aslct_within_docker_container.sh.sha512sum | 2 +- .../exaslct_within_docker_container.sh | 1 + starter_scripts/test/test_scripts.sh | 2 +- 7 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 doc/changes/changes_0.9.0.md diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 7408a568..0b535f50 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [0.9.0](changes_0.9.0.md) * [0.8.0](changes_0.8.0.md) * [0.7.0](changes_0.7.0.md) * [0.6.0](changes_0.6.0.md) diff --git a/doc/changes/changes_0.9.0.md b/doc/changes/changes_0.9.0.md new file mode 100644 index 00000000..65ef9442 --- /dev/null +++ b/doc/changes/changes_0.9.0.md @@ -0,0 +1,19 @@ +# Script-Languages-Container-Tool 0.9.0, released t.b.d. + +Code name: t.b.d. + +## Summary + +t.b.d. + +## Features / Enhancements + + - #31: Setup shellcheck for the starter scripts + +## Bug Fixes + +n/a + +## Documentation +n/a + diff --git a/pyproject.toml b/pyproject.toml index 7a251089..6c993732 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "exasol-script-languages-container-tool" -version = "0.8.0" +version = "0.9.0" description = "Script Languages Container Tool" license = "MIT" diff --git a/setup.py b/setup.py index 1701c45b..ec18677f 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup_kwargs = { 'name': 'exasol-script-languages-container-tool', - 'version': '0.8.0', + 'version': '0.9.0', 'description': 'Script Languages Container Tool', 'long_description': "# Script-Languages-Container-Tool\n\n## Overview\n\nThe Script-Languages-Container-Tool (exaslct) is the build tool for the script language container.\nYou can build, export and upload script-language container from so-called flavors \nwhich are description how to build the script language container. You can find pre-defined flavors \nin the [script-languages-release](https://github.com/exasol/script-languages-release) repository. \nThere we also described how you could customize these flavors to your needs.\n\n## In a Nutshell\n\n### Prerequisites\n\n#### For installation\n\nIn order to install this tool, your system needs to provide \nthe following prerequisites:\n\n* Software\n * Linux\n * [bash](https://www.gnu.org/software/bash/) >= 4.2\n * MacOsX\n * [bash](https://www.gnu.org/software/bash/) > 3.2\n * [coreutils](https://www.gnu.org/software/coreutils/)\n * sha512sum\n * sed\n * [curl](https://curl.se/)\n\n\n#### For running\n\nIn order to use this tool, your system needs to fulfill the following prerequisites:\n\n* Software\n * Linux\n * [bash](https://www.gnu.org/software/bash/) >= 4.2\n * [coreutils](https://www.gnu.org/software/coreutils/)\n * readlink with -f option\n * realpath \n * dirname\n * MacOsX (Please see limitations on [MacOsX](#macosx-limitations))\n * [bash](https://www.gnu.org/software/bash/) >= 3.2\n * [coreutils](https://www.gnu.org/software/coreutils/)\n * greadlink with -f option\n * realpath \n * dirname\n * [Docker](https://docs.docker.com/) >= 17.05 \n * with support for [multi-stage builds required](https://docs.docker.com/develop/develop-images/multistage-build/)\n * host volume mounts need to be allowed\n \n* System Setup \n * We recommend at least 50 GB free disk space on the partition \n where Docker stores its images, on linux Docker typically stores \n the images at /var/lib/docker.\n * For the partition where the output directory (default: ./.build_output)\n is located we recommend additionally at least 10 GB free disk space.\n\nFurther, prerequisites might be necessary for specific tasks. These are listed under the corresponding section.\n\n### Installation\n\nDownload the installation and update script via:\n\n```\ncurl -L -o install_or_update_exaslct.sh https://raw.githubusercontent.com/exasol/script-languages-container-tool/main/installer/install_or_update_exaslct.sh\n```\n\nBefore you continue with installation, please compute with the following command \nthe sha512 hash sum of the downloaded file and compare it with its \n[checksum file](installer/checksums/install_or_update_exaslct.sh.sha512sum):\n\n```\nsha512sum install_or_update_exaslct.sh\n```\n\nIf the checksums are identical, you can continue with the installation. \nPer default, the script installs exaslct into the current working directory.\nIt creates a script directory `exaslct_scripts` and the symlink `exaslct`\nto the starter script. If you want to change the path to the script directory \nyou can set the environment variable `EXASLCT_INSTALL_DIRECTORY` and \nif you want to create the symlink somewhere else you can set `EXASLCT_SYM_LINK_PATH`. \n\n```\nbash install_or_update_exaslct.sh [version|git-commit-id|branch|tag] \n```\n\nYou can use the same script to change the version of your current installation.\nYou only need to provide a different version, git-commit-id, branch or tag. \n\n### Usage\n\n#### How to build an existing flavor?\n\nCreate the language container and export it to the local file system\n\n```bash\n./exaslct export --flavor-path=flavors/ --export-path \n```\n\nor upload it directly into the BucketFS (currently http only, https follows soon)\n\n```bash\n./exaslct upload --flavor-path=flavors/ --database-host --bucketfs-port \\ \n --bucketfs-username w --bucketfs-password --bucketfs-name \\\n --bucket-name --path-in-bucket \n```\n\nOnce it is successfully uploaded, it will print the ALTER SESSION statement\nthat can be used to activate the script language container in the database.\n\n#### How to activate a script language container in the database\n\nIf you uploaded a container manually, you can generate the language activation statement with\n\n```bash\n./exaslct generate-language-activation --flavor-path=flavors/ --bucketfs-name \\\n --bucket-name --path-in-bucket --container-name \n```\n\nwhere \\ is the name of the uploaded archive without its file extension. To activate the language, execute the generated statement in your database session to activate the container for the current session or system wide.\n\nThis command will print a SQL statement to activate the language similar to the following one:\n\n```bash\nALTER SESSION SET SCRIPT_LANGUAGES='=localzmq+protobuf://////?lang=#buckets/////exaudf/exaudfclient[_py3]';\n```\n\n**Please, refer to the User Guide for more detailed information, how to use exalsct.**\n\n## Features\n\n* Build a script language container as docker images\n* Export a script language container as an archive which can be used for extending Exasol UDFs\n* Upload a script language container as an archive to the Exasol DB's BucketFS\n* Generating the activation command for a script language container\n* Can use Docker registries, such as Docker Hub, as a cache to avoid rebuilding image without changes\n* Can push Docker images to Docker registries\n* Run tests for you container against an Exasol DB (docker-db or external db)\n\n## Limitations\n\n* Caution with symbolic links: \n If you use symbolic links inside any directory of the command line arguments\n they must not point to files or directories outside the root of the path of the \n command line argument (i.e. --flavor-path ./flavors/my_flavor/ => There must be no symbolic\n link inside ./flavors/my_flavor point to anywhere outside of ./flavors/my_flavor).\n Background: Local directories paths must be mounted manually to the docker container. \n We currently support only the mounting of the given command line arguments, but we do not analyze\n the content of those directories.\n Plan is to fix this limitation with [#35](https://github.com/exasol/script-languages-container-tool/issues/35)\n\n### MacOsX Limitations\n \n* On MacOsX all arguments (flavors path, output directory, etc.) must point to locations within the current directory (background is that the MacOsX version does not support mount binding additional directories).\n\n## Table of Contents\n\n### Information for Users\n\n* [User Guide](doc/user_guide/user_guide.md)\n* [Changelog](doc/changes/changelog.md)\n\n## Information for Developers\n\n* [Developer Guide](doc/developer_guide/developer_guide.md)\n* [Dependencies](doc/dependencies.md)\n", 'author': 'Torsten Kilias', diff --git a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum index 16768c21..e69ca844 100644 --- a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum @@ -1 +1 @@ -663d2b7b0db2a12bf0c6aa5d2919b784fd82a3b27370a0db4f9f7473cb1d79858b7b2897434d1a7d6e55ade0c03b9845c88309cbc55c76efa7fb65404050bf47 exaslct_within_docker_container.sh +019b7712453f8f416c3d5fffba3e9389fb060c68bd10f03651d4358c8a18d0e57b8d3a99638f36e069c8bd3941a71444c8e40ac0886de77468c734ecc87b7c7a exaslct_within_docker_container.sh diff --git a/starter_scripts/exaslct_within_docker_container.sh b/starter_scripts/exaslct_within_docker_container.sh index baeca4da..d0ca555c 100755 --- a/starter_scripts/exaslct_within_docker_container.sh +++ b/starter_scripts/exaslct_within_docker_container.sh @@ -22,6 +22,7 @@ fi SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" declare -a mount_point_paths +# shellcheck disable=SC2207 mount_point_paths=($(bash "$SCRIPT_DIR"/mount_point_parsing.sh "${@}")) quoted_arguments='' diff --git a/starter_scripts/test/test_scripts.sh b/starter_scripts/test/test_scripts.sh index dc182f7e..d0a7dff8 100755 --- a/starter_scripts/test/test_scripts.sh +++ b/starter_scripts/test/test_scripts.sh @@ -15,7 +15,7 @@ function assert() { cmpB="${*}" if [[ $cmpA != "$cmpB" ]]; then >&2 echo "ERROR: '$cmpA' does not match'$cmpB'" - exit -1 + exit 1 fi } From d7da6e28745187cc74f463d192d54e1b2394d84c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 30 Dec 2021 11:04:37 -0300 Subject: [PATCH 5/7] 1. Added ignore tag to get_all_tests.sh --- scripts/test/get_all_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test/get_all_tests.sh b/scripts/test/get_all_tests.sh index 5e44f94d..fe0b9e56 100755 --- a/scripts/test/get_all_tests.sh +++ b/scripts/test/get_all_tests.sh @@ -2,6 +2,6 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" -# shellcheck disable=SC2207 -declare -a test_array=( $(ls "$SCRIPT_DIR/../../test/test_*.py")) +# shellcheck disable=SC2207,SC2086 +declare -a test_array=( $(ls $SCRIPT_DIR/../../test/test_*.py)) printf '%s\n' "${test_array[@]}" | jq -R . | jq -cs . \ No newline at end of file From d46d92b834d18f737966ce0ca54e6d502919f704 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 4 Jan 2022 06:59:41 -0300 Subject: [PATCH 6/7] Changed shellcheck.sh so that it runs check on all files and returns 1 if any check failed. --- scripts/build/shellcheck.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build/shellcheck.sh b/scripts/build/shellcheck.sh index 089acc59..6932f40d 100755 --- a/scripts/build/shellcheck.sh +++ b/scripts/build/shellcheck.sh @@ -1,11 +1,15 @@ #!/bin/bash -set -euo pipefail +set -u interesting_paths=("scripts" "starter_scripts" "githooks" "installer" "project_management_scripts") SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +status=0 for path in "${interesting_paths[@]}"; do find "$SCRIPT_DIR/../../$path" -name '*.sh' -type f -print0 | xargs -0 -n1 shellcheck -x + test $? -ne 0 && status=1 done + +exit "$status" \ No newline at end of file From e7f96d3efae19cae2234b88deff20e4e6c9815fc Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 6 Jan 2022 09:42:51 -0300 Subject: [PATCH 7/7] Fixed comments from review: 1. Added motivation for ignoring specific rules. 2. Added quotes in check_release.sh --- scripts/build/check_release.sh | 2 +- scripts/test/ci_tests/get_ci_envs.sh | 1 + scripts/test/get_all_tests.sh | 2 ++ .../checksums/exaslct_within_docker_container.sh.sha512sum | 2 +- .../checksums/exaslct_within_docker_container_slim.sh.sha512sum | 2 +- starter_scripts/checksums/mount_point_parsing.sh.sha512sum | 2 +- starter_scripts/exaslct_within_docker_container.sh | 2 ++ starter_scripts/exaslct_within_docker_container_slim.sh | 1 + starter_scripts/mount_point_parsing.sh | 1 + 9 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/build/check_release.sh b/scripts/build/check_release.sh index 1175273d..7c506b45 100755 --- a/scripts/build/check_release.sh +++ b/scripts/build/check_release.sh @@ -13,7 +13,7 @@ init_poetry if [ -n "$POETRY_BIN" ] then - PYTHONPATH="$SCRIPT_DIR/../.." $POETRY_BIN run python3 -u "$SCRIPT_DIR/check_release.py" + PYTHONPATH="$SCRIPT_DIR/../.." "$POETRY_BIN" run python3 -u "$SCRIPT_DIR/check_release.py" else echo "Could not find poetry!" exit 1 diff --git a/scripts/test/ci_tests/get_ci_envs.sh b/scripts/test/ci_tests/get_ci_envs.sh index d7c47f74..5b43ac3c 100755 --- a/scripts/test/ci_tests/get_ci_envs.sh +++ b/scripts/test/ci_tests/get_ci_envs.sh @@ -3,6 +3,7 @@ set -e SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# Ignore shellcheck rule here, as we want to split result by space # shellcheck disable=SC2046 CI_ENVS=$(basename -a $(ls -d "${SCRIPT_DIR}"/*/)) printf '%s\n' "${CI_ENVS[@]}" | jq -R . | jq -cs . \ No newline at end of file diff --git a/scripts/test/get_all_tests.sh b/scripts/test/get_all_tests.sh index fe0b9e56..de6db945 100755 --- a/scripts/test/get_all_tests.sh +++ b/scripts/test/get_all_tests.sh @@ -2,6 +2,8 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +# Ignore shellcheck rule SC2207 here: Alternatives are difficult to read and understand. None of the test filename has spaces or globs, so it's safe. +# Ignore shellcheck rule SC2086 here: ls does not work correct with quotes here (probably because of the asterisk). # shellcheck disable=SC2207,SC2086 declare -a test_array=( $(ls $SCRIPT_DIR/../../test/test_*.py)) printf '%s\n' "${test_array[@]}" | jq -R . | jq -cs . \ No newline at end of file diff --git a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum index e69ca844..232c9989 100644 --- a/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum @@ -1 +1 @@ -019b7712453f8f416c3d5fffba3e9389fb060c68bd10f03651d4358c8a18d0e57b8d3a99638f36e069c8bd3941a71444c8e40ac0886de77468c734ecc87b7c7a exaslct_within_docker_container.sh +85e705ec636a4fb6bb9487f5378405e95173a184d43fe3fc42ed19eb12ed56a92ea614a2171aaff7650bd6645d7ef0d5cdfb86602d63d9e169671a1fe08aa8ea exaslct_within_docker_container.sh diff --git a/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum b/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum index 9e096043..8f98e4b0 100644 --- a/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum +++ b/starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum @@ -1 +1 @@ -8c81b04d13ea1e8b176663597ef9c4cc9b8806e61d5e613d3d6360847d7f04d8b6616167fe2a8106777a2b0189eee8e36645c3cfdfaa8bb00eb974e0a849a305 exaslct_within_docker_container_slim.sh +979851c4a9f5376c1c758dead76ca3827fee1184f040a38fac6cb5033811d997c6292dbfe079042098e26a5c41f11b73d5c27acbe7cd2169e82fa10e92b0a6b1 exaslct_within_docker_container_slim.sh diff --git a/starter_scripts/checksums/mount_point_parsing.sh.sha512sum b/starter_scripts/checksums/mount_point_parsing.sh.sha512sum index 18832934..f8f75ce9 100644 --- a/starter_scripts/checksums/mount_point_parsing.sh.sha512sum +++ b/starter_scripts/checksums/mount_point_parsing.sh.sha512sum @@ -1 +1 @@ -5ab6c49d7a91d4d169bdae947e876ae1ab12de747ec369d73e4e411751d641431b657d6eb7e506805e4a0e23cae1eedb4cbe8bd277a93358fe0c002b459ca16b mount_point_parsing.sh +3428c95db1c3d5e99d58be17e9987f9264a537cfcaf7bb0a245386c20e94e1e84a4c2b4566b62abf543e8a2faf69b36e51f965911a7beed0a0f0af9e282b27d9 mount_point_parsing.sh diff --git a/starter_scripts/exaslct_within_docker_container.sh b/starter_scripts/exaslct_within_docker_container.sh index d0ca555c..f91e1d4b 100755 --- a/starter_scripts/exaslct_within_docker_container.sh +++ b/starter_scripts/exaslct_within_docker_container.sh @@ -22,6 +22,7 @@ fi SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" declare -a mount_point_paths +# Ignore shellcheck rules here as alternatives are worse. # shellcheck disable=SC2207 mount_point_paths=($(bash "$SCRIPT_DIR"/mount_point_parsing.sh "${@}")) @@ -94,6 +95,7 @@ tmpfile_env=$(mktemp) trap 'rm -f -- "$tmpfile_env"' INT TERM HUP EXIT create_env_file_debug_protected "$tmpfile_env" +# Ignore shellcheck rule as we need to split elements of array by space (they are in form "-v %MOUNT_POINT") # shellcheck disable=SC2068 docker run --network host --env-file "$tmpfile_env" --rm $terminal_parameter -v "$PWD:$PWD" -v "$DOCKER_SOCKET_MOUNT" -w "$PWD" ${mount_point_parameter[@]} "$RUNNER_IMAGE_NAME" bash -c "$RUN_COMMAND" diff --git a/starter_scripts/exaslct_within_docker_container_slim.sh b/starter_scripts/exaslct_within_docker_container_slim.sh index 000f744a..2d679d53 100755 --- a/starter_scripts/exaslct_within_docker_container_slim.sh +++ b/starter_scripts/exaslct_within_docker_container_slim.sh @@ -18,6 +18,7 @@ else fi RED='\033[0;31m' +#Ignore shellcheck rule, as using printf does not change the color in the terminal. #shellcheck disable=SC2059 printf "${RED} Using slim version of exaslct. Please note that all input/output arguments (input flavor directories, output, log, etc.) needs to point to directories within the current directory.\n" diff --git a/starter_scripts/mount_point_parsing.sh b/starter_scripts/mount_point_parsing.sh index 8198cc67..44b0dd2a 100755 --- a/starter_scripts/mount_point_parsing.sh +++ b/starter_scripts/mount_point_parsing.sh @@ -124,6 +124,7 @@ function _format_parameters() { declare -a mount_point_paths formatted_params=$(_format_parameters "${@}") +# Ignore shellcheck rule as we want to split parameters by space intentionally # shellcheck disable=SC2086 _get_mount_point_paths $formatted_params