#31: Setup shellcheck for the starter scripts

.github/workflows/shellcheck.yaml

@@ -0,0 +1,11 @@
+name: Check bash scripts
+
+on: [push, pull_request]
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run shellcheck
+        run: ./scripts/build/shellcheck.sh

doc/changes/changelog.md

@@ -1,5 +1,6 @@
 # Changes
 
+* [0.9.0](changes_0.9.0.md)
 * [0.8.0](changes_0.8.0.md)
 * [0.7.0](changes_0.7.0.md)
 * [0.6.0](changes_0.6.0.md)

doc/changes/changes_0.9.0.md

@@ -0,0 +1,19 @@
+# Script-Languages-Container-Tool 0.9.0, released t.b.d.
+
+Code name: t.b.d.
+
+## Summary 
+
+t.b.d.
+
+## Features / Enhancements
+
+ - #31: Setup shellcheck for the starter scripts
+
+## Bug Fixes
+
+n/a
+
+## Documentation
+n/a
+

githooks/create_checksums.sh

@@ -6,4 +6,4 @@ CHECKSUM_DIRECTORY="$PWD/checksums"
 if [ ! -e "$CHECKSUM_DIRECTORY" ]; then
   mkdir "$CHECKSUM_DIRECTORY"
 fi
-find $PWD -maxdepth 1 -type f -printf "%f\\0" | xargs --null -n1 -I{} bash -c "sha512sum {} > '$CHECKSUM_DIRECTORY/{}.sha512sum'"
+find "$PWD" -maxdepth 1 -type f -printf "%f\\0" | xargs --null -n1 -I{} bash -c "sha512sum {} > '$CHECKSUM_DIRECTORY/{}.sha512sum'"

githooks/install.sh

@@ -26,7 +26,8 @@ GITHOOKS_PATH="$(readlink -f "${GITHOOKS_PATH}")"
 copy_hook() {
     local SCRIPT_PATH="$SCRIPT_DIR/$1"
     local GITHOOK_PATH="$GITHOOKS_PATH/$2"
-    local RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH")
+    local RELATIVE_PATH=""
+    RELATIVE_PATH=$(realpath --relative-to="$GITHOOKS_PATH" "$SCRIPT_PATH")
     pushd "$GITHOOKS_PATH" > /dev/null
     if [ -e "$GITHOOK_PATH" ] || [ -L "$GITHOOK_PATH" ]
     then

githooks/update_packaging.sh

@@ -6,7 +6,6 @@ set -o pipefail
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
 # define colors for use in output
-green='\033[0;32m'
 no_color='\033[0m'
 grey='\033[0;90m'
 
@@ -31,9 +30,9 @@ then
 fi
 poetry build > /dev/null
 pushd dist > /dev/null
-tar_file=$(ls *.tar.gz)
+tar_file=$(ls -- *.tar.gz)
 extracted_dir=${tar_file%.tar.gz}
-tar -xf $tar_file
+tar -xf "$tar_file"
 cp "$extracted_dir/setup.py" ../setup.py
 rm -r "$extracted_dir"
 popd > /dev/null

installer/checksums/exaslct_install_template.sh.sha512sum

@@ -1 +1 @@
-088853eea60a69592c4e5e0f3c811f60353dfebb4172313bdd01727cbc36f9ac8b6d555986d0cac0b32f1073cea3f0b6ff5ea26e0cf08052069b8e57467f5586  exaslct_install_template.sh
+d0a4303e68cff1d94ceb1568a167d978a3e3b6753334f15d1e5ba6788c97fb40ac0d3673de0b6f2b401c9c3656cc1ca35e586f0a4216c36200d45a21d1bdee22  exaslct_install_template.sh

installer/exaslct_install_template.sh

@@ -21,5 +21,5 @@ SCRIPT_DIR="$(dirname "$($rl -f "${BASH_SOURCE[0]}")")"
 EXASLCT_GIT_REF="<<<<EXASLCT_GIT_REF>>>>"
 RUNNER_IMAGE_NAME="$(bash "$SCRIPT_DIR/construct_docker_runner_image_name.sh" "$EXASLCT_GIT_REF")"
 
-bash $SCRIPT_DIR/exaslct_within_docker_container_without_container_build.sh "$RUNNER_IMAGE_NAME" "${@}"
+bash "$SCRIPT_DIR/exaslct_within_docker_container_without_container_build.sh" "$RUNNER_IMAGE_NAME" "${@}"
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-script-languages-container-tool"
-version = "0.8.0"
+version = "0.9.0"
 description = "Script Languages Container Tool"
 
 license = "MIT"

scripts/build/build_release.sh

@@ -2,7 +2,8 @@
 
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" 
-
+
+#shellcheck source=./starter_scripts/poetry_utils.sh
 source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh"
 
 check_requirements

scripts/build/check_release.sh

@@ -2,6 +2,7 @@
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
+#shellcheck source=./starter_scripts/poetry_utils.sh
 source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh"
 
 check_requirements
@@ -12,7 +13,7 @@ init_poetry
 
 if [ -n "$POETRY_BIN" ]
 then
-  PYTHONPATH=$SCRIPT_DIR/../.. $POETRY_BIN run python3 -u $SCRIPT_DIR/check_release.py
+  PYTHONPATH="$SCRIPT_DIR/../.." "$POETRY_BIN" run python3 -u "$SCRIPT_DIR/check_release.py"
 else
   echo "Could not find poetry!"
   exit 1

scripts/build/shellcheck.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -u
+
+interesting_paths=("scripts" "starter_scripts" "githooks" "installer" "project_management_scripts")
+
+SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+status=0
+
+for path in "${interesting_paths[@]}"; do
+  find "$SCRIPT_DIR/../../$path" -name '*.sh' -type f -print0 | xargs -0 -n1 shellcheck -x
+  test $? -ne 0 && status=1
+done
+
+exit "$status"

scripts/test/ci_tests/get_ci_envs.sh

@@ -3,5 +3,7 @@
 set -e
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+# Ignore shellcheck rule here, as we want to split result by space
+# shellcheck disable=SC2046
 CI_ENVS=$(basename -a $(ls -d "${SCRIPT_DIR}"/*/))
-printf '%s\n' ${CI_ENVS[@]} | jq -R . | jq -cs .
+printf '%s\n' "${CI_ENVS[@]}" | jq -R . | jq -cs .

scripts/test/ci_tests/run_ci_test.sh

@@ -17,4 +17,4 @@ docker container rm -f -v ci_env_test || true
 #Need to mount docker socket as we use docker within the CI environment
 #Also need to mount root directory of script-languages-container-tool because we want to test this commit
 PROJ_DIR="$(realpath "$SCRIPT_DIR/../../..")"
-docker run  --network host -v /var/run/docker.sock:/var/run/docker.sock -v "${PROJ_DIR}":"${PROJ_DIR}" --name ci_env_test -t $IMAGE_NAME "${PROJ_DIR}"
+docker run  --network host -v /var/run/docker.sock:/var/run/docker.sock -v "${PROJ_DIR}":"${PROJ_DIR}" --name ci_env_test -t "$IMAGE_NAME" "${PROJ_DIR}"

scripts/test/ci_tests/run_ci_test_within_docker.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-cd $1
+cd "$1"
 
 echo "************* RUN BASH TEST **********"
 ./starter_scripts/test/test_scripts.sh

scripts/test/get_all_tests.sh

@@ -2,5 +2,8 @@
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
+# Ignore shellcheck rule SC2207 here: Alternatives are difficult to read and understand. None of the test filename has spaces or globs, so it's safe.
+# Ignore shellcheck rule SC2086 here: ls does not work correct with quotes here (probably because of the asterisk).
+# shellcheck disable=SC2207,SC2086
 declare -a test_array=( $(ls $SCRIPT_DIR/../../test/test_*.py))
 printf '%s\n' "${test_array[@]}" | jq -R . | jq -cs .

scripts/test/run_test.sh

@@ -2,6 +2,7 @@
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
+#shellcheck source=./starter_scripts/poetry_utils.sh
 source "$SCRIPT_DIR/../../starter_scripts/poetry_utils.sh"
 
 check_requirements

setup.py

@@ -27,7 +27,7 @@
 
 setup_kwargs = {
     'name': 'exasol-script-languages-container-tool',
-    'version': '0.8.0',
+    'version': '0.9.0',
     'description': 'Script Languages Container Tool',
     'long_description': "# Script-Languages-Container-Tool\n\n## Overview\n\nThe Script-Languages-Container-Tool (exaslct) is the build tool for the script language container.\nYou can build, export and upload script-language container from so-called flavors \nwhich are description how to build the script language container. You can find pre-defined flavors \nin the [script-languages-release](https://github.com/exasol/script-languages-release) repository. \nThere we also described how you could customize these flavors to your needs.\n\n## In a Nutshell\n\n### Prerequisites\n\n#### For installation\n\nIn order to install this tool, your system needs to provide \nthe following prerequisites:\n\n* Software\n    * Linux\n      * [bash](https://www.gnu.org/software/bash/) >= 4.2\n    * MacOsX\n      * [bash](https://www.gnu.org/software/bash/) > 3.2\n    * [coreutils](https://www.gnu.org/software/coreutils/)\n      * sha512sum\n      * sed\n    * [curl](https://curl.se/)\n\n\n#### For running\n\nIn order to use this tool, your system needs to fulfill the following prerequisites:\n\n* Software\n    * Linux\n      * [bash](https://www.gnu.org/software/bash/) >= 4.2\n      * [coreutils](https://www.gnu.org/software/coreutils/)\n        * readlink with -f option\n        * realpath  \n        * dirname\n    * MacOsX (Please see limitations on [MacOsX](#macosx-limitations))\n      * [bash](https://www.gnu.org/software/bash/) >= 3.2\n      * [coreutils](https://www.gnu.org/software/coreutils/)\n        * greadlink with -f option\n        * realpath  \n        * dirname\n    * [Docker](https://docs.docker.com/) >= 17.05 \n      * with support for [multi-stage builds required](https://docs.docker.com/develop/develop-images/multistage-build/)\n      * host volume mounts need to be allowed\n    \n* System Setup  \n    * We recommend at least 50 GB free disk space on the partition \n      where Docker stores its images, on linux Docker typically stores \n      the images at /var/lib/docker.\n    * For the partition where the output directory (default: ./.build_output)\n      is located we recommend additionally at least 10 GB free disk space.\n\nFurther, prerequisites might be necessary for specific tasks. These are listed under the corresponding section.\n\n### Installation\n\nDownload the installation and update script via:\n\n```\ncurl -L -o install_or_update_exaslct.sh https://raw.githubusercontent.com/exasol/script-languages-container-tool/main/installer/install_or_update_exaslct.sh\n```\n\nBefore you continue with installation, please compute with the following command \nthe sha512 hash sum of the downloaded file and compare it with its \n[checksum file](installer/checksums/install_or_update_exaslct.sh.sha512sum):\n\n```\nsha512sum install_or_update_exaslct.sh\n```\n\nIf the checksums are identical, you can continue with the installation. \nPer default, the script installs exaslct into the current working directory.\nIt creates a script directory `exaslct_scripts` and the symlink `exaslct`\nto the starter script. If you want to change the path to the script directory \nyou can set the environment variable `EXASLCT_INSTALL_DIRECTORY` and \nif you want to create the symlink somewhere else you can set `EXASLCT_SYM_LINK_PATH`.  \n\n```\nbash install_or_update_exaslct.sh [version|git-commit-id|branch|tag] \n```\n\nYou can use the same script to change the version of your current installation.\nYou only need to provide a different version, git-commit-id, branch or tag. \n\n### Usage\n\n#### How to build an existing flavor?\n\nCreate the language container and export it to the local file system\n\n```bash\n./exaslct export --flavor-path=flavors/<flavor-name> --export-path <export-path>\n```\n\nor upload it directly into the BucketFS (currently http only, https follows soon)\n\n```bash\n./exaslct upload --flavor-path=flavors/<flavor-name> --database-host <hostname-or-ip> --bucketfs-port <port> \\ \n                   --bucketfs-username w --bucketfs-password <password>  --bucketfs-name <bucketfs-name> \\\n                   --bucket-name <bucket-name> --path-in-bucket <path/in/bucket>\n```\n\nOnce it is successfully uploaded, it will print the ALTER SESSION statement\nthat can be used to activate the script language container in the database.\n\n#### How to activate a script language container in the database\n\nIf you uploaded a container manually, you can generate the language activation statement with\n\n```bash\n./exaslct generate-language-activation --flavor-path=flavors/<flavor-name> --bucketfs-name <bucketfs-name> \\\n                                         --bucket-name <bucket-name> --path-in-bucket <path/in/bucket> --container-name <container-name>\n```\n\nwhere \\<container-name> is the name of the uploaded archive without its file extension. To activate the language, execute the generated statement in your database session to activate the container for the current session or system wide.\n\nThis command will print a SQL statement to activate the language similar to the following one:\n\n```bash\nALTER SESSION SET SCRIPT_LANGUAGES='<LANGUAGE_ALIAS>=localzmq+protobuf:///<bucketfs-name>/<bucket-name>/<path-in-bucket>/<container-name>?lang=<language>#buckets/<bucketfs-name>/<bucket-name>/<path-in-bucket>/<container-name>/exaudf/exaudfclient[_py3]';\n```\n\n**Please, refer to the User Guide for more detailed information, how to use exalsct.**\n\n## Features\n\n* Build a script language container as docker images\n* Export a script language container as an archive which can be used for extending Exasol UDFs\n* Upload a script language container as an archive to the Exasol DB's BucketFS\n* Generating the activation command for a script language container\n* Can use Docker registries, such as Docker Hub, as a cache to avoid rebuilding image without changes\n* Can push Docker images to Docker registries\n* Run tests for you container against an Exasol DB (docker-db or external db)\n\n## Limitations\n\n* Caution with symbolic links: \n  If you use symbolic links inside any directory of the command line arguments\n  they must not point to files or directories outside the root of the path of the \n  command line argument (i.e. --flavor-path ./flavors/my_flavor/ => There must be no symbolic\n  link inside ./flavors/my_flavor point to anywhere outside of ./flavors/my_flavor).\n  Background: Local directories paths must be mounted manually to the docker container. \n  We currently support only the mounting of the given command line arguments, but we do not analyze\n  the content of those directories.\n  Plan is to fix this limitation with [#35](https://github.com/exasol/script-languages-container-tool/issues/35)\n\n### MacOsX Limitations\n  \n* On MacOsX all arguments (flavors path, output directory, etc.) must point to locations within the current directory (background is that the MacOsX version does not support mount binding additional directories).\n\n## Table of Contents\n\n### Information for Users\n\n* [User Guide](doc/user_guide/user_guide.md)\n* [Changelog](doc/changes/changelog.md)\n\n## Information for Developers\n\n* [Developer Guide](doc/developer_guide/developer_guide.md)\n* [Dependencies](doc/dependencies.md)\n",
     'author': 'Torsten Kilias',

starter_scripts/checksums/exaslct_with_poetry.sh.sha512sum

@@ -1 +1 @@
-285eb790ad7675bd32ab883a81050578bbc263cd346d905192c7b639117592b9b12ba2ad3c32a74439add315ce455a99b8f54e24663f6e9aa486c00f7e7a2845  exaslct_with_poetry.sh
+603939626632a7a558e9014d4d8b7efecfedf540e7cb488373bfb776f8ac16d1935cd8416e0704458b5cd096801499bf7f3d20aa8c30297bd61276430f0d3acf  exaslct_with_poetry.sh

starter_scripts/checksums/exaslct_within_docker_container.sh.sha512sum

@@ -1 +1 @@
-26f584f6dc72b01fcf37e6666534580415c1bec73899f7fa2959acd6dc7def7b965a1649e580af7ab503d0a139c02dca1e23183332ac236e0ae0d0dd8029c53a  exaslct_within_docker_container.sh
+85e705ec636a4fb6bb9487f5378405e95173a184d43fe3fc42ed19eb12ed56a92ea614a2171aaff7650bd6645d7ef0d5cdfb86602d63d9e169671a1fe08aa8ea  exaslct_within_docker_container.sh

starter_scripts/checksums/exaslct_within_docker_container_slim.sh.sha512sum

@@ -1 +1 @@
-fbe0b3fafce7aa34eca0018d8cbfe0f61a57674065a3c270564e8334c94292af0faa6f3f628e278b7e0c4f79cf12deaeeeb114ae91e66065dfcdc712f1b9ba44  exaslct_within_docker_container_slim.sh
+979851c4a9f5376c1c758dead76ca3827fee1184f040a38fac6cb5033811d997c6292dbfe079042098e26a5c41f11b73d5c27acbe7cd2169e82fa10e92b0a6b1  exaslct_within_docker_container_slim.sh

starter_scripts/checksums/mount_point_parsing.sh.sha512sum

@@ -1 +1 @@
-9481d697fa3a9143a3571171c69b7b5bcb13902aebc83d879f34e874c1878b1e7d74d7217dc18ba77c7b502f753cb505f94afc141e5153458fc7b61a6ac28ee6  mount_point_parsing.sh
+3428c95db1c3d5e99d58be17e9987f9264a537cfcaf7bb0a245386c20e94e1e84a4c2b4566b62abf543e8a2faf69b36e51f965911a7beed0a0f0af9e282b27d9  mount_point_parsing.sh

starter_scripts/checksums/push_docker_runner_image.sh.sha512sum

@@ -1 +1 @@
-a779c2e4c65051d7baa865c24248bbd23faf4998d3d62638181930b81f2bbb209e2a26defbe473f203c15154c15eda5e2e44cfad0533fe8630d619dbbc38699b  push_docker_runner_image.sh
+68f6746d14673ebb413588cdf04ec6c71168aa5676766cee02297a7e859948e55ecf9b4f297bdbf57ee7eb7f11a0b7faef90935632891bcf5313f5474540250c  push_docker_runner_image.sh

starter_scripts/exaslct_with_poetry.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 
 
+
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
+#shellcheck source=./starter_scripts/poetry_utils.sh
 source "$SCRIPT_DIR/poetry_utils.sh"
 
 check_requirements

starter_scripts/exaslct_within_docker_container.sh

@@ -22,6 +22,8 @@ fi
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 declare -a mount_point_paths
+# Ignore shellcheck rules here as alternatives are worse.
+# shellcheck disable=SC2207
 mount_point_paths=($(bash "$SCRIPT_DIR"/mount_point_parsing.sh "${@}"))
 
 quoted_arguments=''
@@ -93,6 +95,8 @@ tmpfile_env=$(mktemp)
 trap 'rm -f -- "$tmpfile_env"' INT TERM HUP EXIT
 
 create_env_file_debug_protected "$tmpfile_env"
+# Ignore shellcheck rule as we need to split elements of array by space (they are in form "-v %MOUNT_POINT")
+# shellcheck disable=SC2068
 docker run --network host --env-file "$tmpfile_env" --rm $terminal_parameter -v "$PWD:$PWD" -v "$DOCKER_SOCKET_MOUNT" -w "$PWD" ${mount_point_parameter[@]} "$RUNNER_IMAGE_NAME" bash -c "$RUN_COMMAND"
 
 umask "$old_umask"

starter_scripts/exaslct_within_docker_container_slim.sh

@@ -18,6 +18,8 @@ else
 fi
 
 RED='\033[0;31m'
+#Ignore shellcheck rule, as using printf does not change the color in the terminal.
+#shellcheck disable=SC2059
 printf "${RED} Using slim version of exaslct. Please note that all input/output arguments (input flavor directories, output, log, etc.) needs to point to directories within the current directory.\n"
 
 rl=readlink
@@ -30,8 +32,6 @@ if [[ ! "$(command -v $rl)" ]]; then
   exit 1
 fi
 
-SCRIPT_DIR="$(dirname "$($rl -f "${BASH_SOURCE[0]}")")"
-
 quoted_arguments=''
 for argument in "${@}"; do
   argument="${argument//\\/\\\\}"

starter_scripts/mount_point_parsing.sh

@@ -124,6 +124,8 @@ function _format_parameters() {
 
 declare -a mount_point_paths
 formatted_params=$(_format_parameters "${@}")
+# Ignore shellcheck rule as we want to split parameters by space intentionally
+# shellcheck disable=SC2086
 _get_mount_point_paths $formatted_params
 
 echo "${mount_point_paths[@]}"

starter_scripts/push_docker_runner_image.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 if [ -n "${1-}" ]; then
   image_suffixes=("$@")
 else
-  image_suffixes="latest"
+  image_suffixes=("latest")
 fi
 
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"

starter_scripts/test/test_scripts.sh

@@ -15,15 +15,15 @@ function assert() {
   cmpB="${*}"
   if [[ $cmpA != "$cmpB" ]]; then
     >&2 echo "ERROR: '$cmpA' does not match'$cmpB'"
-    exit -1
+    exit 1
   fi
 }
 
 flavorDirA="$SCRIPT_DIR/test/abc=def"
 exportDirA="$SCRIPT_DIR/test/exportdir=xyz"
 
 mkdir "$flavorDirA" || true
-trap "rm -rf $flavorDirA $exportDirA" EXIT
+trap 'rm -rf "$flavorDirA" "$exportDirA"' EXIT
 
 testStr=$(bash "$SCRIPT_DIR/mount_point_parsing.sh" --flavor-path="$flavorDirA" --export-path "$exportDirA" dummy)