diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 000000000..a7822ad34 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,24 @@ +name: shellcheck + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + check: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt install shellcheck + - name: Lint sources with shellcheck + run: | + # Recursively find all shell scripts in the build-scripts directory with a shebang + grep -Erl '^(#!/(bin|usr/bin)/(env )?(sh|bash))' build-scripts/ | while read -r file; do + shellcheck --external-sources --source-path=build-scripts "$file" + done diff --git a/build-scripts/bootstrap-tarballs b/build-scripts/bootstrap-tarballs index 408a6627c..89da8dd75 100755 --- a/build-scripts/bootstrap-tarballs +++ b/build-scripts/bootstrap-tarballs @@ -112,9 +112,7 @@ run_and_print_on_failure ./configure echo "$(basename "$0"): Debug: Running make dist on masterfiles repository..." run_and_print_on_failure make dist # source tarball echo "$(basename "$0"): Debug: Running make tar-package on masterfiles repository..." -run_and_print_on_failure make tar-package # package tarball (containing all -# files as if they were installed -# under "prefix".) +run_and_print_on_failure make tar-package # package tarball (containing all files as if they were installed under "prefix".) mv cfengine-masterfiles*.tar.gz "$BASEDIR"/output/tarballs/ echo "$(basename "$0"): Debug: Running make distclean on masterfiles repository..." run_and_print_on_failure make distclean diff --git a/build-scripts/build-environment-check b/build-scripts/build-environment-check index 379b43dc0..490181f25 100755 --- a/build-scripts/build-environment-check +++ b/build-scripts/build-environment-check @@ -1,6 +1,6 @@ #!/bin/sh -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options @@ -45,17 +45,17 @@ DEP_LIST="$DEP_LIST rsync gcc make sudo wget" RET=0 for unwanted in $UNWANTED_DEPS; do - if query_pkg $unwanted; then - echo "ERROR Found unwanted package:" $unwanted + if query_pkg "$unwanted"; then + echo "ERROR Found unwanted package: $unwanted" RET=1 fi done for dep in $DEP_LIST; do - if query_pkg $dep; then + if query_pkg "$dep"; then true else - echo "ERROR Missing system package:" $dep + echo "ERROR Missing system package: $dep" RET=1 fi done diff --git a/build-scripts/clean-results b/build-scripts/clean-results index 69ccb4f81..d0d604179 100755 --- a/build-scripts/clean-results +++ b/build-scripts/clean-results @@ -1,6 +1,6 @@ #!/bin/sh -ex -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . version # This script is designed to clean up old directories, specifically keeping the diff --git a/build-scripts/compile b/build-scripts/compile index f3840fe6b..75746cf6a 100755 --- a/build-scripts/compile +++ b/build-scripts/compile @@ -34,24 +34,24 @@ nova) esac echo "$(basename "$0"): Debug: Running make in core repo..." -run_and_print_on_failure $MAKE -C "$BASEDIR"/core -k +run_and_print_on_failure "$MAKE" -C "$BASEDIR"/core -k echo "$(basename "$0"): Debug: Running make install in core repo..." -run_and_print_on_failure $MAKE -C "$BASEDIR"/core install DESTDIR="$BASEDIR"/cfengine/dist +run_and_print_on_failure "$MAKE" -C "$BASEDIR"/core install DESTDIR="$BASEDIR"/cfengine/dist if [ "$NOVA" = yes ]; then echo "$(basename "$0"): Debug: Running make in enterprise repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/enterprise -k + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/enterprise -k echo "$(basename "$0"): Debug: Running make install in enterprise repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/enterprise install DESTDIR="$BASEDIR"/cfengine/dist + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/enterprise install DESTDIR="$BASEDIR"/cfengine/dist if [ "$ROLE" = hub ]; then echo "$(basename "$0"): Debug: Running make in nova repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/nova -k + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/nova -k echo "$(basename "$0"): Debug: Running make install in nova repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/nova install DESTDIR="$BASEDIR"/cfengine/dist + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/nova install DESTDIR="$BASEDIR"/cfengine/dist echo "$(basename "$0"): Debug: Running make install in masterfiles repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/masterfiles install DESTDIR="$BASEDIR"/cfengine/dist + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/masterfiles install DESTDIR="$BASEDIR"/cfengine/dist fi else echo "$(basename "$0"): Debug: Running make install in masterfiles repo..." - run_and_print_on_failure $MAKE -C "$BASEDIR"/masterfiles install DESTDIR="$BASEDIR"/cfengine/dist + run_and_print_on_failure "$MAKE" -C "$BASEDIR"/masterfiles install DESTDIR="$BASEDIR"/cfengine/dist fi diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 227da1a0f..24629dc3b 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -5,8 +5,8 @@ # Autodect PROJECT if not set -if [ x"$PROJECT" = x ]; then - case x"$JOB_NAME" in +if [ -z "$PROJECT" ]; then + case "$JOB_NAME" in *-community-*) PROJECT=community ;; *-enterprise-*) PROJECT=nova ;; *-hub-*) PROJECT=nova ;; @@ -17,8 +17,8 @@ fi # If still not set, then either we are running outside Jenkins, or this # is not a main "build" type job (it could be the bootstrap job). # Do directory-based auto-detection. -if [ x"$PROJECT" = x ]; then - if [ -d $BASEDIR/nova ]; then +if [ -z "$PROJECT" ]; then + if [ -d "$BASEDIR"/nova ]; then PROJECT=nova else PROJECT=community @@ -31,7 +31,7 @@ export PROJECT # When running manually, you can just export this variable. # It's a flag: if it's set to 1 - then we use system OpenSSL. # Otherwise, we build it. -if [ x"$SYSTEM_SSL" = x ]; then +if [ -z "$SYSTEM_SSL" ]; then # We don't bundle OpenSSL on some redhat-derived systems due to incompatability with libpam and our openssl. _OS_MAJOR_VERSION="$(echo "$OS_VERSION" | cut -d. -f1)" if [ "$OS" = "rhel" ] && expr "$_OS_MAJOR_VERSION" ">=" "8" >/dev/null; then @@ -43,6 +43,9 @@ if [ x"$SYSTEM_SSL" = x ]; then fi fi # Detect using system ssl when running a Jenkins job + # shellcheck disable=SC2154 + # > label is referenced but not assigned. + # This file is sourced by other scripts. label is assigned elsewhere. if expr x"$label" ":" ".*systemssl" >/dev/null; then SYSTEM_SSL=1 fi @@ -82,7 +85,7 @@ solaris) esac # When we don't bundle OpenSSL, then we need to pull it from /usr/lib64. -if [ x"$SYSTEM_SSL" = x1 ]; then +if [ "$SYSTEM_SSL" = 1 ]; then LDFLAGS="$LDFLAGS -L/usr/lib64" fi export LDFLAGS @@ -96,8 +99,11 @@ EMBEDDED_DB="lmdb" ############### Fill in build dependencies in DEPS variable ################ +# shellcheck disable=SC2034 +# > DEPS appears unused. Verify use (or export if used externally). +# This file is sourced by other scripts that uses it DEPS= -[ $OS_FAMILY = mingw ] && var_append DEPS "pthreads-w32 libgnurx" +[ "$OS_FAMILY" = mingw ] && var_append DEPS "pthreads-w32 libgnurx" # libgcc_s.so is needed before we compile any other dependency # on some platforms! @@ -107,7 +113,7 @@ esac var_append DEPS "$EMBEDDED_DB pcre2" -if ! [ x"$SYSTEM_SSL" = x1 ]; then +if [ "$SYSTEM_SSL" != 1 ]; then # FIXME: Why do we need zlib? # ANSWER: Openssl uses it optionally, TODO DISABLE var_append DEPS "zlib openssl" @@ -156,7 +162,7 @@ agent) ROLE=agent ;; hub) ROLE=hub ;; *) # Not running under Jenkins? - if [ x"$JENKINS_SERVER_COOKIE" = x ]; then + if [ -z "$JENKINS_SERVER_COOKIE" ]; then case "$PROJECT-$ARCH-$OS-${OS_VERSION}" in community-*) ROLE=agent ;; # We do not support 32 bits hubs anymore @@ -220,7 +226,12 @@ esac # unit files for it? case "$OS_FAMILY" in linux) WITH_SYSTEMD=yes ;; -*) WITH_SYSTEMD=no ;; +*) + # shellcheck disable=SC2034 + # > DEPS appears unused. Verify use (or export if used externally). + # This file is sourced by other scripts that uses it. + WITH_SYSTEMD=no + ;; esac case "$OS_FAMILY" in diff --git a/build-scripts/detect-environment b/build-scripts/detect-environment index 43c08fa11..2dea044c6 100644 --- a/build-scripts/detect-environment +++ b/build-scripts/detect-environment @@ -17,6 +17,9 @@ fi # compilation targets. However, if CROSS_TARGET is already set in the # environment it will take precedence. detect_cross_target() { + # shellcheck disable=SC2154 + # > label is referenced but not assigned. + # This file is sourced by other scripts. label is assigned elsewhere. case "$label" in *_x86_64_mingw*) CROSS_TARGET=${CROSS_TARGET:-x64-mingw} diff --git a/build-scripts/functions b/build-scripts/functions index 6e120b1f2..28f146141 100644 --- a/build-scripts/functions +++ b/build-scripts/functions @@ -22,6 +22,7 @@ try_exec() { broken_posix_shell() { unset foo + # shellcheck disable=SC3043 local foo=1 test "$foo" != "1" } @@ -137,34 +138,34 @@ grep_q() { # uninstall_rpms() { - PKGS=$(rpm -qa --queryformat "%{Name}-%{Version}\n" | grep '^'$1 || true) + PKGS=$(rpm -qa --queryformat "%{Name}-%{Version}\n" | grep "^$1" || true) if [ -n "$PKGS" ]; then - retry_wrapper sudo rpm -e $PKGS + retry_wrapper sudo rpm -e "$PKGS" fi } uninstall_debs() { - PKGS=$(dpkg -l | tail -n+6 | awk '{print $2}' | grep '^'$1 || true) + PKGS=$(dpkg -l | tail -n+6 | awk '{print $2}' | grep "^$1" || true) if [ -n "$PKGS" ]; then - retry_wrapper sudo dpkg --purge $PKGS + retry_wrapper sudo dpkg --purge "$PKGS" fi } uninstall_solaris_pkgs() { - PKGS=$(pkginfo | awk '{print $2}' | grep '^'$1'$' || true) + PKGS=$(pkginfo | awk '{print $2}' | grep "^$1$" || true) if [ -n "$PKGS" ]; then - retry_wrapper sudo /usr/sbin/pkgrm -n $PKGS + retry_wrapper sudo /usr/sbin/pkgrm -n "$PKGS" fi } uninstall_hpux_pkgs() { - PKGS=$(swlist | awk '{print $1}' | grep '^'$1'$' || true) + PKGS=$(swlist | awk '{print $1}' | grep "^$1$" || true) for p in $PKGS; do - sudo /usr/sbin/swremove $p + sudo /usr/sbin/swremove "$p" done } uninstall_freebsd_pkgs() { - PKGS=$(pkg_info | awk '{print $1}' | grep '^'$1 || true) + PKGS=$(pkg_info | awk '{print $1}' | grep "^$1" || true) if [ -n "$PKGS" ]; then - retry_wrapper sudo pkg_delete $PKGS + retry_wrapper sudo pkg_delete "$PKGS" fi } @@ -197,7 +198,7 @@ uninstall_cfbuild_devel() { query_pkg() { case "$DEP_PACKAGING" in rpm) rpm -qa --provides | grep_q "^$1 " ;; - deb) dpkg -s $1 2>&1 | grep_q '^Status: .*ok installed' ;; + deb) dpkg -s "$1" 2>&1 | grep_q '^Status: .*ok installed' ;; *) echo "ERROR query_pkg not implemented for $DEP_PACKAGING" exit 1 @@ -267,7 +268,7 @@ remote_script_general() { ENVVARS="$ENVVARS TEST_MACHINE=$TEST_MACHINE" ENVVARS="$ENVVARS TEST_SHELL=$TEST_SHELL" - (eval $LOGIN_COMMAND env $ENVVARS "$SCRIPT_BASEDIR"/buildscripts/build-scripts/"$SCRIPT") + (eval "$LOGIN_COMMAND" env "$ENVVARS" "$SCRIPT_BASEDIR"/buildscripts/build-scripts/"$SCRIPT") } remote_script() { @@ -276,10 +277,10 @@ remote_script() { } projects_to_test() { - if test "x$PROJECT" = "xcommunity"; then + if test "$PROJECT" = "community"; then echo "core masterfiles" else - if test "x$ROLE" = "xhub"; then + if test "$ROLE" = "hub"; then echo "core enterprise nova masterfiles" else echo "core enterprise masterfiles" @@ -390,17 +391,17 @@ mount_procfs() { # fatal "exiting with default error code" fatal() { echo "$(basename "$0"): FATAL ERROR: $1" >&2 - exit ${2-1} + exit "${2-1}" } log_info() { - echo "INFO: $@" 1>&2 + echo "INFO:" "$@" 1>&2 } # Append a space and string $2 to variable $1, for example: # func_append V "blah" is equivalent to V="$V blah". var_append() { - eval $1=\$$1\\ \$2 + eval "$1=\$$1"\\ \$2 } # Return True if string $2 exists in variable $1. @@ -450,27 +451,34 @@ func_whereis() { # Only works as mktemp -d /path/to/tmpdir.XXXX func_mktemp() { # Only works as mktemp -d - [ x$1 != x-d ] && fatal "func_mktemp: error, first argument must be -d" - [ x$2 = x ] && fatal "func_mktemp: requires two arguments" + [ "$1" != -d ] && fatal "func_mktemp: error, first argument must be -d" + [ "$2" = "" ] && fatal "func_mktemp: requires two arguments" # $RANDOM does not exist on Solaris 9 /bin/sh, use $$ as fallback - my_tmpdir=$(echo $2 | sed 's/XX*/'${RANDOM-$$}/) + + # shellcheck disable=SC3028 + # > In POSIX sh, RANDOM is undefined. + # That's why we fallback to $$. + + my_tmpdir=$(echo "$2" | sed 's/XX*/'${RANDOM-$$}/) save_mktemp_umask=$(umask) umask 0077 # Set -e will cause this to fail if it already exists mkdir "$my_tmpdir" - umask $save_mktemp_umask + umask "$save_mktemp_umask" - [ -d "$my_tmpdir" ] && - echo "$my_tmpdir" || + if [ -d "$my_tmpdir" ]; then + echo "$my_tmpdir" + else fatal "func_mktemp: failed creating temporary directory $my_tmpdir" + fi } # mktempdir TEMPLATE # Example: mktempdir /tmp/dir.XXXXXX mktempdir() { - [ x$1 = x ] && fatal "mktempdir: TEMPLATE directory argument missing" + [ "$1" = "" ] && fatal "mktempdir: TEMPLATE directory argument missing" # HP-UX has its own non-POSIX mktemp, so override it. # If not on HP-UX, search PATH for the 'mktemp' or 'gmktemp' command. @@ -480,7 +488,7 @@ mktempdir() { my_mktemp=$(func_which mktemp gmktemp) || my_mktemp=func_mktemp - $my_mktemp -d $1 + $my_mktemp -d "$1" } # Print the sha256sum of $1 or stdin. @@ -502,7 +510,7 @@ func_decompress() { *.gz | *.tgz) gzip -dc "$@" ;; *.bz2) bzip2 -dc "$@" ;; - *) fatal "Unknown compression for file: $@" ;; + *) fatal "Unknown compression for file:" "$@" ;; esac } @@ -510,7 +518,7 @@ func_decompress() { # retrying in case of failure. # WARNING: do not pipe the output as stdout is altered! retry_wrapper() { - operation="$@" + operation="$*" [ "$operation" = "" ] && fatal "retry_wrapper: no arguments" # SLEEP: which sleep program to use @@ -520,7 +528,7 @@ retry_wrapper() { maxtries=5 pause=30 - while [ $maxtries != 0 ]; do + while [ "$maxtries" != 0 ]; do echo "* retry_wrapper: $operation" if $operation; then @@ -529,8 +537,9 @@ retry_wrapper() { else err_ret=$? # in case say dpkg locks are held by automatic updates or something + # shellcheck disable=SC2009 ps -efl | grep -P '(apt|dpkg|yum|dnf|zypper|rpm|pkg)' - maxtries=$(expr $maxtries - 1) + maxtries=$((maxtries - 1)) echo "* FAILURE $err_ret" echo "* Sleeping for: $pause seconds" echo "* Retries left: $maxtries" @@ -538,7 +547,7 @@ retry_wrapper() { fi done - return $err_ret + return "$err_ret" } rm_if_empty() { @@ -551,6 +560,7 @@ rm_if_empty() { # Use this function on verbose commands to silence the output unless it returns # a non-zero exit code run_and_print_on_failure() { + # shellcheck disable=SC3043 local temp_output_file if command -v mktemp >/dev/null; then temp_output_file=$(mktemp) @@ -568,6 +578,7 @@ run_and_print_on_failure() { touch "$temp_output_file" fi + # shellcheck disable=SC3043 local exit_code=0 if "$@" >"$temp_output_file" 2>&1; then : # NOOP diff --git a/build-scripts/install-dependencies b/build-scripts/install-dependencies index 18481aa0f..dbbda5806 100755 --- a/build-scripts/install-dependencies +++ b/build-scripts/install-dependencies @@ -62,12 +62,12 @@ check_and_install_perl() { fi if ! $PERL -e 'use List::Util qw(pairs);'; then - log_debug $PERL has List::Util that does not export pairs. Needs to be at least version 1.29 for OpenSSL version 3.3.2. + log_debug "$PERL has List::Util that does not export pairs. Needs to be at least version 1.29 for OpenSSL version 3.3.2." PERL_OK="no" fi if [ "$PERL_OK" != "yes" ]; then - log_debug $PERL is too old or modules are missing, building new one from source... + log_debug "$PERL is too old or modules are missing, building new one from source..." # -fno-stack-protector: Ensure built perl will not depend on libssp.so if echo | gcc -E -fno-stack-protector - >/dev/null 2>&1; then @@ -110,10 +110,10 @@ check_and_install_perl() { "$PERL_EXTRA_FLAGS" "$PERL_CFLAGS" "$PERL_LDFLAGS" "$PERL_LDDLFLAGS" log_debug Running make on perl project... - run_and_print_on_failure $MAKE + run_and_print_on_failure "$MAKE" log_debug Running make install on perl project... - run_and_print_on_failure $MAKE install + run_and_print_on_failure "$MAKE" install PERL=$HOME/perl-my/bin/perl fi @@ -182,7 +182,7 @@ for dep in $DEPS; do # If package cache did not have the packages if [ -z "$package_files" ]; then - log_debug Building package $dep... + log_debug "Building package $dep..." # BUILD the packages if [ "$dep" = "lmdb" ] && [ "$BUILD_TYPE" != "RELEASE" ]; then # Always build LMDB with debug symbols in non-release builds. @@ -204,12 +204,12 @@ for dep in $DEPS; do # CACHE the newly built package # shellcheck disable=SC2086 log_debug Caching package $dep... - pkg-cache putpkg "$dep-$version" $package_files + pkg-cache putpkg "$dep-$version" "$package_files" # CLEAN UP # shellcheck disable=SC2086 log_debug Removing package files for $dep... - rm -f $package_files + rm -f "$package_files" # Now the package should be in the local cache package_files=$(pkg-cache listpkgfiles "$dep-$version") @@ -221,7 +221,7 @@ for dep in $DEPS; do # INSTALL the packages # shellcheck disable=SC2086 log_debug Installing package files for $dep... - install_pkg_files $package_files + install_pkg_files "$package_files" # keep 50 most recent packages to preserve disk space pkg-cache keep_newest 50 diff --git a/build-scripts/package b/build-scripts/package index 797f4989a..4c3c0baa9 100755 --- a/build-scripts/package +++ b/build-scripts/package @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options . version @@ -55,12 +55,12 @@ P="$BASEDIR/buildscripts/packaging/$PKG" ( if [ "$PROJECT-$ROLE" = "nova-hub" ]; then if test -f "$BASEDIR/mission-portal/vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php"; then - cd $BASEDIR/mission-portal + cd "$BASEDIR"/mission-portal # Add Red Hat Text font to TCPDF library that we use in Mission Portal for PDF generation - $PREFIX/httpd/php/bin/php --version # diagnostic for ENT-12777, keep for future reference - $PREFIX/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Regular.ttf - $PREFIX/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Bold.ttf - $PREFIX/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Italic.ttf + "$PREFIX"/httpd/php/bin/php --version # diagnostic for ENT-12777, keep for future reference + "$PREFIX"/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Regular.ttf + "$PREFIX"/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Bold.ttf + "$PREFIX"/httpd/php/bin/php ./vendor/tecnickcom/tcpdf/tools/tcpdf_addfont.php -i ./public/themes/default/bootstrap/cfengine/font/rht/RedHatText-Italic.ttf fi fi ) @@ -68,10 +68,10 @@ P="$BASEDIR/buildscripts/packaging/$PKG" if [ "$BUILDPREFIX" != "/var/cfengine" ]; then safe_prefix="$(echo "$BUILDPREFIX" | sed -e 's:/::g')" file_to_patch=$P/../common/script-templates/$PACKAGING-script-common.sh - if [ -f $file_to_patch ]; then + if [ -f "$file_to_patch" ]; then # replace PREFIX in *-script-common.sh file - sed "s:/var/cfengine:$BUILDPREFIX:" $file_to_patch >$file_to_patch.new - mv $file_to_patch.new $file_to_patch + sed "s:/var/cfengine:$BUILDPREFIX:" "$file_to_patch" >"$file_to_patch".new + mv "$file_to_patch".new "$file_to_patch" fi fi @@ -161,10 +161,12 @@ rpm | lpp) -e "/^%postun\$/r $POSTREMOVE" \ "$SPECIN" >"$SPEC" + # shellcheck disable=SC2044 + # TODO: CFE-4584 for i in $(find "$BASEDIR/buildscripts/packaging/$PKG" ! -name "*.spec"); do ( cd "$BASEDIR/$PKG/SOURCES" - ln -sf "$i" + ln -sf "$i" . ) || false done @@ -178,7 +180,7 @@ rpm | lpp) --define "'_topdir $BASEDIR/$PKG'" \ --define "'buildprefix $BUILDPREFIX'" \ --define "'_basedir $BASEDIR'" \ - $RPMBUILD_OPTIONS "$SPEC" + "$RPMBUILD_OPTIONS" "$SPEC" if [ "$PACKAGING" = lpp ]; then # Create AIX bff packages @@ -195,8 +197,12 @@ rpm | lpp) [ -d BUILDROOT ] && cd BUILDROOT # test that there are 0 or 1 $PKG-* dirs # note that * must be unquoted! - [ "$(ls -1d "$PKG"-* | wc -l)" -lt 2 ] || exit 1 - [ -d "$PKG"-* ] && cd "$PKG"-* + for p in "$PKG"-*; do + if [ -d "$p" ]; then + # If there are more $PKG-* dirs, then the next cd will (most likely) fail + cd "$p" || fatal "More than 1 $PKG-* dirs" + fi + done # $LOCAL_PREFIX is $PREFIX without first slash, i.e. var/cfengine # $LOCAL_DIR is first (outermost) dir in $LOCAL_PREFIX, i.e. var LOCAL_PREFIX="$(echo "$PREFIX" | sed 's_^/__')" @@ -284,15 +290,21 @@ solaris) cat "$P/solaris/prototype.head" # Replace last two words with "root root" and filter out directories that # aren't ours. - sed -e 's/^\([fd].* \)[^ ][^ ]* *[^ ][^ ]*$/\1root root/' prototype.tmp | egrep "^([^d]|d none $PREFIX)" + sed -e 's/^\([fd].* \)[^ ][^ ]* *[^ ][^ ]*$/\1root root/' prototype.tmp | grep -E "^([^d]|d none $PREFIX)" ) >prototype ARCH="$(uname -p)" sed -e "s/@@PKG@@/$PKG/g;s/@@ARCH@@/$ARCH/g;s/@@VERSION@@/$VERSION$safe_prefix/g" "$P/solaris/pkginfo.in" >"$BASEDIR/$PKG/pkg/pkginfo" + # shellcheck disable=SC2094 + # > Make sure not to read and write the same file in the same pipeline. + # The second argument is just treated as a "type" argument, the file is not read from. "$P/../common/produce-script" "$PKG" preinstall pkg >preinstall + # shellcheck disable=SC2094 "$P/../common/produce-script" "$PKG" postinstall pkg >postinstall + # shellcheck disable=SC2094 "$P/../common/produce-script" "$PKG" preremove pkg >preremove + # shellcheck disable=SC2094 "$P/../common/produce-script" "$PKG" postremove pkg >postremove pkgmk -o -r "$(pwd)" -d "$BASEDIR/$PKG/pkg" @@ -316,9 +328,9 @@ freebsd) echo "@comment pkg-plist,v 1.00 $(date)" >>pkg-plist echo "@comment ORIGIN:sysutils/cfengine-nova" >>pkg-plist pkgdir="$BASEDIR/$PKG/pkg/" - for f in $(find "$BASEDIR/$PKG/pkg/" | egrep -v 'pkg-comment|pkg-descr|pkg-plist.foot|pkg-plist'); do + for f in $(find "$BASEDIR/$PKG/pkg/" | grep -Ev 'pkg-comment|pkg-descr|pkg-plist.foot|pkg-plist'); do - destf="${f#$pkgdir}" + destf="${f#"$pkgdir"}" if [ -f "$destf" ]; then echo "$destf" >>pkg-plist fi @@ -340,7 +352,7 @@ freebsd) cat tmp >>CONTENTS rm plist-head tmp mv CONTENTS ./+CONTENTS - tar cjvf "cfengine-nova-$VERSION\_1.tbz" +CONTENTS +DESC +COMMENT * + tar cjvf "cfengine-nova-$VERSION\_1.tbz" +CONTENTS +DESC +COMMENT -- * ;; hpux) ARCH="$UNAME_M" diff --git a/build-scripts/package-msi b/build-scripts/package-msi index c3cc976bc..e11d1f261 100755 --- a/build-scripts/package-msi +++ b/build-scripts/package-msi @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options . version @@ -50,35 +50,35 @@ fi CANDLE="wine $WIXPATH/candle.exe" LIGHT="wine $WIXPATH/light.exe" -if [ -z $JOB_NAME ]; then +if [ -z "$JOB_NAME" ]; then DIRNAME=build-$VERSION-$ARCH else - DIRNAME=$(echo ${JOB_NAME} | sed 's/\(.*\)\/.*/\1/g') + DIRNAME=$(echo "${JOB_NAME}" | sed 's/\(.*\)\/.*/\1/g') fi PKGD=$BASEDIR/packaging/cfengine-nova/pkg P=$PKGD/$DIRNAME pre() { - rm -rf $PKGD - mkdir -p $P/bin - mkdir -p $P/ssl + rm -rf "$PKGD" + mkdir -p "$P"/bin + mkdir -p "$P"/ssl - cp -a $BUILDPREFIX/bin/* $P/bin - cp -a $BUILDPREFIX/ssl/* $P/ssl - cp -a $BASEDIR/cfengine/dist$BUILDPREFIX/bin/* $P/bin + cp -a "$BUILDPREFIX"/bin/* "$P"/bin + cp -a "$BUILDPREFIX"/ssl/* "$P"/ssl + cp -a "$BASEDIR"/cfengine/dist"$BUILDPREFIX"/bin/* "$P"/bin # TODO make not hard-coded paths for debian/ubuntu installations of mingw # I couldn't find an easy way to dynamically find the path to this dll so hard-coded # it is. :( case "$ARCH" in x86) - cp -a /usr/i686-w64-mingw32/lib/libwinpthread-1.dll $P/bin/libwinpthread-1.dll - cp -a $BASEDIR/enterprise/libcfenterprise/cf.events.i686.dll $P/bin/cf.events.dll + cp -a /usr/i686-w64-mingw32/lib/libwinpthread-1.dll "$P"/bin/libwinpthread-1.dll + cp -a "$BASEDIR"/enterprise/libcfenterprise/cf.events.i686.dll "$P"/bin/cf.events.dll ;; x64) - cp -a /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll $P/bin/libwinpthread-1.dll - cp -a $BASEDIR/enterprise/libcfenterprise/cf.events.x86_64.dll $P/bin/cf.events.dll + cp -a /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll "$P"/bin/libwinpthread-1.dll + cp -a "$BASEDIR"/enterprise/libcfenterprise/cf.events.x86_64.dll "$P"/bin/cf.events.dll ;; *) echo "Unknown architecture: $ARCH" @@ -86,20 +86,20 @@ pre() { ;; esac - cp $BASEDIR/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs $P + cp "$BASEDIR"/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs "$P" # OpenSSL libs have different names on x32 and x64 platforms: # on 32-bit platforms: libcrypto_1_1.dll and libssl_1_1.dll # on 64-bit platforms: libcrypto_1_1_x64.dll and libssl_1_1_x64.dll # So in cfengine-nova.wxs we have names for 64-bit platforms, # and on 32-bit platforms we remove the -x64 suffix. if [ "$ARCH" = "x86" ]; then - sed -i '/lib\(crypto\|ssl\)/s/[_-]x64//g' $P/cfengine-nova.wxs + sed -i '/lib\(crypto\|ssl\)/s/[_-]x64//g' "$P"/cfengine-nova.wxs fi } candle() { REVISION="$1" - $CANDLE -dCfSourceDir=. -dCfVersion=$REVISION -dCfArch=$ARCH cfengine-nova.wxs + $CANDLE -dCfSourceDir=. -dCfVersion="$REVISION" -dCfArch="$ARCH" cfengine-nova.wxs } light() { @@ -107,13 +107,13 @@ light() { } package() { - cd $P + cd "$P" - if [ -z $EXPLICIT_VERSION ]; then + if [ -z "$EXPLICIT_VERSION" ]; then # First make sure VERSION does not contain a 4th dot, since on windows we'll # add one later (plus they can only be numeric); so convert # 3.10.0a.abcdef to 3.10.0a. - VERSION=$(echo $VERSION | sed -e 's/\([^.]*\.[^.]*\.[^.]*\).*/\1/') + VERSION=$(echo "$VERSION" | sed -e 's/\([^.]*\.[^.]*\.[^.]*\).*/\1/') case $BUILD_TYPE in RELEASE) @@ -138,20 +138,20 @@ package() { # convert any alphabets in revison to int as no alphabets are allowed in windows package version while true; do - alphabet=$(echo $REVISION | sed -e 's/[^a-zA-Z]*\([a-zA-Z]*\).*/\1/') + alphabet=$(echo "$REVISION" | sed -e 's/[^a-zA-Z]*\([a-zA-Z]*\).*/\1/') if [ -n "$alphabet" ]; then convert=$(printf '%d' "'$alphabet") - REVISION=$(echo $REVISION | sed -e "s/$alphabet/$convert/") + REVISION=$(echo "$REVISION" | sed -e "s/$alphabet/$convert/") else break fi done # ensure all revision components are 4 digits max (MSI allows each part to be up to 65534, but we limit it to 9999) - REVISION=$(echo $REVISION | tr '.' '\n' | sed 's/\(....\).*/\1/' | tr '\n' '.' | sed 's/\.$//') + REVISION=$(echo "$REVISION" | tr '.' '\n' | sed 's/\(....\).*/\1/' | tr '\n' '.' | sed 's/\.$//') # ensure revision has exactly 4 components - add .0.0.0 at the end and cut unneded parts - REVISION=$(echo $REVISION.0.0.0 | cut -d '.' -f 1-4) + REVISION=$(echo "$REVISION".0.0.0 | cut -d '.' -f 1-4) # Wix tools have a ridiculously short maximum file length of 128 characters. # This is easily exceeded when Jenkins workspace paths are involved. Luckily @@ -163,9 +163,9 @@ package() { # more likely to find free letters at the bottom end, so count backwards. WORKSPACE_DRIVE= for letter in y x w v u t s r q p o n m l k j i h g f e d; do - if [ ! -e $HOME/.wine/dosdevices/$letter: ]; then + if [ ! -e "$HOME"/.wine/dosdevices/$letter: ]; then WORKSPACE_DRIVE=$letter: - ln -s $PWD $HOME/.wine/dosdevices/$WORKSPACE_DRIVE + ln -s "$PWD" "$HOME"/.wine/dosdevices/$WORKSPACE_DRIVE break fi done @@ -180,15 +180,15 @@ package() { # Make sure the drive letter from above is cleaned up in case this build slave # is reused. - rm -f $HOME/.wine/dosdevices/$WORKSPACE_DRIVE + rm -f "$HOME"/.wine/dosdevices/$WORKSPACE_DRIVE return $ret } post() { - mkdir -p $BASEDIR/cfengine-nova - mv $P/cfengine-nova.msi $BASEDIR/cfengine-nova - cd $BASEDIR/cfengine-nova + mkdir -p "$BASEDIR"/cfengine-nova + mv "$P"/cfengine-nova.msi "$BASEDIR"/cfengine-nova + cd "$BASEDIR"/cfengine-nova # For some reason candle is giving file no permissions at all. chmod 644 cfengine-nova.msi @@ -204,10 +204,10 @@ post() { case $ARCH in x86) - mv cfengine-nova.msi $PKGNAME-i686.msi + mv cfengine-nova.msi "$PKGNAME-i686.msi" ;; x64) - mv cfengine-nova.msi $PKGNAME-x86_64.msi + mv cfengine-nova.msi "$PKGNAME-x86_64.msi" ;; esac } diff --git a/build-scripts/prepare-results b/build-scripts/prepare-results index cc4bb7743..d68ed3d05 100755 --- a/build-scripts/prepare-results +++ b/build-scripts/prepare-results @@ -1,6 +1,6 @@ #!/bin/sh -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options . version @@ -10,31 +10,31 @@ set -x mkdir -p "$OUTDIR" for dir in cfengine-community cfengine-nova cfengine-nova-hub; do - if [ -d $BASEDIR/$dir ]; then + if [ -d "$BASEDIR"/$dir ]; then case "$PACKAGING" in rpm) - cp $BASEDIR/$dir/RPMS/*/*.rpm "$OUTDIR" - cp $BASEDIR/$dir/RPMS/*.pkg.tar.gz "$OUTDIR" + cp "$BASEDIR"/$dir/RPMS/*/*.rpm "$OUTDIR" + cp "$BASEDIR"/$dir/RPMS/*.pkg.tar.gz "$OUTDIR" ;; deb) - cp $BASEDIR/$dir/*.deb "$OUTDIR" - cp $BASEDIR/$dir/pkg/*.pkg.tar.gz "$OUTDIR" + cp "$BASEDIR"/$dir/*.deb "$OUTDIR" + cp "$BASEDIR"/$dir/pkg/*.pkg.tar.gz "$OUTDIR" ;; msi) - cp $BASEDIR/$dir/*.msi "$OUTDIR" + cp "$BASEDIR"/$dir/*.msi "$OUTDIR" ;; solaris) - cp $BASEDIR/$dir/*.pkg "$OUTDIR" + cp "$BASEDIR"/$dir/*.pkg "$OUTDIR" ;; freebsd) cp build/$dir/pkg/*.tbz "$OUTDIR" ;; hpux) - cp $BASEDIR/$dir/pkg/*.depot "$OUTDIR" + cp "$BASEDIR"/$dir/pkg/*.depot "$OUTDIR" ;; lpp) - cp $BASEDIR/$dir/RPMS/*/*.rpm "$OUTDIR" - cp $HOME/lppdir/out/*.bff "$OUTDIR" + cp "$BASEDIR"/$dir/RPMS/*/*.rpm "$OUTDIR" + cp "$HOME"/lppdir/out/*.bff "$OUTDIR" ;; *) echo "Unknown packaging system: $PACKAGING" @@ -46,9 +46,9 @@ for dir in cfengine-community cfengine-nova cfengine-nova-hub; do done for project in $(projects_to_test); do - if [ -f $BASEDIR/$project/tests/acceptance/summary.log ]; then - cp $BASEDIR/$project/tests/acceptance/summary.log "$OUTDIR"/$project-summary.log - cp $BASEDIR/$project/tests/acceptance/test.log "$OUTDIR"/$project-test.log - cp $BASEDIR/$project/tests/acceptance/test.xml "$OUTDIR"/$project-test.xml + if [ -f "$BASEDIR"/"$project"/tests/acceptance/summary.log ]; then + cp "$BASEDIR"/"$project"/tests/acceptance/summary.log "$OUTDIR"/"$project"-summary.log + cp "$BASEDIR"/"$project"/tests/acceptance/test.log "$OUTDIR"/"$project"-test.log + cp "$BASEDIR"/"$project"/tests/acceptance/test.xml "$OUTDIR"/"$project"-test.xml fi done diff --git a/build-scripts/prepare-testmachine b/build-scripts/prepare-testmachine index 3e450ffa9..40fa47e00 100755 --- a/build-scripts/prepare-testmachine +++ b/build-scripts/prepare-testmachine @@ -1,12 +1,12 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options case "$TEST_MACHINE" in chroot) - $BASEDIR/buildscripts/build-scripts/prepare-testmachine-chroot + "$BASEDIR"/buildscripts/build-scripts/prepare-testmachine-chroot ;; *) # Add VMs, etc here. diff --git a/build-scripts/produce-debug-symbols b/build-scripts/produce-debug-symbols index 18d0d9b5c..8c0a632b4 100755 --- a/build-scripts/produce-debug-symbols +++ b/build-scripts/produce-debug-symbols @@ -1,10 +1,10 @@ #!/bin/sh -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options -if [ $BUILD_TYPE != "RELEASE" ]; then +if [ "$BUILD_TYPE" != "RELEASE" ]; then echo "Debug symbols not requested" exit 0 else @@ -12,9 +12,9 @@ else fi EXECUTABLES_DIR="$BASEDIR/cfengine/dist$PREFIX/bin" -EXECUTABLES_LIST=$(ls $EXECUTABLES_DIR/cf-*) +EXECUTABLES_LIST=$(ls "$EXECUTABLES_DIR"/cf-*) LIBRARIES_DIR="$BASEDIR/cfengine/dist$PREFIX/lib" -LIBRARIES_LIST=$(ls $LIBRARIES_DIR/*.so*) +LIBRARIES_LIST=$(ls "$LIBRARIES_DIR"/*.so*) DEBUG_SUFFIX=debug PAYLOAD_DIR=payload EXECUTABLES_PAYLOAD_DIR=$PAYLOAD_DIR/exe @@ -59,45 +59,51 @@ for i in $LIBRARIES_LIST; do done # Write the installation script -echo "Writing the installation script" -echo "#!/bin/sh" >$PAYLOAD_DIR/install-ds.sh -echo "# Install debug symbols for executables" -echo "for i in \`ls exe/*.gz\`; do" >>$PAYLOAD_DIR/install-ds.sh -echo " gunzip \$i" >>$PAYLOAD_DIR/install-ds.sh -echo "done" >>$PAYLOAD_DIR/install-ds.sh -echo "mv exe/*.$DEBUG_SUFFIX $PREFIX/bin" >>$PAYLOAD_DIR/install-ds.sh -echo "# Install debug symbols for libraries" -echo "for i in \`ls lib/*.gz\`; do" >>$PAYLOAD_DIR/install-ds.sh -echo " gunzip \$i" >>$PAYLOAD_DIR/install-ds.sh -echo "done" >>$PAYLOAD_DIR/install-ds.sh -echo "mv lib/*.$DEBUG_SUFFIX $PREFIX/lib" >>$PAYLOAD_DIR/install-ds.sh -echo "exit 0" >>$PAYLOAD_DIR/install-ds.sh +log_debug "Writing the installation script" +cat <> $PAYLOAD_DIR/install-ds.sh +#!/bin/sh +# Install debug symbols for executables +for i in \`ls exe/*.gz\`; do + gunzip \$i +done +mv exe/*.$DEBUG_SUFFIX $PREFIX/bin +# Install debug symbols for libraries +for i in \`ls lib/*.gz\`; do + gunzip \$i +done +mv lib/*.$DEBUG_SUFFIX $PREFIX/lib +exit 0 +EOF # Write the decompression script -echo "Writing the decompression script" -echo "#!/bin/sh" >decompress-ds.sh -echo "echo \"Installing debug symbols\"" >>decompress-ds.sh -echo "TMPDIR=\`mktemp -d /tmp/ds.XXXXXX\`; export TMPDIR" >>decompress-ds.sh -echo "ARCHIVE=\`awk '/^__ARCHIVE_BELOW__/ {print NR + 1; exit 0; }' \$0\`" >>decompress-ds.sh -echo "tail -n+\$ARCHIVE \$0 | tar xzv -C \$TMPDIR" >>decompress-ds.sh -echo "echo \"Installer ready - starting installation\"" -echo "CDIR=\`pwd\`" >>decompress-ds.sh -echo "cd \$TMPDIR/$PAYLOAD_DIR" >>decompress-ds.sh -echo "chmod 755 install-ds.sh" >>decompress-ds.sh -echo "chmod 755 uninstall-ds.sh" >>decompress-ds.sh -echo "./install-ds.sh" >>decompress-ds.sh -echo "cd \$CDIR" >>decompress-ds.sh +log_debug "Writing the decompression script" +cat <> decompress-ds.sh +#!/bin/sh +echo "Installing debug symbols" +TMPDIR=\`mktemp -d /tmp/ds.XXXXXX\`; export TMPDIR +ARCHIVE=\`awk '/^__ARCHIVE_BELOW__/ {print NR + 1; exit 0; }' \$0\` +tail -n+\$ARCHIVE \$0 | tar xzv -C \$TMPDIR +echo "Installer ready - starting installation" +CDIR=\`pwd\` +cd \$TMPDIR/$PAYLOAD_DIR +chmod 755 install-ds.sh +chmod 755 uninstall-ds.sh +./install-ds.sh +cd \$CDIR #echo "rm -rf \$TMPDIR" >> decompress-ds.sh -echo "exit 0" >>decompress-ds.sh -echo "__ARCHIVE_BELOW__" >>decompress-ds.sh +exit 0 +__ARCHIVE_BELOW__ +EOF # Write the uninstall script -echo "Writing the uninstall script" -echo "#!/bin/sh" >$PAYLOAD_DIR/uninstall-ds.sh -echo "echo \"Uninstalling debug symbols\"" >>$PAYLOAD_DIR/uninstall-ds.sh -echo "rm -f \"$PREFIX\"/bin/*.\$DEBUG_SUFFIX" >>$PAYLOAD_DIR/uninstall-ds.sh -echo "rm -f \"$PREFIX\"/lib/*.\$DEBUG_SUFFIX" >>$PAYLOAD_DIR/uninstall-ds.sh -echo "echo \"Done\"" >>$PAYLOAD_DIR/uninstall-ds.sh +log_debug "Writing the uninstall script" +cat <> $PAYLOAD_DIR/uninstall-ds.sh +#!/bin/sh +echo "Uninstalling debug symbols" +rm -f "$PREFIX"/bin/*.\$DEBUG_SUFFIX +rm -f "$PREFIX"/lib/*.\$DEBUG_SUFFIX +echo "Done" +EOF # Now create the self extracting script echo "Creating the self extracting script" @@ -105,7 +111,7 @@ tar cf payload.tar $PAYLOAD_DIR if [ -e "payload.tar" ]; then gzip payload.tar if [ -e "payload.tar.gz" ]; then - cat decompress-ds.sh payload.tar.gz >cfengine-ds-$OS-$OS_VERSION-$ARCH.bsx + cat decompress-ds.sh payload.tar.gz >cfengine-ds-"$OS"-"$OS_VERSION"-"$ARCH".bsx else echo "File not found" exit 1 @@ -114,7 +120,7 @@ if [ -e "payload.tar" ]; then fi # Put the script in a sensible place mkdir -p output/debug_symbols -mv *.bsx output/debug_symbols +mv -- *.bsx output/debug_symbols # Clean up after ourserlves rm -f decompress-ds.sh diff --git a/build-scripts/test b/build-scripts/test index f4d2d0d46..0f2078cfe 100755 --- a/build-scripts/test +++ b/build-scripts/test @@ -1,10 +1,10 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options -if [ "xtrue" = "x$NO_TESTS" ] || [ "xtrue" = "x$NO_ACCEPTANCE_TESTS" ]; then +if [ "true" = "$NO_TESTS" ] || [ "true" = "$NO_ACCEPTANCE_TESTS" ]; then create-empty-test exit 0 fi diff --git a/build-scripts/test-chroot b/build-scripts/test-chroot index b855b6ed8..4698632b3 100755 --- a/build-scripts/test-chroot +++ b/build-scripts/test-chroot @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options @@ -13,4 +13,4 @@ fi # We should already be root inside the chroot, so sudo is not necessary, # and "GAINROOT=env" might be faster. -env GAINROOT=env UNSAFE_TESTS=1 $BASEDIR/buildscripts/build-scripts/test-on-thismachine +env GAINROOT=env UNSAFE_TESTS=1 "$BASEDIR"/buildscripts/build-scripts/test-on-thismachine diff --git a/build-scripts/test-on-testmachine b/build-scripts/test-on-testmachine index 3256f7900..720833c35 100755 --- a/build-scripts/test-on-testmachine +++ b/build-scripts/test-on-testmachine @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options @@ -12,8 +12,8 @@ chroot) LOGIN_COMMAND="sudo chroot $CHROOT_ROOT /run-in-home-dir.sh" TESTMACHINE_URI=$CHROOT_ROOT - sudo umount ${CHROOT_ROOT}proc || true - mount_procfs ${CHROOT_ROOT}proc + sudo umount "${CHROOT_ROOT}"proc || true + mount_procfs "${CHROOT_ROOT}"proc ;; *) # Add VMs, etc here. @@ -40,10 +40,11 @@ case "$TEST_MACHINE" in chroot) # Fuser has special output. The PIDs arrive on stdout, and all the ornaments # arrive on stderr, so all we have to do is to grep for PID numbers. - for pid in $(sudo $FUSER $CHROOT_ROOT 2>/dev/null | sed -e 's/[^ 0-9]//g'); do + for pid in $(sudo "$FUSER" "$CHROOT_ROOT" 2>/dev/null | sed -e 's/[^ 0-9]//g'); do # Minilogd may sometimes launch on RedHat without any user # intervention. Ignore that. - if ps -ef | egrep "^ *[^ ]+ +$pid" | grep minilogd; then + # shellcheck disable=SC2009 + if ps -ef | grep -E "^ *[^ ]+ +$pid" | grep minilogd; then continue fi # gpg-agent sometimes is spawned on suse during our tests. @@ -67,13 +68,14 @@ chroot) echo "Error: Found processes left behind in the chroot. I'm killing them, but the build will fail, so clean up your mess!" echo "Found PID $pid. Info from 'ps -ef'" echo "--------------" - ps -ef | grep $pid | grep -v grep + # shellcheck disable=SC2009 + ps -ef | grep "$pid" | grep -v grep echo "--------------" ) done - sudo $FUSER -k $CHROOT_ROOT || true - sudo umount ${CHROOT_ROOT}proc || true + sudo "$FUSER" -k "$CHROOT_ROOT" || true + sudo umount "${CHROOT_ROOT}"proc || true ;; esac @@ -81,19 +83,19 @@ esac INCLUDES='--include=test.* --include=summary.*' # Note: Don't use sudo or "rsync -a", because we don't want root-owned files # to show up in the results. -rsync -rv $INCLUDES --exclude="*" \ - $TESTMACHINE_URI$BASEDIR/core/tests/acceptance/ \ - $BASEDIR/core/tests/acceptance/ \ +rsync -rv "$INCLUDES" --exclude="*" \ + "$TESTMACHINE_URI$BASEDIR"/core/tests/acceptance/ \ + "$BASEDIR"/core/tests/acceptance/ \ >>/tmp/rsync.log -if [ $PROJECT = nova ]; then - rsync -rv $INCLUDES --exclude="*" \ - $TESTMACHINE_URI$BASEDIR/enterprise/tests/acceptance/ \ - $BASEDIR/enterprise/tests/acceptance/ \ +if [ "$PROJECT" = nova ]; then + rsync -rv "$INCLUDES" --exclude="*" \ + "$TESTMACHINE_URI$BASEDIR"/enterprise/tests/acceptance/ \ + "$BASEDIR"/enterprise/tests/acceptance/ \ >>/tmp/rsync.log - rsync -rv $INCLUDES --exclude="*" \ - $TESTMACHINE_URI$BASEDIR/masterfiles/tests/acceptance/ \ - $BASEDIR/masterfiles/tests/acceptance/ \ + rsync -rv "$INCLUDES" --exclude="*" \ + "$TESTMACHINE_URI$BASEDIR"/masterfiles/tests/acceptance/ \ + "$BASEDIR"/masterfiles/tests/acceptance/ \ >>/tmp/rsync.log fi diff --git a/build-scripts/test-on-thismachine b/build-scripts/test-on-thismachine index b7c075074..9bb0d6bf8 100755 --- a/build-scripts/test-on-thismachine +++ b/build-scripts/test-on-thismachine @@ -1,13 +1,13 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options . version case "$OS" in mingw) - if [ "xno" != "x$ESCAPETEST" ]; then + if [ "no" != "$ESCAPETEST" ]; then create-empty-test else test-on-windows @@ -27,21 +27,21 @@ if [ "$TEST_SHELL" = "1" ]; then exit 1 else if [ "$TESTS" = all ]; then - chmod -R g-w $BASEDIR/core/tests/acceptance/* - chmod -R g-w $BASEDIR/masterfiles/tests/acceptance/* - chmod -R g-w $BASEDIR/masterfiles/lib/* - chmod -R g-w $BASEDIR/masterfiles/inventory/* - if test "x$PROJECT" != "xcommunity"; then - chmod -R g-w $BASEDIR/enterprise/tests/acceptance/* + chmod -R g-w "$BASEDIR"/core/tests/acceptance/* + chmod -R g-w "$BASEDIR"/masterfiles/tests/acceptance/* + chmod -R g-w "$BASEDIR"/masterfiles/lib/* + chmod -R g-w "$BASEDIR"/masterfiles/inventory/* + if [ "$PROJECT" != "community" ]; then + chmod -R g-w "$BASEDIR"/enterprise/tests/acceptance/* fi for project in $(projects_to_test); do # Unfortunately VERBOSE env variable is being used by both # "testall" script and automake. As a result, setting VERBOSE=1 # causes testall to pass "1" as an argument to cf-agent! # Workarount: VERBOSE=-I - NETWORK_TESTS=0 $MAKE -C $BASEDIR/$project VERBOSE=-I check + NETWORK_TESTS=0 $MAKE -C "$BASEDIR"/"$project" VERBOSE=-I check done elif [ "$TESTS" = unit ]; then - $MAKE -C $BASEDIR/core/tests/unit VERBOSE=-I check + $MAKE -C "$BASEDIR"/core/tests/unit VERBOSE=-I check fi fi diff --git a/build-scripts/test-on-windows b/build-scripts/test-on-windows index 15f2723d4..eea3dd96f 100755 --- a/build-scripts/test-on-windows +++ b/build-scripts/test-on-windows @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options . version @@ -13,7 +13,7 @@ execute_test() { } get_testresult() { - cd $BASEDIR/core/tests + cd "$BASEDIR"/core/tests echo "cd $TESTDIR/acceptance get test.log" | $SFTP "$1" @@ -24,15 +24,15 @@ get test.xml" | $SFTP "$1" prepare_bin prepare_tests -put $WIX_MACHINE +put "$WIX_MACHINE" case $ARCH in x86) - execute_test $WIX_MACHINE - try_run get_testresult $WIX_MACHINE + execute_test "$WIX_MACHINE" + try_run get_testresult "$WIX_MACHINE" ;; x64) - execute_test $WIX_MACHINE - try_run get_testresult $WIX_MACHINE + execute_test "$WIX_MACHINE" + try_run get_testresult "$WIX_MACHINE" ;; esac diff --git a/build-scripts/transfer-results b/build-scripts/transfer-results index eeb70c002..9b0c15656 100755 --- a/build-scripts/transfer-results +++ b/build-scripts/transfer-results @@ -1,15 +1,15 @@ #!/bin/sh -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . version set -x BUILDMACHINE="$1" -mkdir -p $BASEDIR/../../../output/${SCHEDULER}/${BUILD_NUMBER} +mkdir -p "$BASEDIR"/../../../output/"${SCHEDULER}"/"${BUILD_NUMBER}" rsync -avr --delete "$BUILDMACHINE:build/output/*" \ - $BASEDIR/../../../output/${SCHEDULER}/${BUILD_NUMBER} \ + "$BASEDIR"/../../../output/"${SCHEDULER}"/"${BUILD_NUMBER}" \ >/tmp/rsync.log ssh "$BUILDMACHINE" "rm -rf build/output" diff --git a/build-scripts/transfer-to-buildmachine b/build-scripts/transfer-to-buildmachine index a48c2d505..265186d7d 100755 --- a/build-scripts/transfer-to-buildmachine +++ b/build-scripts/transfer-to-buildmachine @@ -1,7 +1,7 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions BUILDMACHINE="$1" -rsync -acvr --exclude=output --exclude=".git" --delete $BASEDIR/ $BUILDMACHINE:build >rsync.log +rsync -acvr --exclude=output --exclude=".git" --delete "$BASEDIR"/ "$BUILDMACHINE":build >rsync.log diff --git a/build-scripts/transfer-to-testmachine b/build-scripts/transfer-to-testmachine index 0188b4637..e69e99066 100755 --- a/build-scripts/transfer-to-testmachine +++ b/build-scripts/transfer-to-testmachine @@ -1,6 +1,6 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options @@ -25,12 +25,22 @@ esac # This way we avoid errorneously replacing "/." in the # middle of BASEDIR like "/home/jenkins/.jenkinsdir/etc" # (Google Cloud Jenkins plugin likes to create them) -BASEDIR_NO_DOT="$(echo $BASEDIR | sed -e 's,/\./,/,g;s,/\.$,,')" +BASEDIR_NO_DOT="$(echo "$BASEDIR" | sed -e 's,/\./,/,g;s,/\.$,,')" touch .keepalive-echo (while test -e .keepalive-echo; do sleep 60 echo Keep alive done) & + +# shellcheck disable=SC2086 +# > Double quote to prevent globbing and word splitting. +# We want word splitting here + +# shellcheck disable=SC2024 +# > sudo doesn't affect redirects. Use ..| sudo tee file +# We don't care sudo rsync -avR $EXCLUDES --delete --delete-excluded "$BASEDIR_NO_DOT/" $TESTMACHINE_URI >/tmp/rsync.log +# shellcheck disable=SC2086 +# shellcheck disable=SC2024 sudo rsync -avR $EXCLUDES --delete --delete-excluded "$PREFIX/" $TESTMACHINE_URI >>/tmp/rsync.log rm .keepalive-echo diff --git a/build-scripts/transfer-to-windowsmachine b/build-scripts/transfer-to-windowsmachine index 39c506b2d..3e7b8eef0 100644 --- a/build-scripts/transfer-to-windowsmachine +++ b/build-scripts/transfer-to-windowsmachine @@ -9,17 +9,17 @@ SFTP="sftp -o BatchMode=yes -o StrictHostKeyChecking=no" SEVENZIP="\"c:/program files/7-zip/7z\"" HOMEPATH="\"c:\\Users\\jenkins\"" -if [ -z $JOB_NAME ]; then +if [ -z "$JOB_NAME" ]; then DIRNAME=build-$VERSION-$ARCH TESTDIR="\"c:\\Users\\jenkins\"" else - DIRNAME=$(echo ${JOB_NAME} | sed 's/\(.*\)\/.*/\1/g') + DIRNAME=$(echo "${JOB_NAME}" | sed 's/\(.*\)\/.*/\1/g') TESTDIR="\"c:\\Users\\jenkins\\$DIRNAME\\tests\"" fi MAX_TRY=50 try_run() { - for i in $(seq $MAX_TRY); do + for _ in $(seq $MAX_TRY); do if "$@"; then return fi @@ -30,33 +30,33 @@ try_run() { prepare_bin() { PKGD=$BASEDIR/packaging/cfengine-nova/pkg P=$PKGD/$DIRNAME - rm -rf $PKGD - mkdir -p $P/bin + rm -rf "$PKGD" + mkdir -p "$P"/bin - cp -a "$PREFIX/bin"/* $P/bin - cp -a "$BASEDIR/cfengine/dist$PREFIX/bin"/* $P/bin + cp -a "$PREFIX/bin"/* "$P"/bin + cp -a "$BASEDIR/cfengine/dist$PREFIX/bin"/* "$P"/bin case "$ARCH" in - x86) cp -a $BASEDIR/enterprise/libcfenterprise/cf.events.i686.dll $P/bin/cf.events.dll ;; - x64) cp -a $BASEDIR/enterprise/libcfenterprise/cf.events.x86_64.dll $P/bin/cf.events.dll ;; + x86) cp -a "$BASEDIR"/enterprise/libcfenterprise/cf.events.i686.dll "$P"/bin/cf.events.dll ;; + x64) cp -a "$BASEDIR"/enterprise/libcfenterprise/cf.events.x86_64.dll "$P"/bin/cf.events.dll ;; *) echo "Unknown architecture: $ARCH" exit 1 ;; esac - cp $BASEDIR/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs $P + cp "$BASEDIR"/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs "$P" ( - cd $PKGD - zip -r $DIRNAME.zip $DIRNAME + cd "$PKGD" + zip -r "$DIRNAME".zip "$DIRNAME" ) || false } prepare_tests() { TESTD=$BASEDIR/core/tests ( - cd $TESTD + cd "$TESTD" zip -r tests.zip . ) || false } @@ -75,7 +75,7 @@ post_put() { } put() { - try_run pre_put $1 - try_run put_zip $1 - try_run post_put $1 + try_run pre_put "$1" + try_run put_zip "$1" + try_run post_put "$1" } diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 0f9a6604f..8d7d737bf 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -1,11 +1,14 @@ #!/bin/sh -x -. $(dirname "$0")/functions +. "$(dirname "$0")"/functions . detect-environment . compile-options SOURCE_TARBALL="$BASEDIR/output/tarballs/cfengine-3.*.tar.gz" -MASTERFILES_TARBALL=$(ls $BASEDIR/output/tarballs/cfengine-masterfiles*.tar.gz | grep -v 'pkg.tar.gz$') +# shellcheck disable=SC2010 +# > Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. +# TODO: CFE-4587 +MASTERFILES_TARBALL=$(ls "$BASEDIR"/output/tarballs/cfengine-masterfiles*.tar.gz | grep -v 'pkg.tar.gz$') # DELETE the git-checked-out directories, they are tainted with # ./configure artifacts anyway. The tarballs are unpacked and symlinked @@ -23,17 +26,23 @@ if [ -e "$BASEDIR/core/" ] || [ -e "$BASEDIR/masterfiles/" ]; then exit 1 fi -if [ x$PROJECT = xcommunity ]; then +if [ "$PROJECT" = community ]; then sudo rm -rf "$BASEDIR/enterprise" "$BASEDIR/nova" "$BASEDIR/mission-portal" fi # NATIVE TAR is being used on purpose, and *not* GNU TAR. echo "UNPACKING SOURCE TARBALL AND SYMLINKING core" -cd $BASEDIR +cd "$BASEDIR" +# shellcheck disable=SC2086 +# > Double quote to prevent globbing and word splitting. +# We want globbing here gzip -dc $SOURCE_TARBALL | tar -xf - ln -s cfengine-3* core echo "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/" +# shellcheck disable=SC2086 +# > Double quote to prevent globbing and word splitting. +# We want globbing here gzip -dc $MASTERFILES_TARBALL | tar -xf - ln -s cfengine-masterfiles-* masterfiles diff --git a/user-scripts/check-scripts.sh b/user-scripts/check-scripts.sh new file mode 100755 index 000000000..66e93324a --- /dev/null +++ b/user-scripts/check-scripts.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +# +# Interactively lint & format shell scripts in the build-scripts directory. +# +# Dependencies: +# * shfmt +# * shellcheck +# +# This script takes no arguments can be executed from anywhere, e.g.: +# $ ./user-scripts/check-scripts.sh +# + +BUILD_SCRIPTS="$(dirname "$0")"/../build-scripts + +grep -Erl '^(#!/(bin|usr/bin)/(env )?(sh|bash))' "$BUILD_SCRIPTS" | sort | while read -r filepath; do + filename=$(basename "$filepath") + + if ! shfmt --diff --indent=4 "$filepath"; then + echo + echo "File '$filename' requires formatting." + read -r -p "Do you wish to format '$filename'? [y/N] " answer