Merge pull request #1017 from lukaszstolarczuk/fix-CI-scripts

Fix CI scripts
pmem · Jul 14, 2021 · 0f79db4 · 0f79db4
2 parents 181723f + e83ada0
commit 0f79db4
Show file tree

Hide file tree

Showing 13 changed files with 216 additions and 197 deletions.
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -1,5 +1,7 @@
 
 name: pmemkv-coverity
+# It runs static analysis build - Coverity. It requires special token (set in CI's secret).
+
 on:
   schedule:
     # run this job at 00:00 UTC everyday

diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml
@@ -1,5 +1,8 @@
 
 name: pmemkv
+# It runs default OSes for each PR, push event or a new tag,
+# checks basic builds with various compilers and executes all sets of tests.
+
 on:
   push:
   pull_request:

diff --git a/.github/workflows/other_OSes.yml → .github/workflows/nightly.yml b/.github/workflows/other_OSes.yml → .github/workflows/nightly.yml
@@ -1,5 +1,7 @@
 
-name: pmemkv-other
+name: pmemkv-nightly
+# It runs non-deafult OSes, listed below with basic set of compilers and tests.
+
 on:
   schedule:
     # run this job at 01:00 UTC everyday
@@ -11,6 +13,7 @@ env:
   CONTAINER_REG:  ghcr.io/pmem/pmemkv
   HOST_WORKDIR:   ${{ github.workspace }}
   WORKDIR:        utils/docker
+  TEST_TIMEOUT:   900
   IMG_VER:        latest
   TYPE:           debug
   PUSH_IMAGE:     1
@@ -20,7 +23,7 @@ env:
 
 jobs:
   linux:
-    name: Linux-other
+    name: Linux-nightly
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -32,19 +35,17 @@ jobs:
                  "OS=debian OS_VER=latest",
                  "OS=fedora OS_VER=33",
                  "OS=fedora OS_VER=34",
-                 "OS=fedora OS_VER=rawhide PUSH_IMAGE=0",
-                 "TYPE=building OS=fedora OS_VER=rawhide",
+                 "OS=fedora OS_VER=rawhide",
                  "OS=opensuse-leap OS_VER=latest",
                  "OS=opensuse-tumbleweed OS_VER=latest",
                  "OS=ubuntu OS_VER=18.04",
                  "OS=ubuntu OS_VER=21.04",
-                 "OS=ubuntu OS_VER=devel PUSH_IMAGE=0",
-                 "TYPE=building OS=ubuntu OS_VER=devel"]
+                 "OS=ubuntu OS_VER=devel"]
     steps:
       - name: Clone the git repo
         uses: actions/checkout@v2
 
-      # other_OSes workflow rebuilds images every time and push them to the container registry
+      # It rebuilds images every time and pushes them to the container registry
       - name: Rebuild and push the image
         run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild
 

diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
@@ -27,7 +27,9 @@ jobs:
         CONFIG: ["TYPE=debug OS=ubuntu OS_VER=20.04 TESTS_USAN=1 TESTS_PMREORDER=0",
                  "TYPE=debug OS=ubuntu OS_VER=20.04 TESTS_UBSAN=1 TESTS_PMREORDER=0",
                  "TYPE=valgrind OS=fedora OS_VER=rawhide TESTS_PMREORDER=0",
-                 "TYPE=memcheck_drd OS=fedora OS_VER=rawhide TESTS_PMREORDER=0"]
+                 "TYPE=memcheck_drd OS=fedora OS_VER=rawhide TESTS_PMREORDER=0",
+                 "TYPE=building OS=fedora OS_VER=rawhide",
+                 "TYPE=building OS=ubuntu OS_VER=devel"]
     steps:
       - name: Clone the git repo
         uses: actions/checkout@v2

diff --git a/utils/docker/build.sh b/utils/docker/build.sh
@@ -17,17 +17,17 @@
 
 set -e
 
-source $(dirname $0)/set-ci-vars.sh
+source $(dirname ${0})/set-ci-vars.sh
 IMG_VER=${IMG_VER:-devel}
 TAG="${OS}-${OS_VER}-${IMG_VER}"
 
-if [[ -z "$OS" || -z "$OS_VER" ]]; then
+if [[ -z "${OS}" || -z "${OS_VER}" ]]; then
 	echo "ERROR: The variables OS and OS_VER have to be set " \
 		"(eg. OS=fedora, OS_VER=31)."
 	exit 1
 fi
 
-if [[ -z "$HOST_WORKDIR" ]]; then
+if [[ -z "${HOST_WORKDIR}" ]]; then
 	echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \
 		"the root of this project on the host machine"
 	exit 1
@@ -36,8 +36,8 @@ fi
 imageName=${CONTAINER_REG}:${TAG}
 containerName=pmemkv-${OS}-${OS_VER}
 
-if [[ "$command" == "" ]]; then
-	case $TYPE in
+if [[ "${command}" == "" ]]; then
+	case ${TYPE} in
 		debug)
 			builds=(tests_gcc_debug_cpp11
 					tests_gcc_debug_cpp14)
@@ -78,56 +78,56 @@ if [[ "$command" == "" ]]; then
 	esac
 fi
 
-if [ "$COVERAGE" == "1" ]; then
+if [ "${COVERAGE}" == "1" ]; then
 	docker_opts="${docker_opts} `bash <(curl -s https://codecov.io/env)`";
 fi
 
-if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi
+if [ -n "${DNS_SERVER}" ]; then DNS_SETTING=" --dns=${DNS_SERVER} "; fi
 
 # Check if we are running on a CI (Travis or GitHub Actions)
-[ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO"
+[ -n "${GITHUB_ACTIONS}" -o -n "${TRAVIS}" ] && CI_RUN="YES" || CI_RUN="NO"
 
 # do not allocate a pseudo-TTY if we are running on GitHub Actions
-[ ! $GITHUB_ACTIONS ] && TTY='-t' || TTY=''
+[ ! ${GITHUB_ACTIONS} ] && TTY='-t' || TTY=''
 
 WORKDIR=/pmemkv
-SCRIPTSDIR=$WORKDIR/utils/docker
+SCRIPTSDIR=${WORKDIR}/utils/docker
 
 echo Building on ${OS}-${OS_VER}
 
 # Run a container with
 #  - environment variables set (--env)
 #  - host directory containing source mounted (-v)
 #  - working directory set (-w)
-docker run --privileged=true --name=$containerName -i $TTY \
-	$DNS_SETTING \
+docker run --privileged=true --name=${containerName} -i ${TTY} \
+	${DNS_SETTING} \
 	${docker_opts} \
-	--env http_proxy=$http_proxy \
-	--env https_proxy=$https_proxy \
+	--env http_proxy=${http_proxy} \
+	--env https_proxy=${https_proxy} \
 	--env TERM=xterm-256color \
-	--env WORKDIR=$WORKDIR \
-	--env SCRIPTSDIR=$SCRIPTSDIR \
-	--env COVERAGE=$COVERAGE \
-	--env CI_RUN=$CI_RUN \
-	--env TRAVIS=$TRAVIS \
-	--env GITHUB_REPO=$GITHUB_REPO \
-	--env CI_COMMIT_RANGE=$CI_COMMIT_RANGE \
-	--env CI_COMMIT=$CI_COMMIT \
-	--env CI_REPO_SLUG=$CI_REPO_SLUG \
-	--env CI_BRANCH=$CI_BRANCH \
-	--env CI_EVENT_TYPE=$CI_EVENT_TYPE \
-	--env GITHUB_ACTIONS=$GITHUB_ACTIONS \
-	--env GITHUB_HEAD_REF=$GITHUB_HEAD_REF \
-	--env GITHUB_REPO=$GITHUB_REPO \
-	--env GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
-	--env GITHUB_REF=$GITHUB_REF \
-	--env GITHUB_RUN_ID=$GITHUB_RUN_ID \
-	--env GITHUB_SHA=$GITHUB_SHA \
-	--env DOC_UPDATE_GITHUB_TOKEN=$DOC_UPDATE_GITHUB_TOKEN \
-	--env DOC_UPDATE_BOT_NAME=$DOC_UPDATE_BOT_NAME \
-	--env DOC_REPO_OWNER=$DOC_REPO_OWNER \
-	--env COVERITY_SCAN_TOKEN=$COVERITY_SCAN_TOKEN \
-	--env COVERITY_SCAN_NOTIFICATION_EMAIL=$COVERITY_SCAN_NOTIFICATION_EMAIL \
+	--env WORKDIR=${WORKDIR} \
+	--env SCRIPTSDIR=${SCRIPTSDIR} \
+	--env COVERAGE=${COVERAGE} \
+	--env CI_RUN=${CI_RUN} \
+	--env TRAVIS=${TRAVIS} \
+	--env GITHUB_REPO=${GITHUB_REPO} \
+	--env CI_COMMIT_RANGE=${CI_COMMIT_RANGE} \
+	--env CI_COMMIT=${CI_COMMIT} \
+	--env CI_REPO_SLUG=${CI_REPO_SLUG} \
+	--env CI_BRANCH=${CI_BRANCH} \
+	--env CI_EVENT_TYPE=${CI_EVENT_TYPE} \
+	--env GITHUB_ACTIONS=${GITHUB_ACTIONS} \
+	--env GITHUB_HEAD_REF=${GITHUB_HEAD_REF} \
+	--env GITHUB_REPO=${GITHUB_REPO} \
+	--env GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \
+	--env GITHUB_REF=${GITHUB_REF} \
+	--env GITHUB_RUN_ID=${GITHUB_RUN_ID} \
+	--env GITHUB_SHA=${GITHUB_SHA} \
+	--env DOC_UPDATE_GITHUB_TOKEN=${DOC_UPDATE_GITHUB_TOKEN} \
+	--env DOC_UPDATE_BOT_NAME=${DOC_UPDATE_BOT_NAME} \
+	--env DOC_REPO_OWNER=${DOC_REPO_OWNER} \
+	--env COVERITY_SCAN_TOKEN=${COVERITY_SCAN_TOKEN} \
+	--env COVERITY_SCAN_NOTIFICATION_EMAIL=${COVERITY_SCAN_NOTIFICATION_EMAIL} \
 	--env TEST_PACKAGES=${TEST_PACKAGES:-ON} \
 	--env TESTS_LONG=${TESTS_LONG:-OFF} \
 	--env TESTS_USAN=${TESTS_USAN:-OFF} \
@@ -137,7 +137,7 @@ docker run --privileged=true --name=$containerName -i $TTY \
 	--env CHECK_CPP_STYLE=${CHECK_CPP_STYLE:-OFF} \
 	--env DEFAULT_TEST_DIR=/dev/shm \
 	--shm-size=4G \
-	-v $HOST_WORKDIR:$WORKDIR \
+	-v ${HOST_WORKDIR}:${WORKDIR} \
 	-v /etc/localtime:/etc/localtime \
-	-w $SCRIPTSDIR \
-	$imageName $command
+	-w ${SCRIPTSDIR} \
+	${imageName} ${command}
diff --git a/utils/docker/prepare-for-build.sh b/utils/docker/prepare-for-build.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2019-2020, Intel Corporation
+# Copyright 2019-2021, Intel Corporation
 
 #
 # prepare-for-build.sh - prepare the Docker image for the builds
@@ -12,91 +12,102 @@ set -e
 EXAMPLE_TEST_DIR="/tmp/build_example"
 PREFIX=/usr
 
-# CMake's version assigned to variable(s) (a single number representation for easier comparison)
-CMAKE_VERSION=$(cmake --version | head -n1 | grep -oE '[0-9].[0-9]*')
-CMAKE_VERSION_MAJOR=$(echo $CMAKE_VERSION | cut -d. -f1)
-CMAKE_VERSION_MINOR=$(echo $CMAKE_VERSION | cut -d. -f2)
-CMAKE_VERSION_NUMBER=$((100 * $CMAKE_VERSION_MAJOR + $CMAKE_VERSION_MINOR))
-
+### Helper functions, used in run-*.sh scripts
 function sudo_password() {
-	echo $USERPASS | sudo -Sk $*
+	echo ${USERPASS} | sudo -Sk $*
 }
 
 function upload_codecov() {
 	printf "\n$(tput setaf 1)$(tput setab 7)COVERAGE ${FUNCNAME[0]} START$(tput sgr 0)\n"
 
 	# set proper gcov command
 	clang_used=$(cmake -LA -N . | grep CMAKE_CXX_COMPILER | grep clang | wc -c)
-	if [[ $clang_used > 0 ]]; then
+	if [[ ${clang_used} -gt 0 ]]; then
 		gcovexe="llvm-cov gcov"
 	else
 		gcovexe="gcov"
 	fi
 
 	# run gcov exe, using their bash (remove parsed coverage files, set flag and exit 1 if not successful)
 	# we rely on parsed report on codecov.io; the output is quite long, hence it's disabled using -X flag
-	/opt/scripts/codecov -c -F $1 -Z -x "$gcovexe" -X "gcovout"
+	/opt/scripts/codecov -c -F ${1} -Z -x "${gcovexe}" -X "gcovout"
 
 	printf "check for any leftover gcov files\n"
 	leftover_files=$(find . -name "*.gcov")
-	if [[ -n "$leftover_files" ]]; then
+	if [[ -n "${leftover_files}" ]]; then
 		# display found files and exit with error (they all should be parsed)
-		echo "$leftover_files"
+		echo "${leftover_files}"
 		return 1
 	fi
 
 	printf "$(tput setaf 1)$(tput setab 7)COVERAGE ${FUNCNAME[0]} END$(tput sgr 0)\n\n"
 }
 
 function compile_example_standalone() {
-	example_name=$1
+	example_name=${1}
 	echo "Compile standalone example: ${example_name}"
 
-	rm -rf $EXAMPLE_TEST_DIR
-	mkdir $EXAMPLE_TEST_DIR
-	cd $EXAMPLE_TEST_DIR
+	rm -rf ${EXAMPLE_TEST_DIR}
+	mkdir ${EXAMPLE_TEST_DIR}
+	pushd ${EXAMPLE_TEST_DIR}
 
-	cmake $WORKDIR/examples/$example_name
+	cmake ${WORKDIR}/examples/${example_name}
 
 	# exit on error
 	if [[ $? != 0 ]]; then
-		cd -
+		popd
 		return 1
 	fi
 
 	make -j$(nproc)
-	cd -
+	popd
 }
 
 function run_example_standalone() {
-	example_name=$1
-	pool_path=$2
+	example_name=${1}
+	pool_path=${2}
 	echo "Run standalone example: ${example_name} with path: ${pool_path}"
 
-	cd $EXAMPLE_TEST_DIR
+	pushd ${EXAMPLE_TEST_DIR}
 
-	./$example_name $pool_path
+	./${example_name} ${pool_path}
 
 	# exit on error
 	if [[ $? != 0 ]]; then
-		cd -
+		popd
 		return 1
 	fi
 
-	rm -f $pool_path
-	cd -
+	rm -f ${pool_path}
+	popd
 }
 
 function workspace_cleanup() {
 	echo "Cleanup build dirs and example poolset:"
 
-	cd ${WORKDIR}
+	pushd ${WORKDIR}
 	rm -rf ${WORKDIR}/build
 	rm -rf ${EXAMPLE_TEST_DIR}
 	pmempool rm -f ${WORKDIR}/examples/example.poolset
 }
 
+### Additional checks, to be run, when this file is sourced
+if [[ -z "${WORKDIR}" ]]; then
+	echo "ERROR: The variable WORKDIR has to contain a path to the root " \
+		"of this project - 'build' sub-directory will be created there."
+	exit 1
+fi
+
 # this should be run only on CIs
-if [ "$CI_RUN" == "YES" ]; then
-	sudo_password chown -R $(id -u).$(id -g) $WORKDIR
+if [ "${CI_RUN}" == "YES" ]; then
+	sudo_password chown -R $(id -u).$(id -g) ${WORKDIR}
 fi || true
+
+echo "CMake version:"
+cmake --version
+
+# assign CMake's version to variable(s)  - a single number representation for easier comparison
+CMAKE_VERSION=$(cmake --version | head -n1 | grep -P -o "\d+\.\d+")
+CMAKE_VERSION_MAJOR=$(echo ${CMAKE_VERSION} | cut -d. -f1)
+CMAKE_VERSION_MINOR=$(echo ${CMAKE_VERSION} | cut -d. -f2)
+CMAKE_VERSION_NUMBER=${CMAKE_VERSION_MAJOR}${CMAKE_VERSION_MINOR}
diff --git a/utils/docker/pull-or-rebuild-image.sh b/utils/docker/pull-or-rebuild-image.sh
@@ -24,13 +24,13 @@
 
 set -e
 
-source $(dirname $0)/set-ci-vars.sh
+source $(dirname ${0})/set-ci-vars.sh
 IMG_VER=${IMG_VER:-latest}
 TAG="${OS}-${OS_VER}-${IMG_VER}"
 IMAGES_DIR_NAME=images
 BASE_DIR=utils/docker/${IMAGES_DIR_NAME}
 
-if [[ -z "$OS" || -z "$OS_VER" ]]; then
+if [[ -z "${OS}" || -z "${OS_VER}" ]]; then
 	echo "ERROR: The variables OS and OS_VER have to be set properly " \
              "(eg. OS=fedora, OS_VER=32)."
 	exit 1

diff --git a/utils/docker/run-bindings.sh b/utils/docker/run-bindings.sh
@@ -8,7 +8,7 @@
 
 set -e
 
-source `dirname $0`/prepare-for-build.sh
+source `dirname ${0}`/prepare-for-build.sh
 
 # master: Merge pull request #44 from lukaszstolarczuk/update-tr..., 21.11.2019
 RUBY_VERSION="3741e3df698245fc8a15822a1aa85b5c211fd332"