CI: add github actions for some CI runs

.github/workflows/run_centos_7.yml

@@ -0,0 +1,59 @@
+# Copyright 2020 Rainer Gerhards and Others
+#
+# https://github.com/rsyslog/rsyslog-pkg-ubuntu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# References:
+#
+# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options
+# https://github.com/settings/notifications
+# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog
+
+
+---
+name: check centos 7
+
+on:
+  pull_request:
+
+jobs:
+  CI:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+
+    steps:
+      - name: git checkout project
+        uses: actions/checkout@v1
+
+      - name: run container CI pipeline
+        run: |
+          chmod -R go+rw .
+          export RSYSLOG_CONTAINER_UID="" # use default
+          export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php'
+          export CFLAGS='-g'
+          export CC='gcc'
+          export USE_AUTO_DEBUG='off'
+          export CI_MAKE_OPT='-j20'
+          export CI_MAKE_CHECK_OPT='-j1'
+          export CI_CHECK_CMD='check'
+          export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:7'
+          # we need to override rsyslog configure options to avoid side-effects!
+          export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE=''
+          devtools/devcontainer.sh --rm devtools/run-ci.sh
+
+      - name: show error logs (if we errored)
+        if:  ${{ failure() || cancelled() }}
+        run: |
+          devtools/gather-check-logs.sh
+          cat failed-tests.log

.github/workflows/run_centos_8.yml

@@ -0,0 +1,59 @@
+# Copyright 2020 Rainer Gerhards and Others
+#
+# https://github.com/rsyslog/rsyslog-pkg-ubuntu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# References:
+#
+# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options
+# https://github.com/settings/notifications
+# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog
+
+
+---
+name: check centos 8
+
+on:
+  pull_request:
+
+jobs:
+  CI:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+
+    steps:
+      - name: git checkout project
+        uses: actions/checkout@v1
+
+      - name: run container CI pipeline
+        run: |
+          chmod -R go+rw .
+          export RSYSLOG_CONTAINER_UID="" # use default
+          export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php'
+          export CFLAGS='-g'
+          export CC='gcc'
+          export USE_AUTO_DEBUG='off'
+          export CI_MAKE_OPT='-j20'
+          export CI_MAKE_CHECK_OPT='-j1'
+          export CI_CHECK_CMD='check'
+          export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:8'
+          # we need to override rsyslog configure options to avoid side-effects!
+          export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE=''
+          devtools/devcontainer.sh --rm devtools/run-ci.sh
+
+      - name: show error logs (if we errored)
+        if:  ${{ failure() || cancelled() }}
+        run: |
+          devtools/gather-check-logs.sh
+          cat failed-tests.log

devtools/default_dev_container

@@ -0,0 +1 @@
+rsyslog/rsyslog_dev_base_ubuntu:20.04

devtools/devcontainer.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# This scripts uses an rsyslog development container to execute given
+# command inside it.
+# Note: command line parameters are passed as parameters to the container,
+# with the notable exception that -ti, if given as first parameter, is
+# passed to "docker run" itself but NOT the container.
+#
+# use env var DOCKER_RUN_EXTRA_OPTS to provide extra options to docker run
+# command.
+#
+#
+# TO MODIFIY BEHAVIOUR, use
+# LIBLOGNORM_CONTAINER_UID, format uid:gid,
+#                   to change the users container is run under
+#                   set to "" to use the container default settings
+#                   (no local mapping)
+set -e
+if [ "$1" == "--rm" ]; then
+	optrm="--rm"
+	shift 1
+fi
+if [ "$1" == "-ti" ]; then
+	ti="-ti"
+	shift 1
+fi
+# check in case -ti was in front...
+if [ "$1" == "--rm" ]; then
+	optrm="--rm"
+	shift 1
+fi
+
+if [ "$LIBLOGNORM_HOME" == "" ]; then
+	export LIBLOGNORM_HOME=$(pwd)
+	echo info: LIBLOGNORM_HOME not set, using $LIBLOGNORM_HOME
+fi
+
+if [ -z "$LIBLOGNORM_DEV_CONTAINER" ]; then
+	LIBLOGNORM_DEV_CONTAINER=$(cat $LIBLOGNORM_HOME/devtools/default_dev_container)
+fi
+
+printf '/rsyslog is mapped to %s \n' "$LIBLOGNORM_HOME"
+printf 'using container %s\n' "$LIBLOGNORM_DEV_CONTAINER"
+printf 'pulling container...\n'
+printf 'user ids: %s:%s\n' $(id -u) $(id -g)
+printf 'container_uid: %s\n' ${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)}
+printf 'container cmd: %s\n' $*
+printf '\nNote: we use the RSYSLOG CONTAINERS, as such project home is /rsyslog!\n\n'
+docker pull $LIBLOGNORM_DEV_CONTAINER
+docker run $ti $optrm $DOCKER_RUN_EXTRA_OPTS \
+	-e LIBLOGNORM_CONFIGURE_OPTIONS_EXTRA \
+	-e LIBLOGNORM_CONFIGURE_OPTIONS_OVERRIDE \
+	-e CC \
+	-e CFLAGS \
+	-e LDFLAGS \
+	-e LSAN_OPTIONS \
+	-e TSAN_OPTIONS \
+	-e UBSAN_OPTIONS \
+	-e CI_MAKE_OPT \
+	-e CI_MAKE_CHECK_OPT \
+	-e CI_CHECK_CMD \
+	-e CI_BUILD_URL \
+	-e CI_CODECOV_TOKEN \
+	-e CI_VALGRIND_SUPPRESSIONS \
+	-e CI_SANITIZE_BLACKLIST \
+	-e ABORT_ALL_ON_TEST_FAIL \
+	-e USE_AUTO_DEBUG \
+	-e LIBLOGNORM_STATSURL \
+	-e VCS_SLUG \
+	--cap-add SYS_ADMIN \
+	--cap-add SYS_PTRACE \
+	${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)} \
+	$DOCKER_RUN_EXTRA_FLAGS \
+	-v "$LIBLOGNORM_HOME":/rsyslog $LIBLOGNORM_DEV_CONTAINER $*

devtools/gather-check-logs.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# gather logs generated by "make [dist]check"
+# this also limits log size so that buildbot does not abort
+# Copyright (C) 2020 by Rainer Gerhards, released under ASL 2.0
+
+show_log() {
+	if grep -q ":test-result: FAIL" "$1"; then
+		printf "\nFAIL: ${1%%.trs} \
+		########################################################\
+		################################\n\n"
+		logfile="${1%%trs}log"
+		if [ -f "$logfile" ]; then
+			lines="$(wc -l < $logfile)"
+			if (( lines > 4000 )); then
+				ls -l $logfile
+				printf 'file is very large (%d lines), showing parts\n' $lines
+				head -n 2000 < "$logfile"
+				printf '\n\n... snip ...\n\n'
+				tail -n 2000 < "$logfile"
+			else
+				cat "$logfile"
+			fi
+		else
+			printf 'log FILE MISSING!\n'
+		fi
+	fi
+}
+
+append_summary() {
+	echo file: $1 # emit file name just in case we have multiple!
+	head -n12 "$1"
+}
+
+# find logs from tests which are potentially aborted. The main indication is
+# that no matching .trs file exists
+check_incomplete_logs() {
+	if grep -q "\.dep_wrk\|rstb_\|config.log" <<<"$1"; then
+		return
+	fi
+	# we emit info only for test log files - this means there must
+	# be a matching .sh file by our conventions
+	if [ -f "${1%%log}sh" ]; then
+		trsfile="${1%%log}trs"
+		if [ ! -f "$trsfile" ]; then
+			printf '\n\nNo matching .trs file for %s\n' "$1"
+			ls -l ${1%%.log}*
+			cat "$1"
+		fi
+	fi
+}
+export -f show_log
+export -f append_summary
+export -f check_incomplete_logs
+
+############################## MAIN ENTRY POINT ##############################
+printf 'find failing tests\n'
+rm -f failed-tests.log
+
+find . -name "*.trs" -exec  bash -c 'show_log "$1" >> failed-tests.log' _ {} \;
+
+find . -name "*.log" -exec  bash -c 'check_incomplete_logs "$1" >> failed-tests.log' _ {} \;
+
+if [ -f failed-tests.log ]; then
+	# show summary stats so that we know how many failed
+	find . -name test-suite.log -exec bash -c 'append_summary "$1" >>failed-tests.log' _ {} \;
+fi

devtools/run-ci.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# script for generic CI runs via container
+printf 'running CI with\n'
+printf 'container: %s\n' $RSYSLOG_DEV_CONTAINER
+printf 'CC:\t%s\n' "$CC"
+printf 'CFLAGS:\t%s:\n' "$CFLAGS"
+printf 'RSYSLOG_CONFIGURE_OPTIONS:\t%s\n' "$RSYSLOG_CONFIGURE_OPTIONS"
+printf 'working directory: %s\n' "$(pwd)"
+printf 'user ids: %s:%s\n' $(id -u) $(id -g)
+if [ "$SUDO" != "" ]; then
+	printf 'check sudo'
+	$SUDO echo sudo works!
+fi
+if [ "$CI_VALGRIND_SUPPRESSIONS" != "" ]; then
+	export RS_TESTBENCH_VALGRIND_EXTRA_OPTS="--suppressions=$(pwd)/tests/CI/$CI_VALGRIND_SUPPRESSIONS"
+fi
+if [ "$CI_SANITIZE_BLACKLIST" != "" ]; then
+	export CFLAGS="$CFLAGS -fsanitize-blacklist=$(pwd)/$CI_SANITIZE_BLACKLIST"
+	printf 'CFLAGS changed to: %s\n', "$CFLAGS"
+fi
+set -e
+
+printf 'STEP: autoreconf / configure ===============================================\n'
+autoreconf -fvi
+./configure
+
+if [ "$CI_CHECK_CMD" != "distcheck" ]; then
+	printf 'STEP: make =================================================================\n'
+	make $CI_MAKE_OPT
+fi
+
+printf 'STEP: make %s ==============================================================\n', \
+	"$CI_CHECK_CMD"
+set +e
+echo CI_CHECK_CMD: $CI_CHECK_CMD
+make $CI_MAKE_CHECK_OPT ${CI_CHECK_CMD:-check}
+rc=$?
+
+printf 'STEP: find failing tests ====================================================\n'
+echo calling gather-check-logs
+devtools/gather-check-logs.sh
+
+printf 'STEP: Codecov upload =======================================================\n'
+if [ "$CI_CODECOV_TOKEN" != "" ]; then
+	curl -s https://codecov.io/bash >codecov.sh
+	chmod +x codecov.sh
+	./codecov.sh -t "$CI_CODECOV_TOKEN" -n 'rsyslog buildbot PR' &> codecov_log
+	rm codecov.sh
+	lines="$(wc -l < codecov_log)"
+	if (( lines > 3000 )); then
+		printf 'codecov log file is very large (%d lines), showing parts\n' $lines
+		head -n 1500 < codecov_log
+		printf '\n\n... snip ...\n\n'
+		tail -n 1500 < codecov_log
+	else
+		cat codecov_log
+	fi
+	rm codecov_log
+fi
+
+exit $rc