REDIS::ADDED:: Implement tests for imhiredis module

- changed diag.sh to be able to start/stop/clean a redis server
- added helper functions in diag.sh to be able to query a redis server instance
- added new tests for imhiredis module to check
  - that the queue mode works, with both lpop and rpop
  - that the module is capable of handling a redis server going down
  - that the module is capable of handling a redis server that appears afterwards
  - that the subscribe mode works

configure.ac

@@ -2461,6 +2461,17 @@ AC_ARG_ENABLE(omhiredis,
         [enable_omhiredis=no]
 )
 
+AC_ARG_ENABLE(redis_tests,
+        [AS_HELP_STRING([--enable-redis-tests],[Enable redis tests, needs redis-server @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_redis_tests="yes" ;;
+          no) enable_redis_tests="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-redis-tests) ;;
+         esac],
+        [enable_redis_tests=no]
+)
+AM_CONDITIONAL(ENABLE_REDIS_TESTS, test x$enable_redis_tests = xyes)
+
 if test "x$enable_omhiredis" = "xyes" -o "x$enable_imhiredis" = "xyes" ; then
     PKG_CHECK_MODULES(HIREDIS, hiredis >= 0.10.1, [],
         [AC_SEARCH_LIBS(redisConnectWithTimeout, hiredis,
@@ -2497,6 +2508,21 @@ if test "x$enable_imhiredis" = "xyes" ; then
         [AC_MSG_ERROR([no libevent >= 2.0 found with pthreads support, imhiredis cannot use pub/sub])])
 fi
 
+if test "x$enable_imhiredis" = "xyes" || test "x$enable_omhiredis" = "xyes"; then
+	if test "x$enable_redis_tests" = "xyes"; then
+		AC_CHECK_PROG(REDIS, [redis-server], [yes], [no])
+		if test "x${REDIS}" = "xno"; then
+			AC_MSG_FAILURE([redis-server, which is a redis-tests dependency, not found])
+		fi
+	fi
+else
+	if test "x$enable_redis_tests" = "xyes"; then
+		AC_MSG_WARN([redis-tests can not be enabled without imhiredis or omhiredis support.
+                        Disabling enable_redis_tests...])
+		enable_redis_tests="no"
+	fi
+fi
+
 AM_CONDITIONAL(ENABLE_OMHIREDIS, test x$enable_omhiredis = xyes)
 AM_CONDITIONAL(ENABLE_IMHIREDIS, test x$enable_imhiredis = xyes)
 
@@ -2896,6 +2922,7 @@ echo "    Elasticsearch Tests:                      $enable_elasticsearch_tests"
 echo "    ClickHouse Tests:                         $enable_clickhouse_tests"
 echo "    PostgreSQL Tests enabled:                 $enable_pgsql_tests"
 echo "    Kafka Tests enabled:                      $enable_kafka_tests"
+echo "    Redis Tests enabled:                      $enable_redis_tests"
 echo "    Imdocker Tests enabled:                   $enable_imdocker_tests"
 echo "    gnutls tests enabled:                     $enable_gnutls_tests"
 echo "    imfile tests enabled:                     $enable_imfile_tests"

tests/Makefile.am

@@ -1039,6 +1039,24 @@ TESTS += \
 endif # HAVE_VALGRIND
 endif # ENABLE_OMRABBITMQ
 
+if ENABLE_REDIS_TESTS
+
+TESTS +=  \
+	imhiredis-queue.sh \
+	imhiredis-queue-lpop.sh \
+	imhiredis-redis-restart.sh \
+	imhiredis-redis-start-after.sh \
+	imhiredis-subscribe.sh
+if HAVE_VALGRIND
+TESTS += \
+	imhiredis-queue-vg.sh \
+	imhiredis-queue-lpop-vg.sh \
+	imhiredis-redis-restart-vg.sh \
+	imhiredis-redis-start-after-vg.sh \
+	imhiredis-subscribe-vg.sh
+endif # HAVE_VALGRIND
+endif # ENABLE_REDIS_TESTS
+
 if ENABLE_IMPSTATS
 TESTS +=  \
 	impstats-hup.sh \
@@ -2009,6 +2027,16 @@ EXTRA_DIST= \
 	omrabbitmq_error_server3.sh \
 	omrabbitmq_json.sh \
 	omrabbitmq_raw.sh \
+	imhiredis-queue.sh \
+	imhiredis-queue-vg.sh \
+	imhiredis-queue-lpop.sh \
+	imhiredis-queue-lpop-vg.sh \
+	imhiredis-redis-restart.sh \
+	imhiredis-redis-restart-vg.sh \
+	imhiredis-redis-start-after.sh \
+	imhiredis-redis-start-after-vg.sh \
+	imhiredis-subscribe.sh \
+	imhiredis-subscribe-vg.sh
 	msgvar-concurrency.sh \
 	msgvar-concurrency-array.sh \
 	testsuites/msgvar-concurrency-array.rulebase \

tests/diag.sh

@@ -1316,6 +1316,11 @@ error_exit() {
 	# Extended Exit handling for kafka / zookeeper instances 
 	kafka_exit_handling "false"
 
+	# Ensure redis instance is stopped
+	if [ -n "$REDIS_DYN_DIR" ]; then
+		stop_redis
+	fi
+
 	error_stats $1 # Report error to rsyslog testbench stats
 	do_cleanup
 
@@ -1528,6 +1533,9 @@ exit_test() {
 	# Extended Exit handling for kafka / zookeeper instances 
 	kafka_exit_handling "true"
 
+	# Ensure redis is stopped
+	stop_redis
+
 	printf '%s Test %s SUCCESSFUL (took %s seconds)\n' "$(tb_timestamp)" "$0" "$(( $(date +%s) - TB_STARTTEST ))"
 	echo  -------------------------------------------------------------------------------
 	exit 0
@@ -2412,6 +2420,97 @@ mysql_cleanup_test() {
 }
 
 
+start_redis() {
+	check_command_available redis-server
+
+	export REDIS_DYN_CONF="${RSYSLOG_DYNNAME}.redis.conf"
+	export REDIS_DYN_DIR="$(pwd)/${RSYSLOG_DYNNAME}-redis"
+
+	# Only set a random port if not set (useful when Redis must be restarted during a test)
+	if [ -z "$REDIS_RANDOM_PORT" ]; then
+		export REDIS_RANDOM_PORT="$(get_free_port)"
+	fi
+
+	cp $srcdir/testsuites/redis.conf $REDIS_DYN_CONF
+	mkdir -p $REDIS_DYN_DIR
+
+	sed -itemp "s+<tmpdir>+${REDIS_DYN_DIR}+g" $REDIS_DYN_CONF
+	sed -itemp "s+<rndport>+${REDIS_RANDOM_PORT}+g" $REDIS_DYN_CONF
+
+	# Start the server
+	echo "Starting redis with conf file $REDIS_DYN_CONF"
+	redis-server $REDIS_DYN_CONF &
+	$TESTTOOL_DIR/msleep 2000
+
+	# Wait for Redis to be fully up
+	timeoutend=10
+	until nc -w1 -z 127.0.0.1 $REDIS_RANDOM_PORT; do
+		echo "Waiting for Redis to start..."
+		$TESTTOOL_DIR/msleep 1000
+		(( timeseconds=timeseconds + 2 ))
+
+		if [ "$timeseconds" -gt "$timeoutend" ]; then
+			echo "--- TIMEOUT ( $timeseconds ) reached!!!"
+			if [ ! -d ${REDIS_DYN_DIR}/redis.log ]; then
+				echo "no Redis logs"
+			else
+				echo "Dumping ${REDIS_DYN_DIR}/redis.log"
+				echo "========================================="
+				cat ${REDIS_DYN_DIR}/redis.log
+				echo "========================================="
+			fi
+			error_exit 1
+		fi
+	done
+}
+
+cleanup_redis() {
+	if [ -d ${REDIS_DYN_DIR} ]; then
+		rm -rf ${REDIS_DYN_DIR}
+	fi
+	if [ -f ${REDIS_DYN_CONF} ]; then
+		rm -f ${REDIS_DYN_CONF}
+	fi
+}
+
+stop_redis() {
+	if [ -f "$REDIS_DYN_DIR/redis.pid" ]; then
+		redispid=$(cat $REDIS_DYN_DIR/redis.pid)
+		echo "Stopping Redis instance"
+		kill $redispid
+
+		i=0
+
+		# Check if redis instance went down!
+		while [ -f $REDIS_DYN_DIR/redis.pid ]; do
+			redispid=$(cat $REDIS_DYN_DIR/redis.pid)
+			if [[ "" !=  "$redispid" ]]; then
+				$TESTTOOL_DIR/msleep 100 # wait 100 milliseconds
+				if test $i -gt $TB_TIMEOUT_STARTSTOP; then
+					echo "redis server (PID $redispid) still running - Performing hard shutdown (-9)"
+					kill -9 $redispid
+					break
+				fi
+				(( i++ ))
+			else
+				# Break the loop
+				break
+			fi
+		done
+	fi
+}
+
+redis_command() {
+	check_command_available redis-cli
+
+	if [ -z "$1" ]; then
+		echo "redis_command: no command provided!"
+		error_exit 1
+	fi
+
+	printf "$1\n" | redis-cli -p "$REDIS_RANDOM_PORT"
+}
+
 # $1 - replacement string
 # $2 - start search string
 # $3 - file name

tests/imhiredis-queue-lpop-vg.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# added 2023-04-20 by Théo Bertin, released under ASL 2.0
+## Uncomment for debugging
+#export RS_REDIR=-d
+
+export USE_VALGRIND="YES"
+source ${srcdir:=.}/imhiredis-queue-lpop.sh

tests/imhiredis-queue-lpop.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# added 2023-04-20 by Théo Bertin, released under ASL 2.0
+## Uncomment for debugging
+#export RS_REDIR=-d
+
+. ${srcdir:=.}/diag.sh init
+
+start_redis
+
+redis_command "RPUSH mykey message1"
+redis_command "RPUSH mykey message2"
+redis_command "RPUSH mykey message3"
+
+generate_conf
+add_conf '
+global(localhostname="server")
+module(load="../contrib/imhiredis/.libs/imhiredis")
+template(name="outfmt" type="string" string="%$/num% %msg%\n")
+
+
+input(type="imhiredis"
+        server="127.0.0.1"
+        port="'$REDIS_RANDOM_PORT'"
+        key="mykey"
+        mode="queue"
+        uselpop="on"
+        ruleset="redis")
+
+ruleset(name="redis") {
+  set $/num = cnum($/num + 1);
+  action(type="omfile"
+            file="'$RSYSLOG_OUT_LOG'"
+            template="outfmt")
+}
+
+action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt")
+'
+startup
+shutdown_when_empty
+wait_shutdown
+
+stop_redis
+
+# Same order
+content_check '1 message1'
+content_check '2 message2'
+content_check '3 message3'
+
+# Removes generated configuration file, log and pid files
+cleanup_redis
+
+exit_test

tests/imhiredis-queue-vg.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# added 2023-04-20 by Théo Bertin, released under ASL 2.0
+## Uncomment for debugging
+#export RS_REDIR=-d
+
+export USE_VALGRIND="YES"
+source ${srcdir:=.}/imhiredis-queue.sh

tests/imhiredis-queue.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# added 2023-04-20 by Théo Bertin, released under ASL 2.0
+## Uncomment for debugging
+#export RS_REDIR=-d
+
+. ${srcdir:=.}/diag.sh init
+
+start_redis
+
+redis_command "RPUSH mykey message1"
+redis_command "RPUSH mykey message2"
+redis_command "RPUSH mykey message3"
+
+generate_conf
+add_conf '
+global(localhostname="server")
+module(load="../contrib/imhiredis/.libs/imhiredis")
+template(name="outfmt" type="string" string="%$/num% %msg%\n")
+
+
+input(type="imhiredis"
+        server="127.0.0.1"
+        port="'$REDIS_RANDOM_PORT'"
+        key="mykey"
+        mode="queue"
+        ruleset="redis")
+
+ruleset(name="redis") {
+  set $/num = cnum($/num + 1);
+  action(type="omfile"
+            file="'$RSYSLOG_OUT_LOG'"
+            template="outfmt")
+}
+
+action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt")
+'
+startup
+shutdown_when_empty
+wait_shutdown
+
+stop_redis
+
+# Opposite order
+content_check '1 message3'
+content_check '2 message2'
+content_check '3 message1'
+
+# Removes generated configuration file, log and pid files
+cleanup_redis
+
+exit_test

tests/imhiredis-redis-restart-vg.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# added 2023-04-20 by Théo Bertin, released under ASL 2.0
+## Uncomment for debugging
+#export RS_REDIR=-d
+
+export USE_VALGRIND="YES"
+source ${srcdir:=.}/imhiredis-redis-restart.sh