Merge pull request #1186

systemtest: fixed issues with systemtests not succeeding on first try

CHANGELOG.md

@@ -94,6 +94,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - dird: fix odd-even weeks parsing bug in schedule [PR #1210]
 - bcopy: fix crash in bcopy when using certain cli arguments [PR #1211]
 - webui: fix password string length limitation [BUG #1480][PR #1251]
+- systemtest: fixed issues with systemtests not succeeding on first try [PR #1186]
 
 ### Changed
 - contrib: rename Python modules to satisfy PEP8 [PR #768]

core/scripts/bareos-ctl-funcs

@@ -25,13 +25,13 @@ pidofproc() {
    if [ ! -z "${PGREP}" -a -x "${PGREP}" ] ; then
       pid=`${PGREP} -f $base`
       if [ "$pid" != "" ] ; then
-         echo $pid
+         echo $pid | cut -d' ' -f 1
          return 0
       fi
    fi
 
    # Finally try to extract it from ps
-   pid=`${PSCMD} | grep $base | ${AWK} '{ print $1 }' | tr '\n' ' '`
+   pid=`${PSCMD} | grep -m 1 $base | ${AWK} '{ print $1 }' | tr '\n' ' '`
    echo $pid
    return 0
 }

core/src/tests/bareos_test_sockets.cc

@@ -1,7 +1,7 @@
 /*
    BAREOS® - Backup Archiving REcovery Open Sourced
 
-   Copyright (C) 2018-2020 Bareos GmbH & Co. KG
+   Copyright (C) 2018-2022 Bareos GmbH & Co. KG
 
    This program is Free Software; you can redistribute it and/or
    modify it under the terms of version three of the GNU Affero General Public
@@ -28,9 +28,10 @@
 
 #include "bareos_test_sockets.h"
 #include "tests/bsock_test.h"
-
 #include "lib/bsock_tcp.h"
 
+#include <thread>
+
 #if HAVE_WIN32
 #  include <cstdlib>
 #  include <mutex>
@@ -78,9 +79,22 @@ static int create_listening_server_socket(int port)
   address.sin_family = AF_INET;
   address.sin_addr.s_addr = INADDR_ANY;
   address.sin_port = htons(port);
-  if (bind(listen_file_descriptor, (struct sockaddr*)&address, sizeof(address))
-      < 0) {
-    perror("bind failed");
+
+  int bindresult = -1;
+  for (int i = 0; i < 6; i++) {
+    bindresult = bind(listen_file_descriptor, (struct sockaddr*)&address,
+                      sizeof(address));
+    if (bindresult == 0) {
+      std::cout << "bind successful after " << i + 1 << " tries" << std::endl;
+      break;
+    } else {
+      std::this_thread::sleep_for(std::chrono::seconds(2));
+    }
+  }
+  std::string errormessage{"bind failed for port "};
+  errormessage.append(std::to_string(port));
+  if (bindresult < 0) {
+    perror(errormessage.c_str());
     return -4;
   }
 
@@ -105,8 +119,17 @@ static int create_listening_server_socket(int port)
     perror("setsockopt");
     return -5;
   }
-
-  if (listen(listen_file_descriptor, 3) < 0) {
+  int listenresult = -1;
+  for (int i = 0; i < 6; i++) {
+    listenresult = listen(listen_file_descriptor, 3);
+    if (listenresult == 0) {
+      std::cout << "listen successful after " << i + 1 << " tries" << std::endl;
+      break;
+    } else {
+      std::this_thread::sleep_for(std::chrono::seconds(2));
+    }
+  }
+  if (listenresult < 0) {
     perror("listen");
     return -6;
   }

systemtests/scripts/check_for_zombie_jobs

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Check for zombie jobs (not terminated).
 # Also scan logs for ERROR messages
@@ -7,6 +7,9 @@
 # shellcheck source=../environment.in
 . ./environment
 
+# shellcheck source= ./functions
+. "${rscripts}"/functions
+
 # check_for_zombie_jobs storage=STORAGE [client=localhost-fd]
 
 if [ $# = 2 ] ; then
@@ -15,6 +18,8 @@ else
     client="client"
 fi
 
+wait_for_jobs_to_terminate "${client}" 20
+
 "${BAREOS_BCONSOLE_BINARY}" -c "${conf}" <<END_OF_DATA >/dev/null 2>&1
 @output "${tmp}"/dir.out
 status dir

systemtests/scripts/cleanup

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -u
 
 #shellcheck source=../environment.in

systemtests/scripts/diff.pl.in

@@ -52,7 +52,7 @@ my $md5 = Digest::MD5->new;
 
 my $dir = getcwd;
 
-chdir($src) or die "ERROR: Can't access to $src";
+chdir($src) or die "ERROR: Can't access to $src : $!";
 $hash = \%src_attr;
 find(\&wanted_src, '.');
 
@@ -63,7 +63,7 @@ if ($wattr) {
 
 chdir ($dir);
 
-chdir($dst) or die "ERROR: Can't access to $dst";
+chdir($dst) or die "ERROR: Can't access to $dst : $!";
 $hash = \%dst_attr;
 find(\&wanted_src, '.');
 

systemtests/scripts/functions

@@ -3,6 +3,20 @@
 # A set of useful functions to be sourced in each test
 #
 
+wait_for_jobs_to_terminate ()
+{
+  status_of_what=$1
+  max_wait_in_seconds=$2
+  count=max_wait_in_seconds
+  while (( --count >= 0 )); do
+    if  "${BAREOS_BCONSOLE_BINARY}" -c "${conf}" <<< "status ${status_of_what}" | grep "No Jobs running.";
+    then
+      break
+    fi
+    sleep 1
+  done
+}
+
 copy_configs()
 {
     COMPONENTS="bareos-dir bareos-sd bareos-fd bconsole tray-monitor"

systemtests/scripts/setup

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #shellcheck source=../environment.in
 . ./environment
 . "${rscripts}/functions"

systemtests/tests/CMakeLists.txt

@@ -39,7 +39,7 @@ add_subdirectory(copy-remote-bscan)
 add_subdirectory(dbcopy-mysql-postgresql)
 add_subdirectory(deprecation)
 add_subdirectory(droplet-s3)
-add_subdirectory(encrypt-signature)
+add_subdirectory(encrypt-signature-no-tls)
 add_subdirectory(encrypt-signature-tls-cert)
 add_subdirectory(fileset-multiple-blocks)
 add_subdirectory(filesets)
@@ -57,7 +57,7 @@ add_subdirectory(py2plug-dir)
 add_subdirectory(py2plug-fd-contrib-bareos_tasks_mysql)
 add_subdirectory(py2plug-fd-ldap)
 add_subdirectory(py2plug-fd-libcloud)
-add_subdirectory(py2plug-fd-local-fileset)
+add_subdirectory(py2plug-fd-local-fileset-basic)
 add_subdirectory(py2plug-fd-local-fileset-restoreobject)
 add_subdirectory(py2plug-fd-mariabackup)
 add_subdirectory(py2plug-fd-ovirt)

systemtests/tests/acl/testrunner

@@ -57,7 +57,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 

systemtests/tests/bareos-acl/testrunner

@@ -42,6 +42,4 @@ grep -q "status conf: is an invalid command" ${logdir}/acl-status.log || \
 grep -q "eprecated configuration settings detected" ${logdir}/acl-status-conf.log || \
   set_error "acl-status-conf.log"
 
-stop_bareos
-
 end_test

systemtests/tests/bconsole-pam/bin/check_pam_exec_available.sh

@@ -1,7 +1,7 @@
-#!/bin/sh
+#!/bin/bash
 #   BAREOS® - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2019-2019 Bareos GmbH & Co. KG
+#   Copyright (C) 2019-2022 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public

systemtests/tests/bscan-bextract-bls-bcopy/testrunner

@@ -163,8 +163,6 @@ run_bconsole "$tmp/bconcmds2"
 
 check_for_zombie_jobs storage=File
 
-stop_bareos
-
 check_two_logs
 check_restore_diff "${BackupDirectory}"
 end_test

systemtests/tests/chflags/testrunner

@@ -62,7 +62,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 

systemtests/tests/client-initiated/testrunner

@@ -90,8 +90,6 @@ run_bconsole "$tmp/bconcmds"
 
 check_for_zombie_jobs storage=File
 
-stop_bareos
-
 check_two_logs
 check_restore_diff "${BackupDirectory}"
 end_test

systemtests/tests/commandline-options/etc/bareos/bareos-sd.d/device/FileStorage.conf

@@ -8,8 +8,4 @@ Device {
   RemovableMedia = no;
   AlwaysOpen = no;
   Description = "File device. A connecting Director must have the same Name and MediaType."
-  Auto Inflate = both
-  Auto Deflate = both
-  Auto Deflate Algorithm = gzip
-
 }

systemtests/tests/commandline-options/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in

@@ -6,8 +6,6 @@ Storage {
   # if "Plugin Names" is defined, only the specified plugins will be loaded,
   # otherwise all storage plugins (*-sd.so) from the "Plugin Directory".
   #
-  Plugin Directory = "@SD_PLUGINS_DIR_TO_TEST@"
-  Plugin Names = "autoxflate"
   Working Directory =  "@working_dir@"
   SD Port = @sd_port@
 }

systemtests/tests/copy-archive-job/testrunner

@@ -58,7 +58,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_sd_files_written $tmp/log2.out
 check_sd_files_written $tmp/log3.out

systemtests/tests/dbcopy-mysql-postgresql/testrunner

@@ -80,7 +80,6 @@ END_OF_DATA
 
 run_bareos
 check_for_zombie_jobs storage=File client=bareos-fd
-stop_bareos
 
 check_two_logs
 check_restore_diff "${BackupDirectory}"

systemtests/tests/droplet-s3/testrunner

@@ -55,7 +55,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 check_restore_diff "${BackupDirectory}"

systemtests/tests/encrypt-signature/CMakeLists.txt → systemtests/tests/encrypt-signature-no-tls/CMakeLists.txt

@@ -1,6 +1,6 @@
 #   BAREOS® - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2021-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2021-2022 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public

systemtests/tests/encrypt-signature/testrunner → systemtests/tests/encrypt-signature-no-tls/testrunner

@@ -42,17 +42,14 @@ messages
 @# now do a restore
 @#
 @$out $tmp/log2.out
-wait
-restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done
-yes
+restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done yes
 wait
 messages
 quit
 END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 check_restore_diff "${BackupDirectory}"

systemtests/tests/encrypt-signature-tls-cert/CMakeLists.txt

@@ -1,6 +1,6 @@
 #   BAREOS® - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2021-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2021-2022 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public

systemtests/tests/encrypt-signature-tls-cert/testrunner

@@ -42,17 +42,14 @@ messages
 @# now do a restore
 @#
 @$out $tmp/log2.out
-wait
-restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done
-yes
+restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done yes
 wait
 messages
 quit
 END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 check_restore_diff ${BackupDirectory}

systemtests/tests/filesets/testrunner

@@ -60,7 +60,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 for i in $TESTS; do
     if ! grep -q $tmp/$i/data2.txt $tmp/estimate-$i.out; then

systemtests/tests/gfapi-fd/testrunner

@@ -35,7 +35,7 @@ while "${SUDO}" mount | grep "${tmp}/data" >/dev/null 2>&1; do
 done
 }
 
-
+trap umount_glusterfs EXIT
 
 #shellcheck source=../scripts/functions
 . "${rscripts}"/functions
@@ -136,7 +136,6 @@ quit
 END_OF_DATA_RESTORE
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 ln -s  "${tmp}" "${tmp}/bareos-restores"

systemtests/tests/glusterfs-backend/testrunner

@@ -51,7 +51,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 check_restore_diff ${BackupDirectory}

systemtests/tests/list-backups/testrunner

@@ -51,7 +51,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 echo "Checking list backups for jobtype=backup"
 grep -F '1 | backup-bareos-fd | bareos-fd' "$tmp/list-backups.out"

systemtests/tests/multiplied-device/testrunner

@@ -85,7 +85,6 @@ quit
 END_OF_DATA
 
 run_bconsole
-stop_bareos
 
 # rename files to be able to run the test again later
 mv ./etc/bareos/bareos-dir.d/storage/fakestorage1.conf_backup ./etc/bareos/bareos-dir.d/storage/fakestorage1.conf

systemtests/tests/ndmp/testrunner

@@ -165,6 +165,4 @@ cleanup_isilon
 
 check_for_zombie_jobs storage=isilonfile client=bareos-fd
 
-stop_bareos
-
 end_test

systemtests/tests/notls/testrunner

@@ -61,7 +61,6 @@ END_OF_DATA
 
 run_bareos "$@"
 check_for_zombie_jobs storage=File
-stop_bareos
 
 check_two_logs
 check_restore_diff ${BackupDirectory}