Merge pull request #790 from bareos/dev/pstorz/master/enhance-testcov…

…erage

systemtests: increase test coverage

.quality-gates.yml

@@ -3,8 +3,8 @@
 coverage:
   lines:
     healthy: 70
-    unhealthy: 20
-    unstable: 22
+    unhealthy: 38
+    unstable: 40
 
 docs:
   warnings_unhealthy: 72

CHANGELOG.md

@@ -62,6 +62,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - buildsystem: switch to cross build chain of Fedora 34 [PR #819]
 - FreeBSD: adapt pkglists for FreeBSD 13.0 [PR #819]
 - Fedora34: do not build mysql db backend, adapt pkglist [PR #819]
+- bscan and bareos systemtests: also test bextract and bls binaries, use autoxflate plugin and FSType fileset options [PR #790]
 
 ### Deprecated
 

core/CMakeLists.txt

@@ -36,6 +36,10 @@ if(POLICY CMP0057)
   cmake_policy(SET CMP0057 NEW)
 endif(POLICY CMP0057)
 
+if(POLICY CMP0109)
+  cmake_policy(SET CMP0109 NEW)
+endif(POLICY CMP0109)
+
 # disable in-source builds
 if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
   message(FATAL_ERROR "In-source builds are not allowed.")
@@ -952,7 +956,6 @@ if(HAVE_WIN32)
   message("   WINDOWS_BITS:                 ${WINDOWS_BITS}")
   message("   WINEPATH environment:         $ENV{WINEPATH}")
 endif()
-
 # REPEAT_UNTIL requires cmake 3.17+
 if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.17.0")
   set(REPEAT_UNTIL "REPEAT UNTIL_PASS:2")

core/cmake/BareosFindPrograms.cmake

@@ -63,6 +63,9 @@ find_program(MDB mdb)
 find_program(XTRABACKUP xtrabackup)
 find_program(S3CMD s3cmd)
 find_program(MINIO minio)
+
 find_program(MYSQL mysql)
 find_program(MYSQLD mysqld)
 find_program(MYSQL_INSTALL_DB mysql_install_db)
+
+find_program(SUDO sudo)

core/cmake/BareosSetVariableDefaults.cmake

@@ -736,3 +736,9 @@ if(autochanger_devices_found)
       "Found these devices for autochanger test: \"${changer-device}\" \"${joined_tape_devices_0}\""
   )
 endif()
+# gfapi-fd
+
+
+if(NOT DEFINED gfapi_fd_testvolume)
+  set(gfapi_fd_testvolume testvol PARENT_SCOPE)
+endif()

systemtests/CMakeLists.txt

@@ -596,8 +596,8 @@ set(SYSTEM_TESTS
     ai-consolidate-ignore-duplicate-job
     bareos
     bareos-acl
+    bscan-bextract-bls
     bconsole-status-client
-    bscan
     client-initiated
     config-dump
     config-syntax-crash
@@ -626,10 +626,13 @@ set(SYSTEM_TESTS
     volume-pruning
 )
 
-if(TARGET gfapi-fd AND gfapi_fd_host)
+if(TARGET gfapi-fd
+   AND gfapi_fd_host
+   AND SUDO
+)
   list(APPEND SYSTEM_TESTS gfapi-fd)
   message(
-    "   gfapi-fd test settings: gfapi_fd_host=${gfapi_fd_host}, gfapi_fd_testvolume=${gfapi_fd_testvolume}"
+    "SUDO: ${SUDO},  gfapi-fd test settings: gfapi_fd_host=${gfapi_fd_host}, gfapi_fd_testvolume=${gfapi_fd_testvolume}"
   )
 else()
   list(APPEND SYSTEM_TESTS_DISABLED gfapi-fd)
@@ -641,6 +644,7 @@ set(SYSTEM_TESTS_DISABLED # initially empty
 # add tests also here that are unreliable they are excluded when running tests
 # during build in jenkins
 set(SYSTEM_TESTS_BROKEN
+    python-restapi
     py3plug-fd-postgres # fails because of psycopg2
     py3plug-fd-local-fileset # Fails with UnicodeEncodeError: 'utf-8' codec
                              # can't encode characters in position 133-142:
@@ -1093,7 +1097,10 @@ foreach(TEST_NAME ${SYSTEM_TESTS})
     )
   endif()
 
-  set_tests_properties(${SYSTEMTEST_PREFIX}${TEST_NAME} PROPERTIES TIMEOUT 120 COST 1.0)
+  set_tests_properties(
+    ${SYSTEMTEST_PREFIX}${TEST_NAME} PROPERTIES TIMEOUT 120 COST 1.0
+                                                SKIP_RETURN_CODE 77
+  )
   math(EXPR BASEPORT "${BASEPORT} + 10")
 
 endforeach()

systemtests/environment.in

@@ -64,6 +64,7 @@ export BAREOS_BSCAN_BINARY=@BSCAN_BINARY@
 export BAREOS_BLS_BINARY=@BLS_BINARY@
 export BAREOS_BCONSOLE_BINARY=@BCONSOLE_BINARY@
 export BAREOS_DBCOPY_BINARY=@BAREOS_DBCOPY_BINARY@
+export BAREOS_BEXTRACT_BINARY=@BEXTRACT_BINARY@
 
 
 
@@ -125,6 +126,7 @@ else
 fi
 export PYTHONPATH=@pythonpath@
 
+gfapi_fd_host=@gfapi_fd_host@
 
 dbHost="@dbHost@"
 test_db_port=@test_db_port@
@@ -150,3 +152,5 @@ POSTGRES_BIN_PATH=@POSTGRES_BIN_PATH@
 MYSQL=@MYSQL@
 MYSQLD=@MYSQLD@
 MYSQL_INSTALL_DB=@MYSQL_INSTALL_DB@
+
+SUDO=@SUDO@

systemtests/scripts/functions

@@ -472,6 +472,26 @@ run_bscan()
    fi
 }
 
+run_bls()
+{
+   print_debug "bls with ${BAREOS_BLS_BINARY}"
+   if test "$debug" -eq 1 ; then
+      "${BAREOS_BLS_BINARY}" -c "${conf}" "$@" </dev/null | tee "${tmp}/bls.out"
+   else
+      "${BAREOS_BLS_BINARY}" -c "${conf}" "$@" </dev/null 2>&1 >"${tmp}/bls.out"
+   fi
+}
+
+run_bextract()
+{
+   print_debug "bextract with ${BAREOS_BEXTRACT_BINARY}"
+   if test "$debug" -eq 1 ; then
+      "${BAREOS_BEXTRACT_BINARY}" -c "${conf}" "$@" </dev/null | tee "${tmp}/bextract.out"
+   else
+      "${BAREOS_BEXTRACT_BINARY}" -c "${conf}" "$@" </dev/null 2>&1 >"${tmp}/bextract.out"
+   fi
+}
+
 run_bscan_db()
 {
   run_bscan -n "$db_name" -u "$db_user" -P "$db_password" "$@"
@@ -636,6 +656,11 @@ check_log()
    return 0
 }
 
+check_recursive_diff_of_dirs()
+{
+   "$rscripts/diff.pl" -s "$1" -d "$2"
+}
+
 check_restore_diff()
 {
    # $dest will be set to

systemtests/tests/bareos/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in

@@ -4,6 +4,14 @@ FileSet {
   Include {
     Options {
       Signature = MD5 # calculate md5 checksum per file
+      fstype = ext2
+      fstype = ext3
+      fstype = ext4
+      fstype = overlay
+      fstype = jfs
+      fstype = ufs
+      fstype = xfs
+      fstype = zfs
     }
    #File = "@sbindir@"
     File=<@tmpdir@/file-list

systemtests/tests/bareos/etc/bareos/bareos-sd.d/device/FileStorage.conf

@@ -8,4 +8,8 @@ Device {
   RemovableMedia = no;
   AlwaysOpen = no;
   Description = "File device. A connecting Director must have the same Name and MediaType."
+  Auto Inflate = both
+  Auto Deflate = both
+  Auto Deflate Algorithm = gzip
+
 }

systemtests/tests/bareos/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in

@@ -6,8 +6,8 @@ Storage {
   # if "Plugin Names" is defined, only the specified plugins will be loaded,
   # otherwise all storage plugins (*-sd.so) from the "Plugin Directory".
   #
-  # Plugin Directory = "@python_plugin_module_src_sd@"
-  # Plugin Names = ""
+  Plugin Directory = "@SD_PLUGINS_DIR_TO_TEST@"
+  Plugin Names = "autoxflate"
   Working Directory =  "@working_dir@"
   Pid Directory =  "@piddir@"
   SD Port = @sd_port@

systemtests/tests/bareos/testrunner

@@ -46,8 +46,8 @@ messages
 @#
 @$out $tmp/log2.out
 wait
-restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done
-yes
+messages
+restore client=bareos-fd fileset=SelfTest where=$tmp/bareos-restores select all done yes
 wait
 messages
 quit

systemtests/tests/bscan-bextract-bls/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in

@@ -0,0 +1,5 @@
+Job {
+  Name = "backup-bareos-fd"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+}

systemtests/tests/bscan/testrunner → systemtests/tests/bscan-bextract-bls/testrunner

@@ -49,6 +49,39 @@ END_OF_DATA
 
 run_bconsole "$tmp/bconcmds"
 
+run_bls -vv -V TestVolume001 FileStorage
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "bls exit code: $ret"
+  stop_bareos
+  exit $ret
+fi
+
+if ! grep -q "weird-files/dangling-link" "$tmp/bls.out"; then
+  echo 'Expected string \"weird-files/dangling-link\" not found in  bls output'
+  stop_bareos
+  exit 1
+fi
+
+
+rm -rf "$tmp/bareos-restores"
+mkdir -p  "$tmp/bareos-restores"
+run_bextract -vv -V TestVolume001 FileStorage "$tmp/bareos-restores"
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "bextract exit code: $ret"
+  stop_bareos
+  exit $ret
+else
+  find "$tmp/bareos-restores"
+  check_restore_diff "${BackupDirectory}" # check extracted files
+  mv "$tmp/bareos-restores" "$tmp/bextract-restores" # move extracted files away
+  mkdir -p  "$tmp/bareos-restores"
+fi
+
+
+
+
 run_bscan_db -vv -s -V TestVolume001 FileStorage
 ret=$?
 if [ $ret -ne 0 ]; then
@@ -88,7 +121,7 @@ if [ "$my_jobid_records" -ne 2 ]; then
   exit 1
 fi
 if [ "$my_jobid_records" -ne "$total_jobid_records" ]; then
-  echo "Volume containes excess session records:"
+  echo "Volume contains excess session records:"
   echo "Got $total_jobid_records in total, only $my_jobid_records mention my original jobid"
   stop_bareos
   exit 1

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in

@@ -0,0 +1,8 @@
+Catalog {
+  Name = MyCatalog
+  #dbdriver = "@DEFAULT_DB_TYPE@"
+  dbdriver = "XXX_REPLACE_WITH_DATABASE_DRIVER_XXX"
+  dbname = "@db_name@"
+  dbuser = "@db_user@"
+  dbpassword = "@db_password@"
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in

@@ -0,0 +1,7 @@
+Client {
+  Name = bareos-fd
+  Description = "Client resource of the Director itself."
+  Address = @hostname@
+  Password = "@fd_password@"          # password for FileDaemon
+  FD PORT = @fd_port@
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in

@@ -0,0 +1,7 @@
+Console {
+  Name = bareos-mon
+  Description = "Restricted console used by tray-monitor to get the status of the director."
+  Password = "@mon_dir_password@"
+  CommandACL = status, .status
+  JobACL = *all*
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in

@@ -0,0 +1,27 @@
+Director {                            # define myself
+  Name = bareos-dir
+  QueryFile = "@scriptdir@/query.sql"
+  Maximum Concurrent Jobs = 10
+  Password = "@dir_password@"         # Console password
+  Messages = Daemon
+  Auditing = yes
+
+  # Enable the Heartbeat if you experience connection losses
+  # (eg. because of your router or firewall configuration).
+  # Additionally the Heartbeat can be enabled in bareos-sd and bareos-fd.
+  #
+  # Heartbeat Interval = 1 min
+
+  # remove comment in next line to load dynamic backends from specified directory
+  Backend Directory = @backenddir@
+
+  # remove comment from "Plugin Directory" to load plugins from specified directory.
+  # if "Plugin Names" is defined, only the specified plugins will be loaded,
+  # otherwise all director plugins (*-dir.so) from the "Plugin Directory".
+  #
+  # Plugin Directory = "@python_plugin_module_src_dir@"
+  # Plugin Names = ""
+  Working Directory =  "@working_dir@"
+  Pid Directory =  "@piddir@"
+  DirPort = @dir_port@
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in

@@ -0,0 +1,11 @@
+FileSet {
+  Name = "Catalog"
+  Description = "Backup the catalog dump and Bareos configuration files."
+  Include {
+    Options {
+      signature = MD5
+    }
+    File = "@working_dir@/@db_name@.sql" # database dump
+    File = "@confdir@"                   # configuration
+  }
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in

@@ -0,0 +1,10 @@
+FileSet {
+  Name = "SelfTest"
+  Description = "fileset just to backup some files for selftest"
+  Include {
+    Options {
+      Signature = MD5 # calculate md5 checksum per file
+    }
+    Plugin = "gfapi:volume=gluster\\://@gfapi_fd_host@/@gfapi_fd_testvolume@:"`
+  }
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in

@@ -0,0 +1,20 @@
+Job {
+  Name = "BackupCatalog"
+  Description = "Backup the catalog database (after the nightly save)"
+  JobDefs = "DefaultJob"
+  Level = Full
+  FileSet="Catalog"
+
+  # This creates an ASCII copy of the catalog
+  # Arguments to make_catalog_backup.pl are:
+  #  make_catalog_backup.pl <catalog-name>
+  RunBeforeJob = "@scriptdir@/make_catalog_backup.pl MyCatalog"
+
+  # This deletes the copy of the catalog
+  RunAfterJob  = "@scriptdir@/delete_catalog_backup"
+
+  # This sends the bootstrap via mail for disaster recovery.
+  # Should be sent to another system, please change recipient accordingly
+  Write Bootstrap = "|@bindir@/bsmtp -h @smtp_host@ -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" @job_email@" # (#01)
+  Priority = 11                   # run after main backup
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in

@@ -0,0 +1,11 @@
+Job {
+  Name = "RestoreFiles"
+  Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..."
+  Type = Restore
+  Client = bareos-fd
+  FileSet = SelfTest
+  Storage = File
+  Pool = Incremental
+  Messages = Standard
+  Where = @tmp@/bareos-restores
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf

@@ -0,0 +1,5 @@
+Job {
+  Name = "backup-bareos-fd"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+}

systemtests/tests/gfapi-fd/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in

@@ -0,0 +1,15 @@
+JobDefs {
+  Name = "DefaultJob"
+  Type = Backup
+  Level = Incremental
+  Client = bareos-fd
+  FileSet = "SelfTest"
+  Storage = File
+  Messages = Standard
+  Pool = Incremental
+  Priority = 10
+  Write Bootstrap = "@working_dir@/%c.bsr"
+  Full Backup Pool = Full                  # write Full Backups into "Full" Pool
+  Differential Backup Pool = Differential  # write Diff Backups into "Differential" Pool
+  Incremental Backup Pool = Incremental    # write Incr Backups into "Incremental" Pool
+}