added tests for filesets with multiple blocks

DartTestfile.txt.in

@@ -50,6 +50,8 @@ ADD_TEST(disk:incremental-2media "@regressdir@/tests/incremental-2media")
 ADD_TEST(disk:incremental-test "@regressdir@/tests/incremental-test")
 ADD_TEST(disk:maxbytes-test "@regressdir@/tests/maxbytes-test")
 ADD_TEST(disk:maxtime-test "@regressdir@/tests/maxtime-test")
+ADD_TEST(disk:fileset-multiple-include-blocks "@regressdir@/tests/fileset-multiple-include-blocks")
+ADD_TEST(disk:fileset-multiple-options-blocks "@regressdir@/tests/fileset-multiple-options-blocks")
 SET_TESTS_PROPERTIES(disk:maxtime-test PROPERTIES
    PASS_REGULAR_EXPRESSION "Fatal error")
 ADD_TEST(disk:maxruntime-test "@regressdir@/tests/maxruntime-test")

configs/fileset-multiple-include-blocks/bareos-dir.d/fileset/FS_TESTJOB.conf.in

@@ -0,0 +1,31 @@
+FileSet {
+  Name = FS_TESTJOB
+  Include {
+     Options {
+        signature = MD5
+     }
+     File=@tmpdir@/data/data1
+  }
+  Include {
+     Options {
+        signature = MD5
+        compression = GZIP
+     }
+     File=@tmpdir@/data/data2
+  }
+  Include {
+     Options {
+        signature = MD5
+        compression = GZIP9
+     }
+     File=@tmpdir@/data/data3
+  }
+  Include {
+     Options {
+        signature = MD5
+        compression = LZ4
+     }
+     File=@tmpdir@/data/data4
+  }
+}
+

configs/fileset-multiple-options-blocks/bareos-dir.d/fileset/FS_TESTJOB.conf.in

@@ -0,0 +1,38 @@
+FileSet {
+  Name = FS_TESTJOB
+  Include {
+
+     Options {
+        exclude = yes
+        signature=MD5
+        wild=@tmpdir@/data/*1
+     }
+     Options {
+        signature=MD5
+        compression = GZIP
+        wild=@tmpdir@/data/*2
+     }
+     Options {
+        signature=MD5
+        compression = GZIP9
+        wild=@tmpdir@/data/*3
+     }
+     Options {
+        signature=MD5
+        compression = LZ4
+        wild=@tmpdir@/data/*4
+     }
+     Options {
+        exclude = yes
+        wild=@tmpdir@/data/*5
+     }
+     Options {
+        signature=MD5
+        compression = GZIP9
+        wild=@tmpdir@/data/*6
+     }
+
+     File=@tmpdir@/data/
+  }
+}
+

scripts/functions

@@ -175,6 +175,31 @@ check_files_written()
     fi
 }
 
+
+
+check_compression()
+{
+   local STORAGE=${1}
+   local VOLUME=${2}
+   # JobId is not yet evaluated
+   local JOBID=${3}
+   local FILENAME=${4}
+   local COMPRESSION=${5:-'GZIP'}
+   local COMPRESSION_LEVEL="$6"
+
+   local COMPRESSION_DESCRIPTION="${COMPRESSION}"
+   if [ "${COMPRESSION_LEVEL}" ]; then
+      COMPRESSION_DESCRIPTION="${COMPRESSION}, level=${COMPRESSION_LEVEL}"
+   fi
+
+   print_debug "Is ${FILENAME} compressed with ${COMPRESSION_DESCRIPTION} ?"
+   if ! ${bin}/bls "${STORAGE}" -V "${VOLUME}" -v | grep -A1 "| ${FILENAME}$" | grep -i "| ${COMPRESSION_DESCRIPTION}, "; then
+      set_error "Use of compression algorithm ${COMPRESSION_DESCRIPTION} in job=${JOBID}, file=${FILENAME} not detected."
+   fi
+}
+
+
+
 ################################################################
 # Get information from logs
 get_mig_info()
@@ -374,7 +399,10 @@ check_restore_diff()
    else
       diff -ur ${dest} ${tmp}/bareos-restores/${dest} 2>&1 >/dev/null
    fi
-   dstat=$?
+   result=$?
+   if [ $result -ne 0 -a ${dstat:-0} -eq 0 ]; then
+      dstat=$result
+   fi
 }
 
 check_restore_bin_diff()

tests/fileset-multiple-include-blocks

@@ -0,0 +1,102 @@
+#!/bin/sh
+
+. scripts/functions
+
+#
+# Check to use multiple fileset include blocks.
+# Verify by using different compression per block.
+#
+
+# Define the Name of the test as "TestName".
+# Should be the same as the filename, therefore we use the filename as default.
+TestName="`basename $0`"
+
+# set other test specific variables
+Client=bareos-fd
+JobName=backup-bareos-fd
+Storage=File1
+VolumeName=TestVolume001
+
+# Directory to backup.
+# This example uses a subdirectory of the bareos source/build directory,
+# that contains some file but isn't to large.
+BackupDirectory="${tmp}/data/"
+
+# Remove old configuration, working and tmp files. Setup the database.
+cleanup
+
+# Config files will be copied to required location.
+# Base configuration files come from the
+# configs/BASE/ directory, see
+# configs/BASE/README.txt for an overview of the provides resources.
+# Store your additonal resources to
+# configs/$TestName.
+# It is also possible to overwrite resources from configs/BASE.
+# If you define a full config file (instead of config files for individual resources),
+# only these will be taken.
+copy_configs
+
+for i in 1 2 3 4; do
+    mkdir -p ${tmp}/data/data$i
+    echo "test" > ${tmp}/data/data$i/test.dat
+done
+
+# start the test
+start_test
+
+# ${tmp}/bconcmds lists the bconsole commands executed by "run_bareos"
+cat <<END_OF_DATA >${tmp}/bconcmds
+messages
+show fileset=FS_TESTJOB
+@$out ${tmp}/log1.out w
+label storage=${Storage} volume=${VolumeName} pool=Default
+run job=$JobName storage=${Storage} yes
+wait
+messages
+@#
+@# now do a restore
+@#
+@$out ${tmp}/log2.out w
+restore client=${Client} where=${tmp}/bareos-restores select storage=${Storage}
+unmark *
+mark *
+done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+# Start the bareos daemons
+# and run the bconsole commands from ${tmp}/bconcmds
+# Further bconsole commands can be executed by "run_bconsole".
+run_bareos
+
+# verify that all are terminated
+check_for_zombie_jobs storage=File1 client=${Client}
+
+# stop bareos
+stop_bareos
+
+# check tmp/log1.out and tmp/log2.out for errors
+check_two_logs
+
+# check for differences between original files and restored files
+# gives an error, because top-level data directory is not backuped
+# (and has therefore other permissions)
+check_restore_diff ${BackupDirectory}
+
+# do some manual testing
+if ! [ -d ${BackupDirectory} ]; then
+    set_error "Directory ${BackupDirectory} does not exists any more."
+fi
+
+
+check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data2/test.dat" "gzip"
+check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data3/test.dat" "gzip" "9"
+# Warning: LZ4 compression support requested in fileset but not available on this platform. Disabling ...
+#check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data3/test.dat" "LZ4"
+
+# end tests and check for error codes
+end_test
+

tests/fileset-multiple-options-blocks

@@ -0,0 +1,120 @@
+#!/bin/sh
+
+. scripts/functions
+
+#
+# Check to use multiple fileset option blocks.
+#
+# NOT WORKING:
+#   * using different compression methods per block (with wild).
+#     Only compression option from last option block is evaluated.
+#   * using different signatures per block (with wild).
+#     Only signature option from last option block is evaluated.
+#
+# WORKING:
+#   * using multiple exclude option blocks (with wild).
+#
+
+# Define the Name of the test as "TestName".
+# Should be the same as the filename, therefore we use the filename as default.
+TestName="`basename $0`"
+
+# set other test specific variables
+Client=bareos-fd
+JobName=backup-bareos-fd
+Storage=File1
+VolumeName=TestVolume001
+
+# Directory to backup.
+# This example uses a subdirectory of the bareos source/build directory,
+# that contains some file but isn't to large.
+BackupDirectory="${tmp}/data/"
+
+# Remove old configuration, working and tmp files. Setup the database.
+cleanup
+
+# Config files will be copied to required location.
+# Base configuration files come from the
+# configs/BASE/ directory, see
+# configs/BASE/README.txt for an overview of the provides resources.
+# Store your additonal resources to
+# configs/$TestName.
+# It is also possible to overwrite resources from configs/BASE.
+# If you define a full config file (instead of config files for individual resources),
+# only these will be taken.
+copy_configs
+
+for i in 1 2 3 4 5 6; do
+    mkdir -p ${tmp}/data/data$i
+    echo "test" > ${tmp}/data/data$i/test.dat
+done
+
+# start the test
+start_test
+
+# ${tmp}/bconcmds lists the bconsole commands executed by "run_bareos"
+cat <<END_OF_DATA >${tmp}/bconcmds
+messages
+show fileset=FS_TESTJOB
+@$out ${tmp}/log1.out w
+label storage=${Storage} volume=${VolumeName} pool=Default
+run job=$JobName storage=${Storage} yes
+wait
+messages
+@#
+@# now do a restore
+@#
+@$out ${tmp}/log2.out w
+restore client=${Client} where=${tmp}/bareos-restores select storage=${Storage}
+unmark *
+mark *
+done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+# Start the bareos daemons
+# and run the bconsole commands from ${tmp}/bconcmds
+# Further bconsole commands can be executed by "run_bconsole".
+run_bareos
+
+# verify that all are terminated
+check_for_zombie_jobs storage=File1 client=${Client}
+
+# stop bareos
+stop_bareos
+
+# check tmp/log1.out and tmp/log2.out for errors
+check_two_logs
+
+# check for differences between original files and restored files
+# gives an error, because top-level data directory is not backuped
+# (and has therefore other permissions)
+#check_restore_diff ${BackupDirectory}/data1
+if [ -e ${tmp}/bareos-restores/${BackupDirectory}/data1 ]; then
+    set_error "Directory data1 has been restored, however it should be excluded from the backup."
+fi
+check_restore_diff ${BackupDirectory}/data2
+check_restore_diff ${BackupDirectory}/data3
+check_restore_diff ${BackupDirectory}/data4
+#check_restore_diff ${BackupDirectory}/data5
+if [ -e ${tmp}/bareos-restores/${BackupDirectory}/data5 ]; then
+    set_error "Directory data5 has been restored, however it should be excluded from the backup."
+fi
+check_restore_diff ${BackupDirectory}/data6
+
+# do some manual testing
+if ! [ -d ${BackupDirectory} ]; then
+    set_error "Directory ${BackupDirectory} does not exists any more."
+fi
+
+check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data2/test.dat" "gzip"
+check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data3/test.dat" "gzip" "9"
+# Warning: LZ4 compression support requested in fileset but not available on this platform. Disabling ...
+#check_compression "${Storage}" "${VolumeName}" "1" "${tmp}/data/data3/test.dat" "LZ4"
+
+# end tests and check for error codes
+end_test
+