From 78e17f851ab9e7c3606315f40e063402616bd00b Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 27 Jul 2016 15:49:36 +0200 Subject: [PATCH] Add regression test for always incremental scheme --- DartTestfile.txt.in | 1 + .../bareos-dir.d/job/Consolidate.conf | 9 ++ .../bareos-dir.d/job/RestoreFiles.conf | 11 ++ .../bareos-dir.d/job/backup-bareos-fd.conf | 16 +++ .../bareos-dir.d/pool/AI-Full.conf | 12 ++ .../bareos-dir.d/pool/AI-Incremental.conf | 12 ++ .../bareos-dir.d/pool/AI-Longterm.conf | 11 ++ .../bareos-dir.d/pool/Default.conf | 9 ++ .../bareos-dir.d/pool/Scratch.conf | 5 + .../bareos-dir.d/schedule/Consolidate.conf | 3 + .../bareos-dir.d/storage/File1.conf.in | 14 ++ .../bareos-dir.d/storage/File2.conf.in | 14 ++ .../bareos-sd.d/device/FileStorage.conf.in | 12 ++ .../bareos-sd.d/device/FileStorage2.conf.in | 12 ++ tests/always-incremental-test | 134 ++++++++++++++++++ 15 files changed, 275 insertions(+) create mode 100644 configs/always-incremental-test/bareos-dir.d/job/Consolidate.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/job/RestoreFiles.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/job/backup-bareos-fd.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/pool/AI-Full.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/pool/AI-Incremental.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/pool/AI-Longterm.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/pool/Default.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/pool/Scratch.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/schedule/Consolidate.conf create mode 100644 configs/always-incremental-test/bareos-dir.d/storage/File1.conf.in create mode 100644 configs/always-incremental-test/bareos-dir.d/storage/File2.conf.in create mode 100644 configs/always-incremental-test/bareos-sd.d/device/FileStorage.conf.in create mode 100644 configs/always-incremental-test/bareos-sd.d/device/FileStorage2.conf.in create mode 100755 tests/always-incremental-test diff --git a/DartTestfile.txt.in b/DartTestfile.txt.in index 772dce2..ac47172 100644 --- a/DartTestfile.txt.in +++ b/DartTestfile.txt.in @@ -1,4 +1,5 @@ ADD_TEST(disk:reload-test "@regressdir@/tests/reload-test") +ADD_TEST(disk:always-incremental-test "@regressdir@/tests/always-incremental-test") ADD_TEST(disk:secure-erase-test "@regressdir@/tests/secure-erase-test") ADD_TEST(disk:migrate-sd-sd-test "@regressdir@/tests/migration-volume-sd-sd-test") ADD_TEST(disk:acl-xattr-test "@regressdir@/tests/acl-xattr-test") diff --git a/configs/always-incremental-test/bareos-dir.d/job/Consolidate.conf b/configs/always-incremental-test/bareos-dir.d/job/Consolidate.conf new file mode 100644 index 0000000..59325fb --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/job/Consolidate.conf @@ -0,0 +1,9 @@ +Job { + Name = "Consolidate" + JobDefs = "DefaultJob" + Schedule = Consolidate + Type = Consolidate + Storage = File1 + Pool = AI-Incremental +} + diff --git a/configs/always-incremental-test/bareos-dir.d/job/RestoreFiles.conf b/configs/always-incremental-test/bareos-dir.d/job/RestoreFiles.conf new file mode 100644 index 0000000..c5834a6 --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/job/RestoreFiles.conf @@ -0,0 +1,11 @@ +Job { + Name = "RestoreFiles" + Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..." + Type = Restore + Client = bareos-fd + FileSet = "FS_TESTJOB" + Storage = File1 + Pool = Default + Messages = Standard + Where = /tmp/bareos-restores +} \ No newline at end of file diff --git a/configs/always-incremental-test/bareos-dir.d/job/backup-bareos-fd.conf b/configs/always-incremental-test/bareos-dir.d/job/backup-bareos-fd.conf new file mode 100644 index 0000000..7296c80 --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/job/backup-bareos-fd.conf @@ -0,0 +1,16 @@ +Job { + Name = "backup-bareos-fd" + JobDefs = "DefaultJob" + Client = "bareos-fd" + + Pool = AI-Incremental + Full Backup Pool = AI-Full + + # Always incremental settings + AlwaysIncremental = yes + AlwaysIncrementalJobRetention = 1 seconds + AlwaysIncrementalKeepNumber = 2 + Accurate = yes + +} + diff --git a/configs/always-incremental-test/bareos-dir.d/pool/AI-Full.conf b/configs/always-incremental-test/bareos-dir.d/pool/AI-Full.conf new file mode 100644 index 0000000..84cdb8d --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/pool/AI-Full.conf @@ -0,0 +1,12 @@ +Pool { + Name = AI-Full + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 180 days # How long should the AI-Full Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Full-" # Volumes will be labeled "AI-Full-" + Volume Use Duration = 23h + Storage = File2 + Next Pool = AI-Longterm +} diff --git a/configs/always-incremental-test/bareos-dir.d/pool/AI-Incremental.conf b/configs/always-incremental-test/bareos-dir.d/pool/AI-Incremental.conf new file mode 100644 index 0000000..3a5865f --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/pool/AI-Incremental.conf @@ -0,0 +1,12 @@ +Pool { + Name = AI-Incremental + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 180 days # How long should the AI-Incremental Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Incremental-" # Volumes will be labeled "AI-Incremental-" + Volume Use Duration = 23h + Storage = File1 + Next Pool = AI-Full +} diff --git a/configs/always-incremental-test/bareos-dir.d/pool/AI-Longterm.conf b/configs/always-incremental-test/bareos-dir.d/pool/AI-Longterm.conf new file mode 100644 index 0000000..938b763 --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/pool/AI-Longterm.conf @@ -0,0 +1,11 @@ +Pool { + Name = AI-Longterm + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 180 days # How long should the AI-Longterm Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Longterm-" # Volumes will be labeled "AI-Longterm-" + Volume Use Duration = 23h + Storage = File1 +} diff --git a/configs/always-incremental-test/bareos-dir.d/pool/Default.conf b/configs/always-incremental-test/bareos-dir.d/pool/Default.conf new file mode 100644 index 0000000..9b2d89f --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/pool/Default.conf @@ -0,0 +1,9 @@ +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Maximum Volumes = 20 + Maximum Volume Bytes = 100M +} + diff --git a/configs/always-incremental-test/bareos-dir.d/pool/Scratch.conf b/configs/always-incremental-test/bareos-dir.d/pool/Scratch.conf new file mode 100644 index 0000000..d44c045 --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/pool/Scratch.conf @@ -0,0 +1,5 @@ +Pool { + Name = Scratch + Pool Type = Scratch +} + diff --git a/configs/always-incremental-test/bareos-dir.d/schedule/Consolidate.conf b/configs/always-incremental-test/bareos-dir.d/schedule/Consolidate.conf new file mode 100644 index 0000000..5a212d1 --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/schedule/Consolidate.conf @@ -0,0 +1,3 @@ +Schedule { + Name = "Consolidate" +} diff --git a/configs/always-incremental-test/bareos-dir.d/storage/File1.conf.in b/configs/always-incremental-test/bareos-dir.d/storage/File1.conf.in new file mode 100644 index 0000000..509366b --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/storage/File1.conf.in @@ -0,0 +1,14 @@ +Storage { + Name = File1 + Address = @hostname@ + Password = "@sd_password@" + Device = File1 + Media Type = File + TLS Require = yes + TLS Certificate = "@scriptdir@/tls/bareos-dir.bareos.org-cert.pem" + TLS Key = "@scriptdir@/tls/bareos-dir.bareos.org-key.pem" + TLS CA Certificate File = "@scriptdir@/tls/bareos-ca.pem" + TLS Verify Peer = no + TLS Allowed CN = bareos-sd1.bareos.org + TLS Allowed CN = bareos-sd2.bareos.org +} diff --git a/configs/always-incremental-test/bareos-dir.d/storage/File2.conf.in b/configs/always-incremental-test/bareos-dir.d/storage/File2.conf.in new file mode 100644 index 0000000..f4059ff --- /dev/null +++ b/configs/always-incremental-test/bareos-dir.d/storage/File2.conf.in @@ -0,0 +1,14 @@ +Storage { + Name = File2 + Address = @hostname@ + Password = "@sd_password@" + Device = File2 + Media Type = File + TLS Require = yes + TLS Certificate = "@scriptdir@/tls/bareos-dir.bareos.org-cert.pem" + TLS Key = "@scriptdir@/tls/bareos-dir.bareos.org-key.pem" + TLS CA Certificate File = "@scriptdir@/tls/bareos-ca.pem" + TLS Verify Peer = no + TLS Allowed CN = bareos-sd1.bareos.org + TLS Allowed CN = bareos-sd2.bareos.org +} diff --git a/configs/always-incremental-test/bareos-sd.d/device/FileStorage.conf.in b/configs/always-incremental-test/bareos-sd.d/device/FileStorage.conf.in new file mode 100644 index 0000000..3d5d9c4 --- /dev/null +++ b/configs/always-incremental-test/bareos-sd.d/device/FileStorage.conf.in @@ -0,0 +1,12 @@ +Device { + Name = File1 + Media Type = File + Archive Device = @archivedir@ + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + Description = "File device. A connecting Director must have the same Name and MediaType." +} + diff --git a/configs/always-incremental-test/bareos-sd.d/device/FileStorage2.conf.in b/configs/always-incremental-test/bareos-sd.d/device/FileStorage2.conf.in new file mode 100644 index 0000000..27a0028 --- /dev/null +++ b/configs/always-incremental-test/bareos-sd.d/device/FileStorage2.conf.in @@ -0,0 +1,12 @@ +Device { + Name = File2 + Media Type = File + Archive Device = @archivedir@ + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + Description = "File device. A connecting Director must have the same Name and MediaType." +} + diff --git a/tests/always-incremental-test b/tests/always-incremental-test new file mode 100755 index 0000000..20888f6 --- /dev/null +++ b/tests/always-incremental-test @@ -0,0 +1,134 @@ +#!/bin/sh + +. scripts/functions + +# +# Example regression test. +# +# Use this as a template for your own tests. +# + +# Define the Name of the test as "TestName". +# Should be the same as the filename, therefore we use the filename as default. +TestName="`basename $0`" +#TestName="1-example-test" + +# set other test specific variables +Client=bareos-fd +JobName=backup-bareos-fd + +# Directory to backup. +# This example uses a subdirectory of the bareos source/build directory, +# that contains some file but isn't to large. +BackupDirectory="${cwd}/build/src/tests/" + +# Remove old configuration, working and tmp files. Setup the database. +cleanup + +# Config files will be copied to required location. +# Base configuration files come from the +# configs/BASE/ directory, see +# configs/BASE/README.txt for an overview of the provides resources. +# Store your additonal resources to +# configs/$TestName. +# It is also possible to overwrite resources from configs/BASE. +# If you define a full config file (instead of config files for individual resources), +# only these will be taken. +copy_configs + +# the default fileset FS_TESTJOB backups all file and directories defined in "${tmp}/file-list". +echo "${BackupDirectory}" >${tmp}/file-list + +# start the test +start_test + + +# create a new file and run multiple backups +for file in 1 2 3 4 5 6 7 ; do + +echo file$file > ${BackupDirectory}/file$file + +cat <${tmp}/bconcmds +messages +@$out ${tmp}/log1.out w +run job=$JobName yes +wait +messages +END_OF_DATA +run_bareos +check_for_zombie_jobs storage=File1 client=${Client} +stop_bareos +done + + +cat <${tmp}/bconcmds +@# +@# try to consolidate +@# +@$out ${tmp}/log1.out w +list jobs +run job=Consolidate yes +wait +list jobs +messages +END_OF_DATA + +# Start the bareos daemon +# and run the bconsole commands from ${tmp}/bconcmds +run_bareos + +# verify that all are terminated +check_for_zombie_jobs storage=File1 client=${Client} + +# stop bareos +stop_bareos +cat <${tmp}/bconcmds +@# +@# now do a restore +@# +@$out ${tmp}/log2.out w +restore client=${Client} where=${tmp}/bareos-restores select +unmark * +mark * +done +yes +wait +messages +quit +END_OF_DATA + +# Start the bareos daemon +# and run the bconsole commands from ${tmp}/bconcmds +run_bareos + +# verify that all are terminated +check_for_zombie_jobs storage=File1 client=${Client} + +# stop bareos +stop_bareos + +# check tmp/log1.out and tmp/log2.out for errors +check_two_logs + +# check for differences between original files and restored files +check_restore_diff ${BackupDirectory} + +# do some manual testing +if ! [ -d ${BackupDirectory} ]; then + set_error "Directory ${BackupDirectory} does not exists any more." +fi + + +# check if consolidation worked as expected +grep "purged JobIds 1,2,3,4,5 as they were consolidated into Job 9" tmp/log1.out || set_error "consolidation did not consolidate jobs that were expected" + +# check if the expected jobids were chosen for restore +grep "You have selected the following JobIds: 9,6,7" tmp/log2.out || set_error "other jobs than expected were chosen for restore" + +# check if all files were restored +for file in 1 2 3 4 5 6 7 ; do + grep $file ${tmp}/bareos-restores/${BackupDirectory}/file$file || set_error "file $file was not recovered" +done + +# end tests and check for error codes +end_test