From 9939bc8b1e25ba403e40289be0eb4535418f383f Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 25 Jun 2013 10:22:24 +0200 Subject: [PATCH] Added regression test for sd-sd copy/migration --- DartTestfile.txt.in | 1 + scripts/bareos-dir-migration-sd-sd.conf.in | 271 +++++++++++++++++++++ scripts/bareos-sd-migration-sd-sd1.conf.in | 70 ++++++ scripts/bareos-sd-migration-sd-sd2.conf.in | 70 ++++++ scripts/create_sed | 2 + tests/migration-volume-sd-sd-test | 118 +++++++++ 6 files changed, 532 insertions(+) create mode 100644 scripts/bareos-dir-migration-sd-sd.conf.in create mode 100644 scripts/bareos-sd-migration-sd-sd1.conf.in create mode 100644 scripts/bareos-sd-migration-sd-sd2.conf.in create mode 100755 tests/migration-volume-sd-sd-test diff --git a/DartTestfile.txt.in b/DartTestfile.txt.in index 1000932..2e51056 100644 --- a/DartTestfile.txt.in +++ b/DartTestfile.txt.in @@ -1,3 +1,4 @@ +ADD_TEST(disk:migrate-sd-sd-test "@regressdir@/tests/migration-volume-sd-sd-test") ADD_TEST(disk:acl-xattr-test "@regressdir@/tests/acl-xattr-test") ADD_TEST(disk:action-on-purge-test "@regressdir@/tests/action-on-purge-test") ADD_TEST(disk:accurate-test "@regressdir@/tests/accurate-test") diff --git a/scripts/bareos-dir-migration-sd-sd.conf.in b/scripts/bareos-dir-migration-sd-sd.conf.in new file mode 100644 index 0000000..7a3b3f5 --- /dev/null +++ b/scripts/bareos-dir-migration-sd-sd.conf.in @@ -0,0 +1,271 @@ +# +# Default Bareos Director Configuration file +# +# The only thing that MUST be changed is to add one or more +# file or directory names in the Include directive of the +# FileSet resource. +# +# You might also want to change the default email address +# from root to your address. See the "mail" and "operator" +# directives in the Messages resource. +# + +Director { # define myself + Name = @hostname@-dir + DIRPort = @dirport@ # where we listen for UA connections + QueryFile = "@scriptdir@/query.sql" + WorkingDirectory = "@working_dir@" + PidDirectory = "@piddir@" + Maximum Concurrent Jobs = 4 + Password = "pNvX1WiXnwv2C/F7E52LGvw6rKjbbPvu2kyuPa9pVaL3" + Messages = Daemon +} + +# +# Define the main nightly save backup job +# By default, this job will back up to disk in @tmpdir@ +Job { + Name = "NightlySave" + Type = Backup + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Maximum Concurrent Jobs = 4 + SpoolData = yes +# Allow Duplicate Jobs = No +# Prefer Mounted Volumes = no +} + +Job { + Name = "migrate-job" + Type = Migrate + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = DiskChanger + Write Bootstrap = "@working_dir@/migratejob.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = Job + Selection Pattern = ".*Save" +# Allow Duplicate Jobs = No +} + +Job { + Name = "migrate-volume" + Type = Migrate + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = DiskChanger + Write Bootstrap = "@working_dir@/migratevol.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = Volume + Selection Pattern = "File*" +# Allow Duplicate Jobs = No +} + +Job { + Name = "copy-uncopied" + Type = Copy + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = DiskChanger + Write Bootstrap = "@working_dir@/migratevol.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = PoolUncopiedJobs +# Allow Duplicate Jobs = No +} + +Job { + Name = "migrate-occupancy" + Type = Migrate + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = DiskChanger + Write Bootstrap = "@working_dir@/migratejob.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = PoolOccupancy +# Allow Duplicate Jobs = No +} + +Job { + Name = "migrate-time" + Type = Migrate + Level = Full + Client=@hostname@-fd + FileSet="Full Set" + Messages = Standard + Storage = DiskChanger + Write Bootstrap = "@working_dir@/migratejob.bsr" + Pool = Default + Maximum Concurrent Jobs = 4 + Selection Type = PoolTime +# Allow Duplicate Jobs = No +} + + + + +# Standard Restore template, to be changed by Console program +Job { + Name = "RestoreFiles" + Type = Restore + Client=@hostname@-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Where = @tmpdir@/bareos-restores +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { Options { signature=MD5 } + File = <@tmpdir@/file-list + } +} + + +# +# When to do the backups, full backup on first sunday of the month, +# differential (i.e. incremental since full) every other sunday, +# and incremental backups other days +Schedule { + Name = "WeeklyCycle" + Run = Full 1st sun at 1:05 + Run = Differential 2nd-5th sun at 1:05 + Run = Incremental mon-sat at 1:05 +} + +# Client (File Services) to backup +Client { + Name = @hostname@-fd + Address = @hostname@ + FDPort = @fdport@ + Catalog = MyCatalog + Password = "xevrjURYoCHhn26RaJoWbeWXEY/a3VqGKp/37tgWiuHc" # password for FileDaemon + File Retention = 30d # 30 days + Job Retention = 180d # six months + AutoPrune = yes # Prune expired Jobs/Files + Maximum Concurrent Jobs = 4 +} + +# Definition of DDS Virtual tape disk storage device +Storage { + Name = DiskChanger + Address = @hostname@ # N.B. Use a fully qualified name here + SDPort = @sdport2@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = DiskChanger # must be same as Device in Storage daemon + Media Type = DiskChangerMedia # must be same as MediaType in Storage daemon + Maximum Concurrent Jobs = 4 + Autochanger = yes +} + +# Definition of file storage device +Storage { + Name = File + Address = @hostname@ + SDPort = @sdport@ + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" # password for Storage daemon + Device = FileStorage + Media Type = File + Maximum Concurrent Jobs = 5 +} + + +# Generic catalog service +Catalog { + Name = MyCatalog + @libdbi@ + dbname = @db_name@; user = @db_user@; password = "@db_password@" +} + +# Reasonable message delivery -- send most everything to email address +# and to the console +Messages { + Name = Standard + mailcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bareos regression Job %j\) %r\" -s \"Regression: %t %e of %c %l\" %r" + operatorcommand = "@sbindir@/bsmtp -h localhost -f \"\(Bareos regression Job %j\) %r\" -s \"Regression: Intervention needed for %j\" %r" +# MailOnError = @job_email@ = all, !terminate +# operator = @job_email@ = mount + console = all, !skipped, !terminate, !restored +# +# WARNING! the following will create a file that you must cycle from +# time to time as it will grow indefinitely. However, it will +# also keep all your messages if the scroll off the console. +# + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + +# +# Message delivery for daemon messages (no job). +Messages { + Name = Daemon + mailcommand = "@sbindir@/bsmtp -h @smtp_host@ -f \"\(Bareos regression\) %r\" -s \"Regression daemon message\" %r" +# mail = @job_email@ = all, !skipped + console = all, !skipped, !saved + append = "@working_dir@/log" = all, !skipped + catalog = all, !skipped +} + + +# Default pool definition +Pool { + Name = Default + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + NextPool = Full + Storage = File + Migration High Bytes = 40M + Migration Low Bytes = 20M + Migration Time = 1 +} + +Pool { + Name = Special + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + NextPool = Full + Storage = File + Migration High Bytes = 40M + Migration Low Bytes = 20M + Migration Time = 5h +} + +Pool { + Name = Full + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365d # one year + Storage = DiskChanger +} + +Pool { + Name = Scratch + Recycle = yes + AutoPrune = yes + Pool Type = Scratch + RecyclePool = Scratch +} diff --git a/scripts/bareos-sd-migration-sd-sd1.conf.in b/scripts/bareos-sd-migration-sd-sd1.conf.in new file mode 100644 index 0000000..ba9bdd2 --- /dev/null +++ b/scripts/bareos-sd-migration-sd-sd1.conf.in @@ -0,0 +1,70 @@ +# +# Default Bareos Storage Daemon Configuration file +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd + SDPort = @sdport@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +# +# Devices supported by this Storage daemon +# To connect, the Director's bareos-dir.conf must have the +# same Name and MediaType. +# +#Autochanger { +# Name = DiskChanger +# Changer Device = @disk_drive@/conf +# Changer Command ="@scriptdir@/disk-changer %c %o %S %a %d" +# Device = Drive-0 +#} + +#Device { +# Name = Drive-0 # +# Device Type = File +# Media Type = DiskChangerMedia +# Archive Device = @disk_drive@/drive0 +# AutomaticMount = yes; # when device opened, read it +# Autochanger = yes +# Drive Index = 0 +# AlwaysOpen = yes; +# RemovableMedia = yes; +## Maximum File Size = 1000000 +#} + +Device { + Name = FileStorage + Media Type = File + Archive Device = @tmpdir@ + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = Yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; +} + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} diff --git a/scripts/bareos-sd-migration-sd-sd2.conf.in b/scripts/bareos-sd-migration-sd-sd2.conf.in new file mode 100644 index 0000000..a276848 --- /dev/null +++ b/scripts/bareos-sd-migration-sd-sd2.conf.in @@ -0,0 +1,70 @@ +# +# Default Bareos Storage Daemon Configuration file +# +# You may need to change the name of your tape drive +# on the "Archive Device" directive in the Device +# resource. If you change the Name and/or the +# "Media Type" in the Device resource, please ensure +# that dird.conf has corresponding changes. +# + +Storage { # definition of myself + Name = @hostname@-sd2 + SDPort = @sdport2@ # Director's port + WorkingDirectory = "@working_dir@" + Pid Directory = "@piddir@" + Subsys Directory = "@subsysdir@" +} + +# +# List Directors who are permitted to contact Storage daemon +# +Director { + Name = @hostname@-dir + Password = "ccV3lVTsQRsdIUGyab0N4sMDavui2hOBkmpBU0aQKOr9" +} + +# +# Devices supported by this Storage daemon +# To connect, the Director's bareos-dir.conf must have the +# same Name and MediaType. +# +Autochanger { + Name = DiskChanger + Changer Device = @disk_drive@/conf + Changer Command ="@scriptdir@/disk-changer %c %o %S %a %d" + Device = Drive-0 +} + +Device { + Name = Drive-0 # + Device Type = File + Media Type = DiskChangerMedia + Archive Device = @disk_drive@/drive0 + AutomaticMount = yes; # when device opened, read it + Autochanger = yes + Drive Index = 0 + AlwaysOpen = yes; + RemovableMedia = yes; +# Maximum File Size = 1000000 +} + +#Device { +# Name = FileStorage +# Media Type = File +# Archive Device = @tmpdir@ +# LabelMedia = yes; # lets Bareos label unlabeled media +# Random Access = Yes; +# AutomaticMount = yes; # when device opened, read it +# RemovableMedia = no; +# AlwaysOpen = no; +#} + +# +# Send all messages to the Director, +# mount messages also are sent to the email address +# +Messages { + Name = Standard + director = @hostname@-dir = all, !terminate +} diff --git a/scripts/create_sed b/scripts/create_sed index 143aea8..e800b09 100755 --- a/scripts/create_sed +++ b/scripts/create_sed @@ -17,6 +17,7 @@ srcdir=${BAREOS_SOURCE} dirport=${BASEPORT} fdport=`expr ${BASEPORT} '+' 1` sdport=`expr ${BASEPORT} '+' 2` +sdport2=`expr ${BASEPORT} '+' 3` db_name=${db_name:-"regress"} db_user=${db_user:-"regress"} db_password=${db_password:-""} @@ -61,6 +62,7 @@ echo "s%@hostname2_password@%${hostname2_password}%g" >>${out} echo "s%@hostname3_password@%${hostname3_password}%g" >>${out} echo "s%@dirport@%${dirport}%g" >>${out} echo "s%@sdport@%${sdport}%g" >>${out} +echo "s%@sdport2@%${sdport2}%g" >>${out} echo "s%@fdport@%${fdport}%g" >>${out} echo "s%@regressdir@%${cwd}%g" >>${out} echo "s%@dbengine@%${dbengine}%g" >>${out} diff --git a/tests/migration-volume-sd-sd-test b/tests/migration-volume-sd-sd-test new file mode 100755 index 0000000..ae70663 --- /dev/null +++ b/tests/migration-volume-sd-sd-test @@ -0,0 +1,118 @@ +#!/bin/sh +# +# Run a backup of the Bareos build directory on two Volumes +# then migrate it to another device. +# +# This script uses the virtual disk autochanger +# +TestName="migration-volume-test" +JobName=MigVolBackup +. scripts/functions + +scripts/cleanup + +#scripts/copy-migration-confs +/bin/cp -f scripts/bareos-dir-migration-sd-sd.conf bin/bareos-dir.conf + +# create start script for second sd +/bin/cp -f bin/bareos-ctl-sd bin/bareos-ctl-sd2 +/bin/sed 's/bareos-sd.conf/bareos-sd2.conf/g' < bin/bareos-ctl-sd > bin/bareos-ctl-sd2 + +/bin/cp -f scripts/bareos-sd-migration-sd-sd1.conf bin/bareos-sd.conf +/bin/cp -f scripts/bareos-sd-migration-sd-sd2.conf bin/bareos-sd2.conf + +/bin/cp -f scripts/test-bareos-fd.conf bin/bareos-fd.conf +/bin/cp -f scripts/test-console.conf bin/bconsole.conf +/bin/cp -f scripts/test-console.conf bin/bat.conf + +scripts/prepare-disk-changer + +echo "${cwd}/build" >${cwd}/tmp/file-list +#cp ${cwd}/bin/bareos-sd.conf ${cwd}/tmp/1 +#sed "s%# Maximum File Size% Maximum File Size%" ${cwd}/tmp/1 >${cwd}/bin/bareos-sd.conf + +change_jobname NightlySave $JobName +start_test + +# +# Note, we first backup into Pool Default, +# then Migrate into Pool Full. +# Pool Default uses Storage=File +# Pool Full uses Storage=DiskChanger + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=FileVolume001 Pool=Default +label storage=File volume=FileVolume002 Pool=Default +update Volume=FileVolume001 MaxVolBytes=3000000 pool=Default +label storage=DiskChanger volume=ChangerVolume001 slot=1 Pool=Full drive=0 +label storage=DiskChanger volume=ChangerVolume002 slot=2 Pool=Full drive=0 +@# +run job=$JobName yes +wait +run job=$JobName yes +wait +update volume=FileVolume001 VolStatus=Used +update volume=FileVolume002 VolStatus=Used +@#list volumes +@#list jobs +@# should migrate two jobs +@# setdebug level=11 dir +@echo "Run migrate-volume Job" +@#setdebug level=100 dir +run job=migrate-volume yes +@sleep 5 +list jobs +llist jobid=1 +llist jobid=2 +llist jobid=3 +llist jobid=4 +llist jobid=5 +status storage=DiskChanger +wait +list volumes +list jobs +llist jobid=1 +llist jobid=2 +llist jobid=3 +llist jobid=4 +llist jobid=5 +sql +select * from JobMedia where JobId=1; + +@# Now run a second Migration job, which should do nothing +run job=migrate-volume yes +wait +list jobs +@# ensure we don't pickup old backup by purging volume +@# purge volume=FileVolume001 +@# purge volume=FileVolume002 +@echo "Now do a restore" +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +restore where=${cwd}/tmp/bareos-restores select storage=DiskChanger +unmark * +mark * +done +yes +wait +messages +quit +END_OF_DATA + +bin/bareos-ctl-sd2 start +run_bareos + +check_for_zombie_jobs storage=File +bin/bareos-ctl-sd2 stop +stop_bareos + +check_two_logs +check_restore_diff +end_test