systemtests: add autoxflate + copy test

systemtests/tests/CMakeLists.txt

@@ -21,6 +21,7 @@ message("Adding systemtests from ${CMAKE_CURRENT_SOURCE_DIR}")
 add_subdirectory(acl)
 add_subdirectory(always-incremental-consolidate)
 add_subdirectory(autochanger)
+add_subdirectory(autoxflate)
 add_subdirectory(bareos-basic)
 add_subdirectory(bareos-acl)
 add_subdirectory(bconsole-basic)

systemtests/tests/autoxflate/CMakeLists.txt

@@ -0,0 +1,21 @@
+#   BAREOS® - Backup Archiving REcovery Open Sourced
+#
+#   Copyright (C) 2021-2022 Bareos GmbH & Co. KG
+#
+#   This program is Free Software; you can redistribute it and/or
+#   modify it under the terms of version three of the GNU Affero General Public
+#   License as published by the Free Software Foundation and included
+#   in the file LICENSE.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+#   02110-1301, USA.
+
+get_filename_component(BASENAME ${CMAKE_CURRENT_BINARY_DIR} NAME)
+create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME})

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in

@@ -0,0 +1,6 @@
+Catalog {
+  Name = MyCatalog
+  dbname = "@db_name@"
+  dbuser = "@db_user@"
+  dbpassword = "@db_password@"
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in

@@ -0,0 +1,7 @@
+Client {
+  Name = bareos-fd
+  Description = "Client resource of the Director itself."
+  Address = @hostname@
+  Password = "@fd_password@"          # password for FileDaemon
+  FD PORT = @fd_port@
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in

@@ -0,0 +1,23 @@
+Director {                            # define myself
+  Name = bareos-dir
+  QueryFile = "@scriptdir@/query.sql"
+  Maximum Concurrent Jobs = 10
+  Password = "@dir_password@"         # Console password
+  Messages = Daemon
+  Auditing = yes
+
+  # Enable the Heartbeat if you experience connection losses
+  # (eg. because of your router or firewall configuration).
+  # Additionally the Heartbeat can be enabled in bareos-sd and bareos-fd.
+  #
+  # Heartbeat Interval = 1 min
+
+  # remove comment from "Plugin Directory" to load plugins from specified directory.
+  # if "Plugin Names" is defined, only the specified plugins will be loaded,
+  # otherwise all director plugins (*-dir.so) from the "Plugin Directory".
+  #
+  # Plugin Directory = "@python_plugin_module_src_dir@"
+  # Plugin Names = ""
+  Working Directory =  "@working_dir@"
+  DirPort = @dir_port@
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in

@@ -0,0 +1,11 @@
+FileSet {
+  Name = "Catalog"
+  Description = "Backup the catalog dump and Bareos configuration files."
+  Include {
+    Options {
+      Signature = XXH128
+    }
+    File = "@working_dir@/@db_name@.sql" # database dump
+    File = "@confdir@"                   # configuration
+  }
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in

@@ -0,0 +1,13 @@
+FileSet {
+  Name = "SelfTest"
+  Description = "fileset just to backup some files for selftest"
+  Include {
+    Options {
+      Signature = XXH128
+      HardLinks = Yes
+      Compression = LZ4  # Compress data as it's sent to Director
+    }
+   #File = "@sbindir@"
+    File=<@tmpdir@/file-list
+  }
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in

@@ -0,0 +1,11 @@
+Job {
+  Name = "RestoreFiles"
+  Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..."
+  Type = Restore
+  Client = bareos-fd
+  FileSet = SelfTest
+  Storage = File
+  Pool = Incremental
+  Messages = Standard
+  Where = @tmp@/bareos-restores
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf

@@ -0,0 +1,5 @@
+Job {
+  Name = "backup-bareos-fd"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/job/copy.conf

@@ -0,0 +1,19 @@
+Job {
+  Name = "copy"
+  Type = Copy
+  Pool = Full
+  Selection Type = Volume
+  Selection Pattern = "."
+  Client = "bareos-fd"
+  Messages = Standard
+  Run Script {
+    Runs On Client = No
+    Runs When = Before
+    Command = "echo 'prevjobid=%O newjobid=%N'"
+  }
+  Run Script {
+    Runs On Client = No
+    Runs When = After
+    Command = "echo 'prevjobid=%O newjobid=%N'"
+  }
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in

@@ -0,0 +1,15 @@
+JobDefs {
+  Name = "DefaultJob"
+  Type = Backup
+  Level = Incremental
+  Client = bareos-fd
+  FileSet = "SelfTest"
+  Storage = File
+  Messages = Standard
+  Pool = Incremental
+  Priority = 10
+  Write Bootstrap = "@working_dir@/%c.bsr"
+  Full Backup Pool = Full                  # write Full Backups into "Full" Pool
+  Differential Backup Pool = Differential  # write Diff Backups into "Differential" Pool
+  Incremental Backup Pool = Incremental    # write Incr Backups into "Incremental" Pool
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/messages/Daemon.conf.in

@@ -0,0 +1,7 @@
+Messages {
+  Name = Daemon
+  Description = "Message delivery for daemon messages (no job)."
+  console = all, !skipped, !saved, !audit
+  append = "@logdir@/bareos.log" = all, !skipped, !audit
+  append = "@logdir@/bareos-audit.log" = audit
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/messages/Standard.conf.in

@@ -0,0 +1,7 @@
+Messages {
+  Name = Standard
+  Description = "Reasonable message delivery -- send most everything to email address and to the console."
+  console = all, !skipped, !saved, !audit
+  append = "@logdir@/bareos.log" = all, !skipped, !saved, !audit
+  catalog = all, !skipped, !saved, !audit
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/pool/Differential.conf

@@ -0,0 +1,10 @@
+Pool {
+  Name = Differential
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 90 days          # How long should the Differential Backups be kept? (#09)
+  Maximum Volume Bytes = 10G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Differential-"      # Volumes will be labeled "Differential-<volume-id>"
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/pool/Full.conf

@@ -0,0 +1,11 @@
+Pool {
+  Name = Full
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 1 sec         # How long should the Full Backups be kept? (#06)
+  Maximum Volume Bytes = 50G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Full-"              # Volumes will be labeled "Full-<volume-id>"
+  Next Pool = FullCopy
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/pool/FullCopy.conf

@@ -0,0 +1,11 @@
+Pool {
+  Name = FullCopy
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 1 sec         # How long should the Full Backups be kept? (#06)
+  Maximum Volume Bytes = 50G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Full-"              # Volumes will be labeled "Full-<volume-id>"
+  Storage = "File2"
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/pool/Incremental.conf

@@ -0,0 +1,10 @@
+Pool {
+  Name = Incremental
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 30 days          # How long should the Incremental Backups be kept?  (#12)
+  Maximum Volume Bytes = 1G           # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Incremental-"       # Volumes will be labeled "Incremental-<volume-id>"
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/pool/Scratch.conf

@@ -0,0 +1,4 @@
+Pool {
+  Name = Scratch
+  Pool Type = Scratch
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/profile/operator.conf

@@ -0,0 +1,18 @@
+Profile {
+   Name = operator
+   Description = "Profile allowing normal Bareos operations."
+
+   Command ACL = !.bvfs_clear_cache, !.exit, !.sql
+   Command ACL = !configure, !create, !delete, !purge, !prune, !sqlquery, !umount, !unmount
+   Command ACL = *all*
+
+   Catalog ACL = *all*
+   Client ACL = *all*
+   FileSet ACL = *all*
+   Job ACL = *all*
+   Plugin Options ACL = *all*
+   Pool ACL = *all*
+   Schedule ACL = *all*
+   Storage ACL = *all*
+   Where ACL = *all*
+}

systemtests/tests/autoxflate/etc/bareos/bareos-dir.d/storage/File.conf.in

@@ -0,0 +1,17 @@
+Storage {
+  Name = File
+  Address = @hostname@
+  Password = "@sd_password@"
+  Device = FileStorage
+  Media Type = File
+  SD Port = @sd_port@
+}
+
+Storage {
+  Name = File2
+  Address = @hostname@
+  Password = "@sd_password@"
+  Device = FileStorage2
+  Media Type = File
+  SD Port = @sd_port@
+}

systemtests/tests/autoxflate/etc/bareos/bareos-fd.d/client/myself.conf.in

@@ -0,0 +1,15 @@
+Client {
+  Name = @basename@-fd
+  Maximum Concurrent Jobs = 20
+
+  # remove comment from "Plugin Directory" to load plugins from specified directory.
+  # if "Plugin Names" is defined, only the specified plugins will be loaded,
+  # otherwise all filedaemon plugins (*-fd.so) from the "Plugin Directory".
+  #
+  # Plugin Directory = "@python_plugin_module_src_fd@"
+  # Plugin Names = ""
+
+  Working Directory =  "@working_dir@"
+  FD Port = @fd_port@
+
+}

systemtests/tests/autoxflate/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in

@@ -0,0 +1,5 @@
+Director {
+  Name = bareos-dir
+  Password = "@fd_password@"
+  Description = "Allow the configured Director to access this file daemon."
+}

systemtests/tests/autoxflate/etc/bareos/bareos-fd.d/messages/Standard.conf

@@ -0,0 +1,5 @@
+Messages {
+  Name = Standard
+  Director = bareos-dir = all, !skipped, !restored
+  Description = "Send relevant messages to the Director."
+}

systemtests/tests/autoxflate/etc/bareos/bareos-sd.d/device/FileStorage.conf

@@ -0,0 +1,24 @@
+Device {
+  Name = FileStorage
+  Media Type = File
+  Archive Device = storage
+  LabelMedia = yes;                   # lets Bareos label unlabeled media
+  Random Access = yes;
+  AutomaticMount = yes;               # when device opened, read it
+  RemovableMedia = no;
+  AlwaysOpen = no;
+  Description = "File device. A connecting Director must have the same Name and MediaType."
+}
+Device {
+  Name = FileStorage2
+  Media Type = File
+  Archive Device = storage
+  LabelMedia = yes;                   # lets Bareos label unlabeled media
+  Random Access = yes;
+  AutomaticMount = yes;               # when device opened, read it
+  RemovableMedia = no;
+  AlwaysOpen = no;
+  Description = "File device. A connecting Director must have the same Name and MediaType."
+
+  Auto Inflate = out  # Decompress data as it's written to disk
+}

systemtests/tests/autoxflate/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in

@@ -0,0 +1,5 @@
+Director {
+  Name = bareos-dir
+  Password = "@sd_password@"
+  Description = "Director, who is permitted to contact this storage daemon."
+}

systemtests/tests/autoxflate/etc/bareos/bareos-sd.d/messages/Standard.conf

@@ -0,0 +1,5 @@
+Messages {
+  Name = Standard
+  Director = bareos-dir = all
+  Description = "Send all messages to the Director."
+}

systemtests/tests/autoxflate/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in

@@ -0,0 +1,12 @@
+Storage {
+  Name = bareos-sd
+  Maximum Concurrent Jobs = 20
+
+  Working Directory =  "@working_dir@"
+  SD Port = @sd_port@
+  @sd_backend_config@
+
+  Plugin Directory = "@SD_PLUGINS_DIR_TO_TEST@"
+  Plugin Names = "autoxflate"
+  Auto XFlate On Replication = yes
+}

systemtests/tests/autoxflate/etc/bareos/bconsole.conf.in

@@ -0,0 +1,10 @@
+#
+# Bareos User Agent (or Console) Configuration File
+#
+
+Director {
+  Name = @basename@-dir
+  DIRport = @dir_port@
+  Address = @hostname@
+  Password = "@dir_password@"
+}

systemtests/tests/autoxflate/testrunner

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+set -o pipefail
+set -u
+
+TestName="$(basename "$(pwd)")"
+export TestName
+
+#shellcheck source=../environment.in
+. ./environment
+
+#shellcheck source=../scripts/functions
+. "${rscripts}"/functions
+"${rscripts}"/cleanup
+"${rscripts}"/setup
+
+setup_data
+
+start_test
+
+# 1) Create a full backup with LZ4 compression (as per SelfTest fileset config)
+# 2) Copy full backup to other pool and decompress (as per FileStorage2 device config)
+# 3) Restore copied job
+cat <<END_OF_DATA >"$tmp/bconcmds"
+@$out /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Full
+run job=backup-bareos-fd level=Full yes
+wait
+update volume=TestVolume001 volstatus=Used
+messages
+label volume=TestCopyVolume001 storage=File2 pool=FullCopy
+run copy yes
+wait
+messages
+restore jobid=4 client=bareos-fd where=$tmp/bareos-restores all done yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bareos "$@"
+
+# There should be no difference, but with autoxflate decompression
+# the restore reports success ("Restore OK") but seems to fail silently
+# and doesn't actually restore anything
+check_restore_diff "${BackupDirectory}"
+
+end_test