Merge pull request #717 from bareos/dev/fbergkemper/master/s4653

dird: Fix Copy Job behaviour regarding Archive Jobs

CHANGELOG.md

@@ -15,6 +15,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - Package **bareos-database-postgresql**: add recommendation for package **dbconfig-pgsql**.
 - Adapt the init scripts for some platform to not refer to a specific (outdated) configuration file, but to use the default config file instead.
 - scripts: cleaned up code for postgresql db creation [PR #709]
+- Change Copy Job behaviour regarding Archive Jobs [PR #717]
 
 ### Deprecated
 

core/src/dird/migrate.cc

@@ -3,7 +3,7 @@
 
    Copyright (C) 2004-2012 Free Software Foundation Europe e.V.
    Copyright (C) 2011-2016 Planets Communications B.V.
-   Copyright (C) 2013-2020 Bareos GmbH & Co. KG
+   Copyright (C) 2013-2021 Bareos GmbH & Co. KG
 
    This program is Free Software; you can redistribute it and/or
    modify it under the terms of version three of the GNU Affero General Public
@@ -195,7 +195,7 @@ static const char* sql_pool_time
 static const char* sql_jobids_of_pool_uncopied_jobs
     = "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
       " WHERE Pool.Name = '%s' AND Pool.PoolId = Job.PoolId"
-      " AND Job.Type = 'B' AND Job.JobStatus IN ('T','W')"
+      " AND Job.Type IN ('B','A') AND Job.JobStatus IN ('T','W')"
       " AND Job.jobBytes > 0"
       " AND Job.JobId NOT IN"
       " (SELECT PriorJobId FROM Job WHERE"
@@ -1361,7 +1361,8 @@ static inline bool DoActualMigration(JobControlRecord* jcr)
    * Make sure this job was not already migrated
    */
   if (jcr->impl->previous_jr.JobType != JT_BACKUP
-      && jcr->impl->previous_jr.JobType != JT_JOB_COPY) {
+      && jcr->impl->previous_jr.JobType != JT_JOB_COPY
+      && jcr->impl->previous_jr.JobType != JT_ARCHIVE) {
     Jmsg(jcr, M_INFO, 0,
          _("JobId %s already %s probably by another Job. %s stopped.\n"),
          edit_int64(jcr->impl->previous_jr.JobId, ed1),

systemtests/CMakeLists.txt

@@ -564,6 +564,7 @@ set(SYSTEM_TESTS
     bscan
     bconsole-status-client
     config-syntax-crash
+    copy-archive-job
     copy-bscan
     copy-remote-bscan
     deprecation

systemtests/scripts/functions

@@ -301,6 +301,19 @@ check_files_written()
     fi
 }
 
+check_sd_files_written()
+{
+    LOG=$1
+    FILES="$(awk '/SD Files Written:/ { last=$4 } END { print last }' "$LOG")"
+
+    if [[ "$FILES" -lt 1 || -z "$FILES" ]]; then
+        print_debug "ERROR: Expected number of SD files written > 0, got \"$FILES\"."
+        return 1
+    fi
+
+    return 0
+}
+
 check_linked_against()
 {
    LIB="$1"

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in

@@ -0,0 +1,8 @@
+Catalog {
+  Name = MyCatalog
+  #dbdriver = "@DEFAULT_DB_TYPE@"
+  dbdriver = "XXX_REPLACE_WITH_DATABASE_DRIVER_XXX"
+  dbname = "@db_name@"
+  dbuser = "@db_user@"
+  dbpassword = "@db_password@"
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in

@@ -0,0 +1,7 @@
+Client {
+  Name = bareos-fd
+  Description = "Client resource of the Director itself."
+  Address = @hostname@
+  Password = "@fd_password@"          # password for FileDaemon
+  FD PORT = @fd_port@
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in

@@ -0,0 +1,7 @@
+Console {
+  Name = bareos-mon
+  Description = "Restricted console used by tray-monitor to get the status of the director."
+  Password = "@mon_dir_password@"
+  CommandACL = status, .status
+  JobACL = *all*
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in

@@ -0,0 +1,27 @@
+Director {                            # define myself
+  Name = bareos-dir
+  QueryFile = "@scriptdir@/query.sql"
+  Maximum Concurrent Jobs = 10
+  Password = "@dir_password@"         # Console password
+  Messages = Daemon
+  Auditing = yes
+
+  # Enable the Heartbeat if you experience connection losses
+  # (eg. because of your router or firewall configuration).
+  # Additionally the Heartbeat can be enabled in bareos-sd and bareos-fd.
+  #
+  # Heartbeat Interval = 1 min
+
+  # remove comment in next line to load dynamic backends from specified directory
+  Backend Directory = @backenddir@
+
+  # remove comment from "Plugin Directory" to load plugins from specified directory.
+  # if "Plugin Names" is defined, only the specified plugins will be loaded,
+  # otherwise all director plugins (*-dir.so) from the "Plugin Directory".
+  #
+  # Plugin Directory = "@python_plugin_module_src_dir@"
+  # Plugin Names = ""
+  Working Directory =  "@working_dir@"
+  Pid Directory =  "@piddir@"
+  DirPort = @dir_port@
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in

@@ -0,0 +1,11 @@
+FileSet {
+  Name = "Catalog"
+  Description = "Backup the catalog dump and Bareos configuration files."
+  Include {
+    Options {
+      signature = MD5
+    }
+    File = "@working_dir@/@db_name@.sql" # database dump
+    File = "@confdir@"                   # configuration
+  }
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in

@@ -0,0 +1,11 @@
+FileSet {
+  Name = "SelfTest"
+  Description = "fileset just to backup some files for selftest"
+  Include {
+    Options {
+      Signature = MD5 # calculate md5 checksum per file
+    }
+   #File = "@sbindir@"
+    File=<@tmpdir@/file-list
+  }
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in

@@ -0,0 +1,20 @@
+Job {
+  Name = "BackupCatalog"
+  Description = "Backup the catalog database (after the nightly save)"
+  JobDefs = "DefaultJob"
+  Level = Full
+  FileSet="Catalog"
+
+  # This creates an ASCII copy of the catalog
+  # Arguments to make_catalog_backup.pl are:
+  #  make_catalog_backup.pl <catalog-name>
+  RunBeforeJob = "@scriptdir@/make_catalog_backup.pl MyCatalog"
+
+  # This deletes the copy of the catalog
+  RunAfterJob  = "@scriptdir@/delete_catalog_backup"
+
+  # This sends the bootstrap via mail for disaster recovery.
+  # Should be sent to another system, please change recipient accordingly
+  Write Bootstrap = "|@bindir@/bsmtp -h @smtp_host@ -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" @job_email@" # (#01)
+  Priority = 11                   # run after main backup
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in

@@ -0,0 +1,11 @@
+Job {
+  Name = "RestoreFiles"
+  Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..."
+  Type = Restore
+  Client = bareos-fd
+  FileSet = SelfTest
+  Storage = File
+  Pool = Incremental
+  Messages = Standard
+  Where = @tmp@/bareos-restores
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in

@@ -0,0 +1,11 @@
+Job {
+  Name = "backup-bareos-fd"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+  Run Script {
+        console = "update jobid=%i jobtype=A"
+        Runs When = After
+        Runs On Client = No
+        Runs On Failure = No
+  }
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/job/copy-job1.conf.in

@@ -0,0 +1,11 @@
+Job {
+  Name = "copy-job1"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+  Type = Copy
+  Level = Full
+  Pool = Full
+  Storage = File2
+  Selection Type = PoolUncopiedJobs
+  Messages = Standard
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/job/copy-job2.conf.in

@@ -0,0 +1,12 @@
+Job {
+  Name = "copy-job2"
+  JobDefs = "DefaultJob"
+  Client = "bareos-fd"
+  Type = Copy
+  Level = Full
+  Pool = Full
+  Storage = File2
+  Selection Type = SQLQuery
+  Selection Pattern = "SELECT jobid FROM job WHERE poolid=3 AND type='A' AND jobstatus IN ('T','W');"
+  Messages = Standard
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in

@@ -0,0 +1,15 @@
+JobDefs {
+  Name = "DefaultJob"
+  Type = Backup
+  Level = Incremental
+  Client = bareos-fd
+  FileSet = "SelfTest"
+  Storage = File
+  Messages = Standard
+  Pool = Incremental
+  Priority = 10
+  Write Bootstrap = "@working_dir@/%c.bsr"
+  Full Backup Pool = Full                  # write Full Backups into "Full" Pool
+  Differential Backup Pool = Differential  # write Diff Backups into "Differential" Pool
+  Incremental Backup Pool = Incremental    # write Incr Backups into "Incremental" Pool
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/messages/Daemon.conf.in

@@ -0,0 +1,7 @@
+Messages {
+  Name = Daemon
+  Description = "Message delivery for daemon messages (no job)."
+  console = all, !skipped, !saved, !audit
+  append = "@logdir@/bareos.log" = all, !skipped, !audit
+  append = "@logdir@/bareos-audit.log" = audit
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/messages/Standard.conf.in

@@ -0,0 +1,7 @@
+Messages {
+  Name = Standard
+  Description = "Reasonable message delivery -- send most everything to email address and to the console."
+  console = all, !skipped, !saved, !audit
+  append = "@logdir@/bareos.log" = all, !skipped, !saved, !audit
+  catalog = all, !skipped, !saved, !audit
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/pool/Copy.conf

@@ -0,0 +1,11 @@
+Pool {
+  Name = Copy
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 365 days         # How long should the Full Backups be kept? (#06)
+  Maximum Volume Bytes = 50G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Copy-"              # Volumes will be labeled "Full-<volume-id>"
+  Storage = File2
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/pool/Differential.conf

@@ -0,0 +1,10 @@
+Pool {
+  Name = Differential
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 90 days          # How long should the Differential Backups be kept? (#09)
+  Maximum Volume Bytes = 10G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Differential-"      # Volumes will be labeled "Differential-<volume-id>"
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/pool/Full.conf

@@ -0,0 +1,12 @@
+Pool {
+  Name = Full
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 365 days         # How long should the Full Backups be kept? (#06)
+  Maximum Volume Bytes = 50G          # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Full-"              # Volumes will be labeled "Full-<volume-id>"
+  Next Pool = Copy
+  Storage = File
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/pool/Incremental.conf

@@ -0,0 +1,10 @@
+Pool {
+  Name = Incremental
+  Pool Type = Backup
+  Recycle = yes                       # Bareos can automatically recycle Volumes
+  AutoPrune = yes                     # Prune expired volumes
+  Volume Retention = 30 days          # How long should the Incremental Backups be kept?  (#12)
+  Maximum Volume Bytes = 1G           # Limit Volume size to something reasonable
+  Maximum Volumes = 100               # Limit number of Volumes in Pool
+  Label Format = "Incremental-"       # Volumes will be labeled "Incremental-<volume-id>"
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/pool/Scratch.conf

@@ -0,0 +1,4 @@
+Pool {
+  Name = Scratch
+  Pool Type = Scratch
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/profile/operator.conf

@@ -0,0 +1,18 @@
+Profile {
+   Name = operator
+   Description = "Profile allowing normal Bareos operations."
+
+   Command ACL = !.bvfs_clear_cache, !.exit, !.sql
+   Command ACL = !configure, !create, !delete, !purge, !prune, !sqlquery, !umount, !unmount
+   Command ACL = *all*
+
+   Catalog ACL = *all*
+   Client ACL = *all*
+   FileSet ACL = *all*
+   Job ACL = *all*
+   Plugin Options ACL = *all*
+   Pool ACL = *all*
+   Schedule ACL = *all*
+   Storage ACL = *all*
+   Where ACL = *all*
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/storage/File.conf.in

@@ -0,0 +1,10 @@
+Storage {
+  Name = File
+  Address = @hostname@
+  Password = "@sd_password@"
+  Device = FileStorage
+  Device = FileStorage2
+  Media Type = File
+  SD Port = @sd_port@
+  Maximum Concurrent Jobs = 2
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-dir.d/storage/File2.conf.in

@@ -0,0 +1,10 @@
+Storage {
+  Name = File2
+  Address = @hostname@
+  Password = "@sd_password@"
+  Device = FileStorage3
+  Device = FileStorage4
+  Media Type = File2
+  SD Port = @sd_port@
+  Maximum Concurrent Jobs = 2
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-fd.d/client/myself.conf.in

@@ -0,0 +1,20 @@
+Client {
+  Name = @basename@-fd
+  Maximum Concurrent Jobs = 20
+
+  # remove comment from "Plugin Directory" to load plugins from specified directory.
+  # if "Plugin Names" is defined, only the specified plugins will be loaded,
+  # otherwise all filedaemon plugins (*-fd.so) from the "Plugin Directory".
+  #
+  # Plugin Directory = "@python_plugin_module_src_fd@"
+  # Plugin Names = ""
+
+  # if compatible is set to yes, we are compatible with bacula
+  # if set to no, new bareos features are enabled which is the default
+  # compatible = yes
+
+  Working Directory =  "@working_dir@"
+  Pid Directory =  "@piddir@"
+  FD Port = @fd_port@
+
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in

@@ -0,0 +1,5 @@
+Director {
+  Name = bareos-dir
+  Password = "@fd_password@"
+  Description = "Allow the configured Director to access this file daemon."
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in

@@ -0,0 +1,6 @@
+Director {
+  Name = bareos-mon
+  Password = "@mon_fd_password@"
+  Monitor = yes
+  Description = "Restricted Director, used by tray-monitor to get the status of this file daemon."
+}

systemtests/tests/copy-archive-job/etc/bareos/bareos-fd.d/messages/Standard.conf

@@ -0,0 +1,5 @@
+Messages {
+  Name = Standard
+  Director = bareos-dir = all, !skipped, !restored
+  Description = "Send relevant messages to the Director."
+}