libcloud-plugin: add optional argument treat_download_errors_as_warnings

core/src/plugins/filed/BareosFdPluginLibcloud.py

@@ -39,6 +39,7 @@
 
 from BareosLibcloudApi import SUCCESS
 from BareosLibcloudApi import ERROR
+from BareosLibcloudApi import ABORT
 from BareosLibcloudApi import BareosLibcloudApi
 from bareos_fd_consts import bRCs, bCFs, bIOPS, bJobMessageType, bFileType
 from libcloud.storage.types import Provider
@@ -76,6 +77,7 @@ def __init__(self, context, plugindef):
 
         super(BareosFdPluginLibcloud, self).__init__(context, plugindef)
         super(BareosFdPluginLibcloud, self).parse_plugin_definition(context, plugindef)
+        self.options["treat_download_errors_as_warnings"] = False
         self.__parse_options(context)
 
         self.last_run = datetime.datetime.fromtimestamp(self.since)
@@ -146,6 +148,8 @@ def __check_config(self, context, config_filename):
             "prefetch_size",
             "temporary_download_directory",
         ]
+        optional_options = {}
+        optional_options["misc"] = ["treat_download_errors_as_warnings"]
 
         # this maps config file options to libcloud options
         option_map = {
@@ -197,6 +201,19 @@ def __check_config(self, context, config_filename):
                     )
                     return False
 
+        for option in optional_options["misc"]:
+            if self.config.has_option(section, option):
+                try:
+                    value = self.config.get(section, option)
+                    self.options["treat_download_errors_as_warnings"] = strtobool(value)
+                except:
+                    debugmessage(
+                        100,
+                        "Could not evaluate: %s in config file %s"
+                        % (value, config_filename),
+                    )
+                    return False
+
         return True
 
     def start_backup_job(self, context):
@@ -261,7 +278,14 @@ def __shutdown(self):
     def start_backup_file(self, context, savepkt):
         error = False
         while self.active:
-            if self.api.check_worker_messages() != SUCCESS:
+            worker_result = self.api.check_worker_messages()
+            if worker_result == ERROR:
+                if self.options["treat_download_errors_as_warnings"]:
+                    pass
+                else:
+                    self.active = False
+                    error = True
+            elif worker_result == ABORT:
                 self.active = False
                 error = True
             else:
@@ -311,12 +335,21 @@ def start_backup_file(self, context, savepkt):
             try:
                 self.FILE = IterStringIO(self.current_backup_task["data"].as_stream())
             except ObjectDoesNotExistError:
-                jobmessage(
-                    "M_WARNING",
-                    "Skipped file %s because it does not exist anymore"
-                    % (self.current_backup_task["name"]),
-                )
-                return bRCs["bRC_Skip"]
+                if self.options["treat_download_errors_as_warnings"]:
+                    jobmessage(
+                        "M_WARNING",
+                        "Skipped file %s because it does not exist anymore"
+                        % (self.current_backup_task["name"]),
+                    )
+                    return bRCs["bRC_Skip"]
+                else:
+                    jobmessage(
+                        "M_ERROR",
+                        "File %s does not exist anymore"
+                        % (self.current_backup_task["name"]),
+                    )
+                    return bRCs["bRC_Error"]
+
         else:
             raise Exception(value='Wrong argument for current_backup_task["type"]')
 
@@ -365,7 +398,10 @@ def plugin_io(self, context, IOP):
                     ),
                 )
                 IOP.status = 0
-                return bRCs["bRC_Error"]
+                if self.options["treat_download_errors_as_warnings"]:
+                    return bRCs["bRC_Skip"]
+                else:
+                    return bRCs["bRC_Error"]
 
         elif IOP.func == bIOPS["IO_WRITE"]:
             try:

core/src/plugins/filed/BareosLibcloudApi.py

@@ -19,7 +19,7 @@
 
 SUCCESS = 0
 ERROR = 1
-
+ABORT = 2
 
 class BareosLibcloudApi(object):
     @staticmethod
@@ -94,7 +94,7 @@ def check_worker_messages(self):
                     else:
                         self.count_worker_ready += 1
                 elif message.type == MESSAGE_TYPE.ABORT_MESSAGE:
-                    return ERROR
+                    return ABORT
                 elif message.type == MESSAGE_TYPE.DEBUG_MESSAGE:
                     debugmessage(message.level, message.message_string)
                 else:

core/src/plugins/filed/bareos_libcloud_api/process_base.py

@@ -38,6 +38,7 @@ def __init__(
         self.worker_id = worker_id
 
     def run_process(self):
+        # implementation of derived class
         pass
 
     def run(self):

core/src/plugins/filed/bareos_libcloud_api/worker.py

@@ -121,6 +121,7 @@ def __run_job(self, job):
                 job["type"] = TASK_TYPE.TEMP_FILE
             except OSError as e:
                 self.error_message("Could not open temporary file %s" % e.filename)
+                self.abort_message()
                 return FINISH
             except ObjectDoesNotExistError as e:
                 self.error_message(

docs/manuals/source/TasksAndConcepts/Plugins.rst

@@ -1287,7 +1287,7 @@ And the plugin config file as follows:
 
 Mandatory Plugin Options:
 
-All options in the config file are mandatory:
+These options in the config file are mandatory:
 
 hostname
    The hostname/ip address of the storage backend server
@@ -1324,6 +1324,16 @@ temporary_download_directory
    the filedaemon process needs read and write access to this path
 
 
+Optional Plugin Options:
+
+This option in the config file is optional:
+
+treat_download_errors_as_warnings
+   This parameter can be set to True to keep a job running if for some reason a file cannot
+   be downloaded from a bucket because it is either deleted or moved to another space during
+   download. The default for this value is False.
+
+
 .. _PerconaXtrabackupPlugin:
 .. _backup-mysql-XtraBackup: