Send CRITICAL state to NSCA on backup failure

mongodb_consistent_backup/Main.py

@@ -49,6 +49,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
         self.logger                   = None
         self.current_log_file         = None
         self.backup_log_file          = None
+        self.last_error_msg           = ''
 
         try:
             self.setup_config()
@@ -57,6 +58,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
             self.get_lock()
             self.logger.update_symlink()
             self.init()
+            self.setup_notifier()
             self.set_backup_dirs()
             self.get_db_conn()
             self.setup_state()
@@ -98,6 +100,18 @@ def setup_state(self):
         self.state = StateBackup(self.backup_directory, self.config, self.backup_time, self.uri, sys.argv)
         self.state.write()
 
+    def setup_notifier(self):
+        try:
+            self.notify = Notify(
+                self.manager,
+                self.config,
+                self.timer,
+                self.backup_root_subdirectory,
+                self.backup_directory
+            )
+        except Exception, e:
+            self.exception("Problem starting notifier! Error: %s" % e, e)
+
     def get_db_conn(self):
         self.uri = MongoUri(self.config.host, self.config.port)
         try:
@@ -171,23 +185,23 @@ def cleanup_and_exit(self, code, frame):
 
         if self.manager:
             self.manager.shutdown()
+        if self.db:
+            self.db.close()
 
         if self.notify:
             try:
-                self.notify.notify("%s: backup '%s' failed!" % (
-                    self.config,
-                    self.program_name
-                ), False)
+                self.notify.notify("%s: backup '%s/%s' failed! Error: '%s'" % (
+                    self.program_name,
+                    self.config.backup.name,
+                    self.backup_time,
+                    self.last_error_msg
+                ))
                 self.notify.run()
                 self.notify.close()
-            except:
-                pass
-
-        if self.db:
-            self.db.close()
+            except Exception, e:
+                logging.error("Error from notifier: %s" % e)
 
         logging.info("Cleanup complete, exiting")
-
         if self.logger:
             self.logger.rotate()
             self.logger.close()
@@ -196,6 +210,7 @@ def cleanup_and_exit(self, code, frame):
         sys.exit(1)
 
     def exception(self, error_message, error):
+        self.last_error_msg = error_message
         if isinstance(error, NotifyError):
             logging.error(error_message)
         else:
@@ -237,18 +252,6 @@ def run(self):
         except Exception, e:
             self.exception("Problem starting archiver! Error: %s" % e, e)
 
-        # Setup the notifier
-        try:
-            self.notify = Notify(
-                self.manager,
-                self.config,
-                self.timer,
-                self.backup_root_subdirectory,
-                self.backup_directory
-            )
-        except Exception, e:
-            self.exception("Problem starting notifier! Error: %s" % e, e)
-
         # Setup the uploader
         try:
             self.upload = Upload(

mongodb_consistent_backup/Notify/Notify.py

@@ -1,5 +1,6 @@
 import logging
 
+from mongodb_consistent_backup.Errors import Error, NotifyError
 from mongodb_consistent_backup.Notify.Nsca import Nsca
 from mongodb_consistent_backup.Pipeline import Stage
 
@@ -9,7 +10,6 @@ def __init__(self, manager, config, timer, base_dir, backup_dir):
         super(Notify, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir)
         self.task = self.config.notify.method
 
-        self.completed = False
         self.notifications = []
         self.init()
 
@@ -19,19 +19,19 @@ def notify(self, message, success=False):
 
     def run(self, *args):
         if self._task and len(self.notifications) > 0:
-            logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server))
-            self.timers.start(self.stage)
-            while len(self.notifications) > 0:
-                try:
-                    (success, message) = self.notifications.pop()
-                    state = self._task.failed
-                    if success:
-                        state = self._task.success
-                    self._task.run(state, message)
-                except:
-                    continue
-            self.timers.stop(self.stage)
-        self.completed = True
+            try:
+                logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server))
+                while len(self.notifications) > 0:
+                    try:
+                        (success, message) = self.notifications.pop()
+                        state = self._task.failed
+                        if success == True:
+                            state = self._task.success
+                        self._task.run(state, message)
+                    except NotifyError:
+                        continue
+            except Exception, e:
+                raise Error(e)
 
     def close(self):
         if self._task:

mongodb_consistent_backup/Notify/Nsca/Nsca.py

@@ -54,7 +54,6 @@ def close(self):
 
     def run(self, ret_code, output):
         if self.notifier:
-            self.timer.start(self.timer_name)
             logging.info("Sending %sNSCA report to check host/name '%s/%s' at NSCA host %s" % (
                 self.mode_type,
                 self.check_host,
@@ -66,7 +65,6 @@ def run(self, ret_code, output):
             try:
                 self.notifier.svc_result(self.check_host, self.check_name, ret_code, str(output))
                 logging.debug('Sent %sNSCA report to host %s' % (self.mode_type, self.server))
-                self.timer.stop(self.timer_name)
             except Exception, e:
                 logging.error('Failed to send %sNSCA report to host %s: %s' % (self.mode_type, self.server, sys.exc_info()[1]))
                 raise NotifyError(e)