diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index 23f001db..a53bd988 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -49,6 +49,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"): self.logger = None self.current_log_file = None self.backup_log_file = None + self.last_error_msg = '' try: self.setup_config() @@ -57,6 +58,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"): self.get_lock() self.logger.update_symlink() self.init() + self.setup_notifier() self.set_backup_dirs() self.get_db_conn() self.setup_state() @@ -98,6 +100,18 @@ def setup_state(self): self.state = StateBackup(self.backup_directory, self.config, self.backup_time, self.uri, sys.argv) self.state.write() + def setup_notifier(self): + try: + self.notify = Notify( + self.manager, + self.config, + self.timer, + self.backup_root_subdirectory, + self.backup_directory + ) + except Exception, e: + self.exception("Problem starting notifier! Error: %s" % e, e) + def get_db_conn(self): self.uri = MongoUri(self.config.host, self.config.port) try: @@ -171,23 +185,23 @@ def cleanup_and_exit(self, code, frame): if self.manager: self.manager.shutdown() + if self.db: + self.db.close() if self.notify: try: - self.notify.notify("%s: backup '%s' failed!" % ( - self.config, - self.program_name - ), False) + self.notify.notify("%s: backup '%s/%s' failed! Error: '%s'" % ( + self.program_name, + self.config.backup.name, + self.backup_time, + self.last_error_msg + )) self.notify.run() self.notify.close() - except: - pass - - if self.db: - self.db.close() + except Exception, e: + logging.error("Error from notifier: %s" % e) logging.info("Cleanup complete, exiting") - if self.logger: self.logger.rotate() self.logger.close() @@ -196,6 +210,7 @@ def cleanup_and_exit(self, code, frame): sys.exit(1) def exception(self, error_message, error): + self.last_error_msg = error_message if isinstance(error, NotifyError): logging.error(error_message) else: @@ -237,18 +252,6 @@ def run(self): except Exception, e: self.exception("Problem starting archiver! Error: %s" % e, e) - # Setup the notifier - try: - self.notify = Notify( - self.manager, - self.config, - self.timer, - self.backup_root_subdirectory, - self.backup_directory - ) - except Exception, e: - self.exception("Problem starting notifier! Error: %s" % e, e) - # Setup the uploader try: self.upload = Upload( diff --git a/mongodb_consistent_backup/Notify/Notify.py b/mongodb_consistent_backup/Notify/Notify.py index 80f1b2d3..2a51ebe9 100644 --- a/mongodb_consistent_backup/Notify/Notify.py +++ b/mongodb_consistent_backup/Notify/Notify.py @@ -1,5 +1,6 @@ import logging +from mongodb_consistent_backup.Errors import Error, NotifyError from mongodb_consistent_backup.Notify.Nsca import Nsca from mongodb_consistent_backup.Pipeline import Stage @@ -9,7 +10,6 @@ def __init__(self, manager, config, timer, base_dir, backup_dir): super(Notify, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir) self.task = self.config.notify.method - self.completed = False self.notifications = [] self.init() @@ -19,19 +19,19 @@ def notify(self, message, success=False): def run(self, *args): if self._task and len(self.notifications) > 0: - logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server)) - self.timers.start(self.stage) - while len(self.notifications) > 0: - try: - (success, message) = self.notifications.pop() - state = self._task.failed - if success: - state = self._task.success - self._task.run(state, message) - except: - continue - self.timers.stop(self.stage) - self.completed = True + try: + logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server)) + while len(self.notifications) > 0: + try: + (success, message) = self.notifications.pop() + state = self._task.failed + if success == True: + state = self._task.success + self._task.run(state, message) + except NotifyError: + continue + except Exception, e: + raise Error(e) def close(self): if self._task: diff --git a/mongodb_consistent_backup/Notify/Nsca/Nsca.py b/mongodb_consistent_backup/Notify/Nsca/Nsca.py index 406fda94..4ddf5410 100644 --- a/mongodb_consistent_backup/Notify/Nsca/Nsca.py +++ b/mongodb_consistent_backup/Notify/Nsca/Nsca.py @@ -54,7 +54,6 @@ def close(self): def run(self, ret_code, output): if self.notifier: - self.timer.start(self.timer_name) logging.info("Sending %sNSCA report to check host/name '%s/%s' at NSCA host %s" % ( self.mode_type, self.check_host, @@ -66,7 +65,6 @@ def run(self, ret_code, output): try: self.notifier.svc_result(self.check_host, self.check_name, ret_code, str(output)) logging.debug('Sent %sNSCA report to host %s' % (self.mode_type, self.server)) - self.timer.stop(self.timer_name) except Exception, e: logging.error('Failed to send %sNSCA report to host %s: %s' % (self.mode_type, self.server, sys.exc_info()[1])) raise NotifyError(e)