From da55abd598ada38c3c169cc5b88ff6d90c12c670 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 17 May 2017 17:56:04 +0200 Subject: [PATCH 1/3] Start Notifier early at init time in Main.py, not when backup begins --- mongodb_consistent_backup/Main.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index 23f001db..3609238a 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -57,6 +57,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"): self.get_lock() self.logger.update_symlink() self.init() + self.setup_notifier() self.set_backup_dirs() self.get_db_conn() self.setup_state() @@ -98,6 +99,18 @@ def setup_state(self): self.state = StateBackup(self.backup_directory, self.config, self.backup_time, self.uri, sys.argv) self.state.write() + def setup_notifier(self): + try: + self.notify = Notify( + self.manager, + self.config, + self.timer, + self.backup_root_subdirectory, + self.backup_directory + ) + except Exception, e: + self.exception("Problem starting notifier! Error: %s" % e, e) + def get_db_conn(self): self.uri = MongoUri(self.config.host, self.config.port) try: @@ -237,18 +250,6 @@ def run(self): except Exception, e: self.exception("Problem starting archiver! Error: %s" % e, e) - # Setup the notifier - try: - self.notify = Notify( - self.manager, - self.config, - self.timer, - self.backup_root_subdirectory, - self.backup_directory - ) - except Exception, e: - self.exception("Problem starting notifier! Error: %s" % e, e) - # Setup the uploader try: self.upload = Upload( From 170b7022bbe5b02f548aa31747da6a96f096b6f8 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 17 May 2017 19:05:11 +0200 Subject: [PATCH 2/3] Return non-success on failure instead of OK (code=0), also include the error encountered. Cleaned up Notify/Notify.py --- mongodb_consistent_backup/Main.py | 22 ++++++++------- mongodb_consistent_backup/Notify/Notify.py | 28 +++++++++---------- mongodb_consistent_backup/Notify/Nsca/Nsca.py | 2 -- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index 3609238a..0bd24273 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -49,6 +49,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"): self.logger = None self.current_log_file = None self.backup_log_file = None + self.last_error_msg = None try: self.setup_config() @@ -184,23 +185,23 @@ def cleanup_and_exit(self, code, frame): if self.manager: self.manager.shutdown() + if self.db: + self.db.close() if self.notify: try: - self.notify.notify("%s: backup '%s' failed!" % ( - self.config, - self.program_name - ), False) + self.notify.notify("%s: backup '%s/%s' failed! Error: '%s'" % ( + self.program_name, + self.config.backup.name, + self.backup_time, + self.last_error_msg + )) self.notify.run() self.notify.close() - except: - pass - - if self.db: - self.db.close() + except Exception, e: + logging.error("Error from notifier: %s" % e) logging.info("Cleanup complete, exiting") - if self.logger: self.logger.rotate() self.logger.close() @@ -209,6 +210,7 @@ def cleanup_and_exit(self, code, frame): sys.exit(1) def exception(self, error_message, error): + self.last_error_msg = error_message if isinstance(error, NotifyError): logging.error(error_message) else: diff --git a/mongodb_consistent_backup/Notify/Notify.py b/mongodb_consistent_backup/Notify/Notify.py index 80f1b2d3..2a51ebe9 100644 --- a/mongodb_consistent_backup/Notify/Notify.py +++ b/mongodb_consistent_backup/Notify/Notify.py @@ -1,5 +1,6 @@ import logging +from mongodb_consistent_backup.Errors import Error, NotifyError from mongodb_consistent_backup.Notify.Nsca import Nsca from mongodb_consistent_backup.Pipeline import Stage @@ -9,7 +10,6 @@ def __init__(self, manager, config, timer, base_dir, backup_dir): super(Notify, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir) self.task = self.config.notify.method - self.completed = False self.notifications = [] self.init() @@ -19,19 +19,19 @@ def notify(self, message, success=False): def run(self, *args): if self._task and len(self.notifications) > 0: - logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server)) - self.timers.start(self.stage) - while len(self.notifications) > 0: - try: - (success, message) = self.notifications.pop() - state = self._task.failed - if success: - state = self._task.success - self._task.run(state, message) - except: - continue - self.timers.stop(self.stage) - self.completed = True + try: + logging.info("Sending %i notification(s) to: %s" % (len(self.notifications), self._task.server)) + while len(self.notifications) > 0: + try: + (success, message) = self.notifications.pop() + state = self._task.failed + if success == True: + state = self._task.success + self._task.run(state, message) + except NotifyError: + continue + except Exception, e: + raise Error(e) def close(self): if self._task: diff --git a/mongodb_consistent_backup/Notify/Nsca/Nsca.py b/mongodb_consistent_backup/Notify/Nsca/Nsca.py index 406fda94..4ddf5410 100644 --- a/mongodb_consistent_backup/Notify/Nsca/Nsca.py +++ b/mongodb_consistent_backup/Notify/Nsca/Nsca.py @@ -54,7 +54,6 @@ def close(self): def run(self, ret_code, output): if self.notifier: - self.timer.start(self.timer_name) logging.info("Sending %sNSCA report to check host/name '%s/%s' at NSCA host %s" % ( self.mode_type, self.check_host, @@ -66,7 +65,6 @@ def run(self, ret_code, output): try: self.notifier.svc_result(self.check_host, self.check_name, ret_code, str(output)) logging.debug('Sent %sNSCA report to host %s' % (self.mode_type, self.server)) - self.timer.stop(self.timer_name) except Exception, e: logging.error('Failed to send %sNSCA report to host %s: %s' % (self.mode_type, self.server, sys.exc_info()[1])) raise NotifyError(e) From efdb0c5fe9f01a6c4ae680b761c8dbd5f9a32bc5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 17 May 2017 19:31:18 +0200 Subject: [PATCH 3/3] Make last_error_msg a string, not None --- mongodb_consistent_backup/Main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index 0bd24273..a53bd988 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -49,7 +49,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"): self.logger = None self.current_log_file = None self.backup_log_file = None - self.last_error_msg = None + self.last_error_msg = '' try: self.setup_config()