diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index fc3c99c9..8d3ac4d7 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -2,12 +2,12 @@ import logging import signal -from fabric.api import hide, settings, local from math import floor from multiprocessing import cpu_count +from subprocess import check_output from time import sleep -from mongodb_consistent_backup.Common import MongoUri +from mongodb_consistent_backup.Common import MongoUri, config_to_string from mongodb_consistent_backup.Errors import Error, OperationError from mongodb_consistent_backup.Oplog import OplogState from mongodb_consistent_backup.Pipeline import Task @@ -28,19 +28,33 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard self.compression_supported = ['auto', 'none', 'gzip'] self.version = 'unknown' + self.version_extra = {} self.threads_max = 16 self.config_replset = False self.dump_threads = [] self.states = {} self._summary = {} + self.parse_mongodump_version() + self.choose_compression() + if self.config.backup.mongodump.threads and self.config.backup.mongodump.threads > 0: self.threads(self.config.backup.mongodump.threads) - with hide('running', 'warnings'), settings(warn_only=True): - self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True) - - self.choose_compression() + def parse_mongodump_version(self): + if os.path.isfile(self.binary): + output = check_output([self.binary, "--version"]) + lines = output.rstrip().split("\n") + for line in lines: + if "version:" in line: + name, version_num = line.split(" version: ") + if name == 'mongodump': + self.version = version_num + if '-' in version_num: + self.version = version_num.split("-")[0] + self.version_extra[name.lower()] = version_num + return self.version, self.version_extra + raise OperationError("Could not parse mongodump --version output!") def choose_compression(self): if self.can_gzip(): @@ -97,7 +111,7 @@ def wait(self): def threads(self, threads=None): if threads: self.thread_count = int(threads) - elif not self.thread_count: + elif not self.thread_count and self.version is not 'unknown': if tuple(self.version.split(".")) >= tuple("3.2.0".split(".")): self.thread_count = 1 if self.cpu_count > len(self.replsets): @@ -135,8 +149,13 @@ def run(self): if not len(self.dump_threads) > 0: raise OperationError('No backup threads started!') + options = { + 'compression': self.compression(), + 'threads_per_dump': self.threads() + } + options.update(self.version_extra) logging.info( - "Starting backups using mongodump %s (options: compression=%s, threads_per_dump=%i)" % (self.version, self.compression(), self.threads())) + "Starting backups using mongodump %s (options: %s)" % (self.version, config_to_string(options))) for thread in self.dump_threads: thread.start() self.wait() diff --git a/mongodb_consistent_backup/Common/Util.py b/mongodb_consistent_backup/Common/Util.py index a2c4713a..e09df743 100644 --- a/mongodb_consistent_backup/Common/Util.py +++ b/mongodb_consistent_backup/Common/Util.py @@ -6,10 +6,10 @@ def config_to_string(config): - config_vars = "" + config_pairs = [] for key in config: - config_vars += "%s=%s, " % (key, config[key]) - return config_vars[:-1] + config_pairs.append("%s=%s" % (key, config[key])) + return ", ".join(config_pairs) def is_datetime(string): try: