diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index fef12a24..24959f2c 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -10,9 +10,9 @@ production: name: default location: /var/lib/mongodb-consistent-backup # mongodump: - # binary: [path] (default: /usr/bin/mongodump) - # compression: [none|gzip] (default: true - if supported) - # threads: [1-16] (default: auto-generated - shards/cpu) + # binary: [path] (default: /usr/bin/mongodump) + # compression: [auto|none|gzip] (default: auto - enable gzip if supported) + # threads: [1-16] (default: auto-generated - shards/cpu) #replication: # max_lag_secs: [1+] (default: 5) # min_priority: [0-999] (default: 0) @@ -23,14 +23,14 @@ production: # wait_secs: [1+] (default: 300) # ping_secs: [1+] (default: 3) #oplog: - # compression: [none|gzip] (default: true - if used by backup stage) + # compression: [none|gzip] (default: gzip - if gzip is used by backup stage) # resolver_threads: [1+] (default: 2 per CPU) # tailer: # status_interval: 30 archive: method: tar # tar: - # compression: [none|gzip] (default: gzip, none if backup is compressed) + # compression: [none|gzip] (default: gzip - none if backup is already compressed) # threads: [1+] (default: 1 per CPU) # zbackup: # binary: [path] (default: /usr/bin/zbackup) diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 3958de9b..62a023b0 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -26,7 +26,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard self.replsets = replsets self.sharding = sharding - self.compression_supported = ['none', 'gzip'] + self.compression_supported = ['auto', 'none', 'gzip'] self.version = 'unknown' self.threads_max = 16 self.config_replset = False @@ -40,10 +40,15 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard with hide('running', 'warnings'), settings(warn_only=True): self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True) - if self.can_gzip() and self.compression() == 'none': - self.compression('gzip') - elif self.compression() == 'gzip': - logging.warning("mongodump gzip compression requested on binary that does not support gzip!") + self.choose_compression() + + def choose_compression(self): + if self.can_gzip(): + if self.compression() == 'auto': + logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression") + self.compression('gzip') + elif self.compression() == 'gzip': + raise OperationError("mongodump gzip compression requested on binary that does not support gzip!") def can_gzip(self): if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK): diff --git a/mongodb_consistent_backup/Backup/Mongodump/__init__.py b/mongodb_consistent_backup/Backup/Mongodump/__init__.py index d7cbeed5..ba960a08 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/__init__.py +++ b/mongodb_consistent_backup/Backup/Mongodump/__init__.py @@ -5,8 +5,8 @@ def config(parser): parser.add_argument("--backup.mongodump.binary", dest="backup.mongodump.binary", help="Path to 'mongodump' binary (default: /usr/bin/mongodump)", default='/usr/bin/mongodump') parser.add_argument("--backup.mongodump.compression", dest="backup.mongodump.compression", - help="Compression method to use on backup (default: gzip)", default="gzip", - choices=["none", "gzip"]) + help="Compression method to use on backup (default: auto)", default="auto", + choices=["auto", "none", "gzip"]) parser.add_argument("--backup.mongodump.threads", dest="backup.mongodump.threads", help="Number of threads to use for each mongodump process. There is 1 x mongodump per shard, be careful! (default: shards/CPUs)", default=0, type=int) diff --git a/mongodb_consistent_backup/Backup/__init__.py b/mongodb_consistent_backup/Backup/__init__.py index b3a39603..27ba8660 100644 --- a/mongodb_consistent_backup/Backup/__init__.py +++ b/mongodb_consistent_backup/Backup/__init__.py @@ -1,3 +1,8 @@ from Backup import Backup +def config(parser): + parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str) + parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str) + parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump']) + return parser diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ef2693ad..7b46eb41 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -54,9 +54,6 @@ def makeParser(self): parser.add_argument("-u", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) - parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str) - parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str) - parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump']) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/scripts/build.sh b/scripts/build.sh index 87229ee1..ba45695e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -100,7 +100,7 @@ if [ -d ${srcdir} ]; then if [ ! -d ${pexdir} ]; then mkdir -p ${pexdir} else - rm -f ${pexdir}/${mod_name}-*.whl + find ${pexdir} -type f -name "${mod_name}-*.whl" -print -delete fi [ ! -d ${bindir} ] && mkdir -p ${bindir} ${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir} ${builddir}