1.0.2 Release

Dockerfile

@@ -1,8 +1,8 @@
 FROM centos:centos7
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
-	yum install -y Percona-Server-MongoDB-32-tools zbackup && yum clean all && \
-	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.1/mongodb-consistent-backup.el7.centos.x86_64 && \
+	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all && \
+	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.2/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]

README.rst

@@ -222,6 +222,7 @@ Links
 
 - https://www.percona.com/blog/2016/07/25/mongodb-consistent-backups/
 - https://www.percona.com/blog/2017/01/09/mongodb-pit-backups-part-2/
+- https://www.percona.com/blog/2017/05/10/percona-lab-mongodb_consistent_backup-1-0-release-explained/
 - https://docs.mongodb.com/manual/reference/program/mongodump/
 - https://docs.mongodb.com/manual/reference/program/mongorestore/
 - http://zbackup.org

VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

conf/mongodb-consistent-backup.example.conf

@@ -10,9 +10,9 @@ production:
     name: default
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
-  #    binary: [path]           (default: /usr/bin/mongodump)
-  #    compression: [none|gzip] (default: true - if supported)
-  #    threads: [1-16]          (default: auto-generated - shards/cpu)
+  #    binary: [path]                (default: /usr/bin/mongodump)
+  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated - shards/cpu)
   #replication:
   #  max_lag_secs: [1+]        (default: 5)
   #  min_priority: [0-999]     (default: 0)
@@ -23,14 +23,14 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: true - if used by backup stage)
+  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
   #  resolver_threads: [1+]   (default: 2 per CPU)
   #  tailer:
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip, none if backup is compressed)
+  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -26,7 +26,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         self.replsets           = replsets
         self.sharding           = sharding
 
-        self.compression_supported = ['none', 'gzip']
+        self.compression_supported = ['auto', 'none', 'gzip']
         self.version               = 'unknown'
         self.threads_max           = 16
         self.config_replset        = False
@@ -40,11 +40,15 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         with hide('running', 'warnings'), settings(warn_only=True):
             self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True)
 
+        self.choose_compression()
+
+    def choose_compression(self):
         if self.can_gzip():
-            if self.compression() == 'none':
+            if self.compression() == 'auto':
+                logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
                 self.compression('gzip')
         elif self.compression() == 'gzip':
-            logging.warning("mongodump gzip compression requested on binary that does not support gzip!")
+            raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
     def can_gzip(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
@@ -121,8 +125,9 @@ def run(self):
                 self.authdb,
                 self.backup_dir,
                 self.binary,
+                self.version,
                 self.threads(),
-                self.do_gzip,
+                self.do_gzip(),
                 self.verbose
             )
             self.dump_threads.append(thread)
@@ -152,8 +157,9 @@ def run(self):
                     self.authdb,
                     self.backup_dir,
                     self.binary,
+                    self.version,
                     self.threads(),
-                    self.do_gzip,
+                    self.do_gzip(),
                     self.verbose
                 )]
                 self.dump_threads[0].start()

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -14,24 +14,28 @@
 
 # noinspection PyStringFormat
 class MongodumpThread(Process):
-    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary,
+    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary, version,
                  threads=0, dump_gzip=False, verbose=False):
         Process.__init__(self)
-        self.state       = state
-        self.uri         = uri
-        self.timer       = timer
-        self.user        = user
-        self.password    = password
-        self.authdb      = authdb
-        self.base_dir    = base_dir
-        self.binary      = binary
-        self.threads     = threads
-        self.dump_gzip   = dump_gzip
-        self.verbose     = verbose
-
-        self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset)
-        self.exit_code  = 1
-        self._command   = None
+        self.state     = state
+        self.uri       = uri
+        self.timer     = timer
+        self.user      = user
+        self.password  = password
+        self.authdb    = authdb
+        self.base_dir  = base_dir
+        self.binary    = binary
+        self.version   = version
+        self.threads   = threads
+        self.dump_gzip = dump_gzip
+        self.verbose   = verbose
+
+        self.timer_name        = "%s-%s" % (self.__class__.__name__, self.uri.replset)
+        self.exit_code         = 1
+        self._command          = None
+        self.do_stdin_passwd   = False
+        self.stdin_passwd_sent = False
+
         self.backup_dir = os.path.join(self.base_dir, self.uri.replset)
         self.dump_dir   = os.path.join(self.backup_dir, "dump")
         self.oplog_file = os.path.join(self.dump_dir, "oplog.bson")
@@ -60,6 +64,18 @@ def parse_mongodump_line(self, line):
         except:
             return None
 
+    def is_password_prompt(self, line):
+        if self.do_stdin_passwd and ("Enter Password:" in line or "reading password from standard input" in line):
+            return True
+        return False
+
+    def handle_password_prompt(self):
+        if self.do_stdin_passwd and not self.stdin_passwd_sent:
+            logging.debug("Received password prompt from mongodump, writing password to stdin")
+            self._process.stdin.write(self.password + "\n")
+            self._process.stdin.flush()
+            self.stdin_passwd_sent = True
+
     def wait(self):
         try:
             while self._process.stderr:
@@ -68,7 +84,11 @@ def wait(self):
                     for fd in poll[0]:
                         read = self._process.stderr.readline()
                         line = self.parse_mongodump_line(read)
-                        if line:
+                        if not line:
+                            continue
+                        elif self.is_password_prompt(read):
+                            self.handle_password_prompt()
+                        else:
                             logging.info(line)
                 if self._process.poll() != None:
                     break
@@ -85,11 +105,19 @@ def mongodump_cmd(self):
             mongodump_flags.extend(["--numParallelCollections="+str(self.threads)])
         if self.dump_gzip:
             mongodump_flags.extend(["--gzip"])
+        if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")):
+            mongodump_flags.extend(["--readPreference=secondary"])
         if self.authdb and self.authdb != "admin":
             logging.debug("Using database %s for authentication" % self.authdb)
             mongodump_flags.extend(["--authenticationDatabase", self.authdb])
         if self.user and self.password:
-            mongodump_flags.extend(["-u", self.user, "-p", self.password])
+            # >= 3.0.2 supports password input via stdin to mask from ps
+            if tuple("3.0.2".split(".")) <= tuple(self.version.split(".")):
+                mongodump_flags.extend(["-u", self.user, "-p", '""'])
+                self.do_stdin_passwd = True
+            else:
+                logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.2.0 to resolve this") 
+                mongodump_flags.extend(["-u", self.user, "-p", self.password])
         mongodump_cmd.extend(mongodump_flags)
         return mongodump_cmd
 
@@ -106,7 +134,7 @@ def run(self):
                 rmtree(self.dump_dir)
             os.makedirs(self.dump_dir)
             logging.debug("Running mongodump cmd: %s" % mongodump_cmd)
-            self._process = Popen(mongodump_cmd, stderr=PIPE)
+            self._process = Popen(mongodump_cmd, stdin=PIPE, stderr=PIPE)
             self.wait()
             self.exit_code = self._process.returncode
             if self.exit_code > 0:

mongodb_consistent_backup/Backup/Mongodump/__init__.py

@@ -5,8 +5,8 @@ def config(parser):
     parser.add_argument("--backup.mongodump.binary", dest="backup.mongodump.binary",
                         help="Path to 'mongodump' binary (default: /usr/bin/mongodump)", default='/usr/bin/mongodump')
     parser.add_argument("--backup.mongodump.compression", dest="backup.mongodump.compression",
-                        help="Compression method to use on backup (default: gzip)", default="gzip",
-                        choices=["none", "gzip"])
+                        help="Compression method to use on backup (default: auto)", default="auto",
+                        choices=["auto", "none", "gzip"])
     parser.add_argument("--backup.mongodump.threads", dest="backup.mongodump.threads",
                         help="Number of threads to use for each mongodump process. There is 1 x mongodump per shard, be careful! (default: shards/CPUs)",
                         default=0, type=int)

mongodb_consistent_backup/Backup/__init__.py

@@ -1,3 +1,8 @@
 from Backup import Backup
 
 
+def config(parser):
+    parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
+    parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
+    parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
+    return parser

mongodb_consistent_backup/Common/Config.py

@@ -51,12 +51,9 @@ def makeParser(self):
         parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose output", default=False, action="store_true")
         parser.add_argument("-H", "--host", dest="host", help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)", default="localhost", type=str)
         parser.add_argument("-P", "--port", dest="port", help="MongoDB Port (default: 27017)", default=27017, type=int)
-        parser.add_argument("-u", "--user", dest="user", help="MongoDB Authentication Username (for optional auth)", type=str)
+        parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
-        parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
-        parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
-        parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)

mongodb_consistent_backup/Common/DB.py

@@ -45,7 +45,7 @@ def connect(self):
                 conn['admin'].command({"ping":1})
         except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
             logging.error("Unable to connect to %s! Error: %s" % (self.uri, e))
-            raise OperationError(e)
+            raise DBConnectionError(e)
         if conn is not None:
             self._conn = conn
         return self._conn

mongodb_consistent_backup/Logger.py

@@ -16,10 +16,10 @@ def __init__(self, config, backup_time):
 
         self.do_file_log = False
         if self.config.log_dir is not '':
-            if os.path.isdir(self.config.log_dir):
-                self.do_file_log = True
-            else:
-                print("ERROR: Log directory: %s does not exist! Skipping file-based logging" % self.config.log_dir)
+            self.do_file_log = True
+            if not os.path.isdir(self.config.log_dir):
+                print "WARNING: Creating logging directory: %s" % self.config.log_dir
+                os.mkdir(self.config.log_dir)
 
         self.log_format = '[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s'
         self.file_log   = None
@@ -41,7 +41,6 @@ def start_file_logger(self):
                 self.file_log.setLevel(self.log_level)
                 self.file_log.setFormatter(logging.Formatter(self.log_format))
                 logging.getLogger('').addHandler(self.file_log)
-                self.update_symlink()
             except OSError, e:
                 logging.warning("Could not start file log handler, writing to stdout only")
                 pass
@@ -50,18 +49,21 @@ def close(self):
         if self.file_log:
             self.file_log.close()
 
-    def compress(self):
+    def compress(self, current=False):
         gz_log = None
         try:
-            if not os.path.isfile(self.last_log) or self.last_log == self.backup_log_file:
-                return
-            logging.info("Compressing previous log file")
-            gz_file = "%s.gz" % self.last_log
+            compress_file = self.backup_log_file
+            if not current:
+                compress_file = self.last_log
+                if not os.path.isfile(self.last_log) or self.last_log == self.backup_log_file:
+                    return
+            logging.info("Compressing log file: %s" % compress_file)
+            gz_file = "%s.gz" % compress_file
             gz_log  = GzipFile(gz_file, "w+")
-            with open(self.last_log) as f:
+            with open(compress_file) as f:
                 for line in f:
                     gz_log.write(line)
-            os.remove(self.last_log)
+            os.remove(compress_file)
         finally:
             if gz_log:
                 gz_log.close()

mongodb_consistent_backup/Main.py

@@ -55,7 +55,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
             self.setup_logger()
             self.setup_signal_handlers()
             self.get_lock()
-            self.logger.start_file_logger()
+            self.logger.update_symlink()
             self.init()
             self.set_backup_dirs()
             self.get_db_conn()
@@ -74,6 +74,7 @@ def setup_logger(self):
         try:
             self.logger = Logger(self.config, self.backup_time)
             self.logger.start()
+            self.logger.start_file_logger()
         except Exception, e:
             self.exception("Could not start logger: %s" % e, e)
 
@@ -117,6 +118,7 @@ def get_lock(self):
             self.lock = Lock(self.config.lock_file)
         except Exception:
             logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lock_file, self.program_name))
+            self.logger.compress(True)
             sys.exit(1)
 
     def release_lock(self):

mongodb_consistent_backup/Oplog/Oplog.py

@@ -3,6 +3,7 @@
 
 from gzip import GzipFile
 from bson import BSON, decode_file_iter
+from bson.codec_options import CodecOptions
 
 from mongodb_consistent_backup.Errors import OperationError
 
@@ -44,7 +45,7 @@ def load(self):
         try:
             oplog = self.open()
             logging.debug("Reading oplog file %s" % self.oplog_file)
-            for change in decode_file_iter(oplog):
+            for change in decode_file_iter(oplog, CodecOptions(unicode_decode_error_handler="ignore")):
                 if 'ts' in change:
                     self._last_ts = change['ts']
                 if self._first_ts is None and self._last_ts is not None: