From 7488ed89d165a28db5cfa516169b701a7dfe7834 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 4 May 2017 15:57:04 +0200
Subject: [PATCH 01/25] Fix for issue #120

---
 mongodb_consistent_backup/Common/Config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py
index e92bad75..ef2693ad 100644
--- a/mongodb_consistent_backup/Common/Config.py
+++ b/mongodb_consistent_backup/Common/Config.py
@@ -51,7 +51,7 @@ def makeParser(self):
         parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose output", default=False, action="store_true")
         parser.add_argument("-H", "--host", dest="host", help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)", default="localhost", type=str)
         parser.add_argument("-P", "--port", dest="port", help="MongoDB Port (default: 27017)", default=27017, type=int)
-        parser.add_argument("-u", "--user", dest="user", help="MongoDB Authentication Username (for optional auth)", type=str)
+        parser.add_argument("-u", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
         parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)

From ed287e25a1d63dc4c22112655ce1a0eed99bb8e7 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 4 May 2017 16:00:15 +0200
Subject: [PATCH 02/25] Update pip cache dir flag (--download-cache was
 removed)

---
 scripts/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index e1236dcf..8e3c43ce 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -88,7 +88,7 @@ if [ -d ${srcdir} ]; then
 	source ${venvdir}/bin/activate
 
 	[ ! -d ${pipdir} ] && mkdir -p ${pipdir}
-	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install --download-cache=${pipdir} pex requests
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install --cache-dir=${pipdir} pex requests
 	if [ $? -gt 0 ]; then
 		echo "Failed to install pex utility for building!"
 		exit 1

From 520d159544991e1bd3eacdd0dd01d7f5ed5ae74a Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 4 May 2017 16:59:55 +0200
Subject: [PATCH 03/25] Support readPreference on mongodump 3.4.0+, set to
 secondary to enforce safety

---
 .../Backup/Mongodump/Mongodump.py             | 11 ++++----
 .../Backup/Mongodump/MongodumpThread.py       | 27 ++++++++++---------
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
index 23ec86e0..3958de9b 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
@@ -40,9 +40,8 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         with hide('running', 'warnings'), settings(warn_only=True):
             self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True)
 
-        if self.can_gzip():
-            if self.compression() == 'none':
-                self.compression('gzip')
+        if self.can_gzip() and self.compression() == 'none':
+            self.compression('gzip')
         elif self.compression() == 'gzip':
             logging.warning("mongodump gzip compression requested on binary that does not support gzip!")
 
@@ -121,8 +120,9 @@ def run(self):
                 self.authdb,
                 self.backup_dir,
                 self.binary,
+                self.version,
                 self.threads(),
-                self.do_gzip,
+                self.do_gzip(),
                 self.verbose
             )
             self.dump_threads.append(thread)
@@ -152,8 +152,9 @@ def run(self):
                     self.authdb,
                     self.backup_dir,
                     self.binary,
+                    self.version,
                     self.threads(),
-                    self.do_gzip,
+                    self.do_gzip(),
                     self.verbose
                 )]
                 self.dump_threads[0].start()
diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
index 32d40ebc..18190949 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
@@ -14,20 +14,21 @@
 
 # noinspection PyStringFormat
 class MongodumpThread(Process):
-    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary,
+    def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary, version,
                  threads=0, dump_gzip=False, verbose=False):
         Process.__init__(self)
-        self.state       = state
-        self.uri         = uri
-        self.timer       = timer
-        self.user        = user
-        self.password    = password
-        self.authdb      = authdb
-        self.base_dir    = base_dir
-        self.binary      = binary
-        self.threads     = threads
-        self.dump_gzip   = dump_gzip
-        self.verbose     = verbose
+        self.state     = state
+        self.uri       = uri
+        self.timer     = timer
+        self.user      = user
+        self.password  = password
+        self.authdb    = authdb
+        self.base_dir  = base_dir
+        self.binary    = binary
+        self.version   = version
+        self.threads   = threads
+        self.dump_gzip = dump_gzip
+        self.verbose   = verbose
 
         self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset)
         self.exit_code  = 1
@@ -85,6 +86,8 @@ def mongodump_cmd(self):
             mongodump_flags.extend(["--numParallelCollections="+str(self.threads)])
         if self.dump_gzip:
             mongodump_flags.extend(["--gzip"])
+        if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")):
+            mongodump_flags.extend(["--readPreference=secondary"])
         if self.authdb and self.authdb != "admin":
             logging.debug("Using database %s for authentication" % self.authdb)
             mongodump_flags.extend(["--authenticationDatabase", self.authdb])

From 73706aeac00152e67f344b21dc7076a613c25605 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Sat, 6 May 2017 00:51:42 +0200
Subject: [PATCH 04/25] Dockerfile to install 3.4 psmdb mongodump for 3.4.x
 support

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f55f6fe0..265adeab 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 FROM centos:centos7
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
-	yum install -y Percona-Server-MongoDB-32-tools zbackup && yum clean all && \
+	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all && \
 	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.1/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]

From 4f27f0a6eb547618e8ca669e19535d39f7b4b18c Mon Sep 17 00:00:00 2001
From: Hu Hailin <h@bulbit.jp>
Date: Thu, 11 May 2017 18:15:25 +0900
Subject: [PATCH 05/25] ignore bson unicode decode error

---
 mongodb_consistent_backup/Oplog/Oplog.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Oplog/Oplog.py b/mongodb_consistent_backup/Oplog/Oplog.py
index 42fd1dc1..2ae4102b 100644
--- a/mongodb_consistent_backup/Oplog/Oplog.py
+++ b/mongodb_consistent_backup/Oplog/Oplog.py
@@ -3,6 +3,7 @@
 
 from gzip import GzipFile
 from bson import BSON, decode_file_iter
+from bson.codec_options import CodecOptions
 
 from mongodb_consistent_backup.Errors import OperationError
 
@@ -44,7 +45,7 @@ def load(self):
         try:
             oplog = self.open()
             logging.debug("Reading oplog file %s" % self.oplog_file)
-            for change in decode_file_iter(oplog):
+            for change in decode_file_iter(oplog, CodecOptions(unicode_decode_error_handler="ignore")):
                 if 'ts' in change:
                     self._last_ts = change['ts']
                 if self._first_ts is None and self._last_ts is not None:

From a11f06e26b53de73637341d692e2ff7ededb8c69 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 13:43:29 +0200
Subject: [PATCH 06/25] Fix secondary quorum errors on large sets

---
 mongodb_consistent_backup/Replication/Replset.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py
index 472ccfeb..1edacaad 100644
--- a/mongodb_consistent_backup/Replication/Replset.py
+++ b/mongodb_consistent_backup/Replication/Replset.py
@@ -151,6 +151,7 @@ def find_secondary(self, force=False, quiet=False):
         if self.secondary and not force:
             return self.secondary
 
+        secondary_count = 0
         for member in rs_status['members']:
             member_uri = MongoUri(member['name'], 27017, rs_name)
             if member['state'] == 7:
@@ -186,12 +187,12 @@ def find_secondary(self, force=False, quiet=False):
                     if self.secondary is None or score > self.secondary['score']:
                         self.secondary = {
                             'replSet': rs_name,
-                            'count': 1 if self.secondary is None else self.secondary['count'] + 1,
                             'uri': member_uri,
                             'optime': optime_ts,
                             'score': score
                         }
                     log_msg = "Found SECONDARY %s" % member_uri
+                    secondary_count += 1
                 else:
                     log_msg = "Found SECONDARY %s with too high replication lag! Skipping" % member_uri
 
@@ -203,8 +204,7 @@ def find_secondary(self, force=False, quiet=False):
                 log_data['score']  = int(score)
                 logging.info("%s: %s" % (log_msg, str(log_data)))
                 self.replset_summary['secondary'] = { "member": member, "uri": member_uri.str(), "data": log_data }
-        if self.secondary is None or (self.secondary['count'] + 1) < quorum:
-            secondary_count = self.secondary['count'] + 1 if self.secondary else 0
+        if self.secondary is None or secondary_count < quorum:
             logging.error("Not enough valid secondaries in replset %s to take backup! Num replset members: %i, required quorum: %i" % (
                 rs_name,
                 secondary_count,

From 01980defae85e2ab71a927d05d3519babf83cd16 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 13:45:19 +0200
Subject: [PATCH 07/25] Increment count by 1 for PRIMARY

---
 mongodb_consistent_backup/Replication/Replset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py
index 1edacaad..9f8fb1ac 100644
--- a/mongodb_consistent_backup/Replication/Replset.py
+++ b/mongodb_consistent_backup/Replication/Replset.py
@@ -204,7 +204,7 @@ def find_secondary(self, force=False, quiet=False):
                 log_data['score']  = int(score)
                 logging.info("%s: %s" % (log_msg, str(log_data)))
                 self.replset_summary['secondary'] = { "member": member, "uri": member_uri.str(), "data": log_data }
-        if self.secondary is None or secondary_count < quorum:
+        if self.secondary is None or (secondary_count + 1) < quorum:
             logging.error("Not enough valid secondaries in replset %s to take backup! Num replset members: %i, required quorum: %i" % (
                 rs_name,
                 secondary_count,

From 198a4b9e163752e95d0f5b79c542bd8c6aff76e1 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 13:57:18 +0200
Subject: [PATCH 08/25] Support both types of pip cache flags

---
 scripts/build.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index 8e3c43ce..87229ee1 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -88,7 +88,10 @@ if [ -d ${srcdir} ]; then
 	source ${venvdir}/bin/activate
 
 	[ ! -d ${pipdir} ] && mkdir -p ${pipdir}
-	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install --cache-dir=${pipdir} pex requests
+	pip_flags="--download-cache=${pipdir}"
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pip --help | grep -q '\-\-cache\-dir'
+	[ $? = 0 ] && pip_flags="--cache-dir=${pipdir}"
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex requests
 	if [ $? -gt 0 ]; then
 		echo "Failed to install pex utility for building!"
 		exit 1
@@ -97,7 +100,7 @@ if [ -d ${srcdir} ]; then
 	if [ ! -d ${pexdir} ]; then
 		mkdir -p ${pexdir}
 	else
-		rm -f ${pexdir}/build/mongodb_consistent_backup-*.whl 
+		rm -f ${pexdir}/${mod_name}-*.whl 
 	fi
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir} ${builddir}

From 561c865c7a74e192d552e34619005e68735b33f7 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 15:30:09 +0200
Subject: [PATCH 09/25] adding blog to links

---
 README.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.rst b/README.rst
index 6edd58f1..52d2d750 100644
--- a/README.rst
+++ b/README.rst
@@ -222,6 +222,7 @@ Links
 
 - https://www.percona.com/blog/2016/07/25/mongodb-consistent-backups/
 - https://www.percona.com/blog/2017/01/09/mongodb-pit-backups-part-2/
+- https://www.percona.com/blog/2017/05/10/percona-lab-mongodb_consistent_backup-1-0-release-explained/
 - https://docs.mongodb.com/manual/reference/program/mongodump/
 - https://docs.mongodb.com/manual/reference/program/mongorestore/
 - http://zbackup.org

From b6501f7e249d2852659d70fa877d838622f824f6 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 16:44:19 +0200
Subject: [PATCH 10/25] Fix compression decision code for mongodump, fix build
 step caching wheel files

---
 conf/mongodb-consistent-backup.example.conf       |  6 +++---
 .../Backup/Mongodump/Mongodump.py                 | 15 ++++++++++-----
 .../Backup/Mongodump/__init__.py                  |  4 ++--
 mongodb_consistent_backup/Backup/__init__.py      |  5 +++++
 mongodb_consistent_backup/Common/Config.py        |  3 ---
 scripts/build.sh                                  |  2 +-
 6 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf
index fef12a24..210ce307 100644
--- a/conf/mongodb-consistent-backup.example.conf
+++ b/conf/mongodb-consistent-backup.example.conf
@@ -10,9 +10,9 @@ production:
     name: default
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
-  #    binary: [path]           (default: /usr/bin/mongodump)
-  #    compression: [none|gzip] (default: true - if supported)
-  #    threads: [1-16]          (default: auto-generated - shards/cpu)
+  #    binary: [path]                (default: /usr/bin/mongodump)
+  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated - shards/cpu)
   #replication:
   #  max_lag_secs: [1+]        (default: 5)
   #  min_priority: [0-999]     (default: 0)
diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
index 3958de9b..62a023b0 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
@@ -26,7 +26,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         self.replsets           = replsets
         self.sharding           = sharding
 
-        self.compression_supported = ['none', 'gzip']
+        self.compression_supported = ['auto', 'none', 'gzip']
         self.version               = 'unknown'
         self.threads_max           = 16
         self.config_replset        = False
@@ -40,10 +40,15 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         with hide('running', 'warnings'), settings(warn_only=True):
             self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True)
 
-        if self.can_gzip() and self.compression() == 'none':
-            self.compression('gzip')
-        elif self.compression() == 'gzip':
-            logging.warning("mongodump gzip compression requested on binary that does not support gzip!")
+        self.choose_compression()
+
+    def choose_compression(self):
+        if self.can_gzip():
+            if self.compression() == 'auto':
+                logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
+                self.compression('gzip')
+	elif self.compression() == 'gzip':
+            raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
     def can_gzip(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
diff --git a/mongodb_consistent_backup/Backup/Mongodump/__init__.py b/mongodb_consistent_backup/Backup/Mongodump/__init__.py
index d7cbeed5..ba960a08 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/__init__.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/__init__.py
@@ -5,8 +5,8 @@ def config(parser):
     parser.add_argument("--backup.mongodump.binary", dest="backup.mongodump.binary",
                         help="Path to 'mongodump' binary (default: /usr/bin/mongodump)", default='/usr/bin/mongodump')
     parser.add_argument("--backup.mongodump.compression", dest="backup.mongodump.compression",
-                        help="Compression method to use on backup (default: gzip)", default="gzip",
-                        choices=["none", "gzip"])
+                        help="Compression method to use on backup (default: auto)", default="auto",
+                        choices=["auto", "none", "gzip"])
     parser.add_argument("--backup.mongodump.threads", dest="backup.mongodump.threads",
                         help="Number of threads to use for each mongodump process. There is 1 x mongodump per shard, be careful! (default: shards/CPUs)",
                         default=0, type=int)
diff --git a/mongodb_consistent_backup/Backup/__init__.py b/mongodb_consistent_backup/Backup/__init__.py
index b3a39603..27ba8660 100644
--- a/mongodb_consistent_backup/Backup/__init__.py
+++ b/mongodb_consistent_backup/Backup/__init__.py
@@ -1,3 +1,8 @@
 from Backup import Backup
 
 
+def config(parser):
+    parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
+    parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
+    parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
+    return parser
diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py
index ef2693ad..7b46eb41 100644
--- a/mongodb_consistent_backup/Common/Config.py
+++ b/mongodb_consistent_backup/Common/Config.py
@@ -54,9 +54,6 @@ def makeParser(self):
         parser.add_argument("-u", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
-        parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
-        parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
-        parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
diff --git a/scripts/build.sh b/scripts/build.sh
index 87229ee1..ba45695e 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -100,7 +100,7 @@ if [ -d ${srcdir} ]; then
 	if [ ! -d ${pexdir} ]; then
 		mkdir -p ${pexdir}
 	else
-		rm -f ${pexdir}/${mod_name}-*.whl 
+		find ${pexdir} -type f -name "${mod_name}-*.whl" -print -delete
 	fi
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir} ${builddir}

From 08be509eab804d71a632d80c284889cdcacdd4b6 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 16:46:03 +0200
Subject: [PATCH 11/25] Conf comment fixes

---
 conf/mongodb-consistent-backup.example.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf
index 210ce307..24959f2c 100644
--- a/conf/mongodb-consistent-backup.example.conf
+++ b/conf/mongodb-consistent-backup.example.conf
@@ -23,14 +23,14 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: true - if used by backup stage)
+  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
   #  resolver_threads: [1+]   (default: 2 per CPU)
   #  tailer:
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip, none if backup is compressed)
+  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)

From ff6c63acda6ce96237a4a7301f4c038456e0c241 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 18:12:20 +0200
Subject: [PATCH 12/25] Raise ResolverThread.py exceptions to Resolver.py (and
 thus Main.py)

---
 .../Oplog/Resolver/Resolver.py                     | 14 +++++++++++---
 .../Oplog/Resolver/ResolverThread.py               |  3 ++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
index 3cc892d9..4b6bfdf2 100644
--- a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
+++ b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
@@ -40,6 +40,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, tailed_oplogs,
         self.completed = False
         self._pool     = None
         self._pooled   = []
+        self._results  = {}
         try:
             self._pool = Pool(processes=self.threads(None, 2))
         except Exception, e:
@@ -74,7 +75,13 @@ def wait(self):
             self._pool.close()
             while len(self._pooled):
                 logging.debug("Waiting for %i oplog resolver thread(s) to stop" % len(self._pooled))
-                sleep(2)
+                try:
+                    for thread_name in self._pooled:
+                        thread = self._results[thread_name]
+                        thread.get(1)
+                        sleep(2)
+                except Exception, e:
+                    raise e
             self._pool.terminate()
             logging.debug("Stopped all oplog resolve threads")
             self.stopped = True
@@ -100,7 +107,8 @@ def run(self):
                     raise OperationError("Backup oplog is newer than the tailed oplog!")
                 else:
                     try:
-                        self._pool.apply_async(ResolverThread(
+                        thread_name = uri.str()
+                        self._results[thread_name] = self._pool.apply_async(ResolverThread(
                             self.resolver_state[shard],
                             uri,
                             tailed_oplog.copy(),
@@ -108,7 +116,7 @@ def run(self):
                             self.get_consistent_end_ts(),
                             self.compression()
                         ).run, callback=self.done)
-                        self._pooled.append(uri.str())
+                        self._pooled.append(thread_name)
                     except Exception, e:
                         logging.fatal("Resolve failed for %s! Error: %s" % (uri, e))
                         raise Error(e)
diff --git a/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py b/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
index 74662c68..a4271ff3 100644
--- a/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
+++ b/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
@@ -4,6 +4,7 @@
 # noinspection PyPackageRequirements
 from bson import decode_file_iter
 
+from mongodb_consistent_backup.Errors import Error
 from mongodb_consistent_backup.Oplog import Oplog
 from mongodb_consistent_backup.Pipeline import PoolThread
 
@@ -46,7 +47,7 @@ def run(self):
             self.state.set('running', False)
             self.exit_code = 0
         except Exception, e:
-            logging.exception("Resolving of oplogs failed! Error: %s" % e)
+            raise Error("Resolving of oplogs failed! Error: %s" % e)
         finally:
             self.close()
 

From d8cb0cdcd27dc9c78832fdb802b3909adf606ca7 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 20:17:48 +0200
Subject: [PATCH 13/25] Write password to mongodump stdin if >= 3.0.2

---
 .../Backup/Mongodump/MongodumpThread.py       | 42 ++++++++++++++++---
 scripts/build.sh                              |  2 +-
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
index 18190949..503051f3 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
@@ -7,6 +7,7 @@
 from shutil import rmtree
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from subprocess import Popen, PIPE
+from time import sleep
 
 from mongodb_consistent_backup.Common import is_datetime
 from mongodb_consistent_backup.Oplog import Oplog
@@ -30,9 +31,12 @@ def __init__(self, state, uri, timer, user, password, authdb, base_dir, binary,
         self.dump_gzip = dump_gzip
         self.verbose   = verbose
 
-        self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset)
-        self.exit_code  = 1
-        self._command   = None
+        self.timer_name        = "%s-%s" % (self.__class__.__name__, self.uri.replset)
+        self.exit_code         = 1
+        self._command          = None
+        self.do_stdin_passwd   = False
+        self.stdin_passwd_sent = False
+
         self.backup_dir = os.path.join(self.base_dir, self.uri.replset)
         self.dump_dir   = os.path.join(self.backup_dir, "dump")
         self.oplog_file = os.path.join(self.dump_dir, "oplog.bson")
@@ -61,6 +65,18 @@ def parse_mongodump_line(self, line):
         except:
             return None
 
+    def is_password_prompt(self, line):
+        if self.do_stdin_passwd and ("Enter Password:" in line or "reading password from standard input" in line):
+            return True
+        return False
+
+    def handle_password_prompt(self):
+        if self.do_stdin_passwd and not self.stdin_passwd_sent:
+            logging.debug("Received password prompt from mongodump, writing password to stdin")
+            self._process.stdin.write(self.password + "\n")
+            self._process.stdin.flush()
+            self.stdin_passwd_sent = True
+
     def wait(self):
         try:
             while self._process.stderr:
@@ -69,7 +85,11 @@ def wait(self):
                     for fd in poll[0]:
                         read = self._process.stderr.readline()
                         line = self.parse_mongodump_line(read)
-                        if line:
+                        if not line:
+                            continue
+                        elif self.is_password_prompt(read):
+                            self.handle_password_prompt()
+                        else:
                             logging.info(line)
                 if self._process.poll() != None:
                     break
@@ -92,7 +112,12 @@ def mongodump_cmd(self):
             logging.debug("Using database %s for authentication" % self.authdb)
             mongodump_flags.extend(["--authenticationDatabase", self.authdb])
         if self.user and self.password:
-            mongodump_flags.extend(["-u", self.user, "-p", self.password])
+            # >= 3.0.2 supports password input via stdin to mask from ps
+            if tuple("3.0.2".split(".")) <= tuple(self.version.split(".")):
+                mongodump_flags.extend(["-u", self.user, "-p", '""'])
+                self.do_stdin_passwd = True
+            else:
+                mongodump_flags.extend(["-u", self.user, "-p", self.password])
         mongodump_cmd.extend(mongodump_flags)
         return mongodump_cmd
 
@@ -109,7 +134,12 @@ def run(self):
                 rmtree(self.dump_dir)
             os.makedirs(self.dump_dir)
             logging.debug("Running mongodump cmd: %s" % mongodump_cmd)
-            self._process = Popen(mongodump_cmd, stderr=PIPE)
+            self._process = Popen(mongodump_cmd, stdin=PIPE, stderr=PIPE)
+            #if self.do_stdin_passwd:
+            #    sleep(5)
+            #    logging.debug("Writing password to stdin")
+            #    self._process.stdin.write(self.password)
+            #    self._process.stdin.flush()
             self.wait()
             self.exit_code = self._process.returncode
             if self.exit_code > 0:
diff --git a/scripts/build.sh b/scripts/build.sh
index ba45695e..2793a8e9 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -100,7 +100,7 @@ if [ -d ${srcdir} ]; then
 	if [ ! -d ${pexdir} ]; then
 		mkdir -p ${pexdir}
 	else
-		find ${pexdir} -type f -name "${mod_name}-*.whl" -print -delete
+		find ${pexdir} -type f -name "${mod_name}-*.whl" -delete
 	fi
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir} ${builddir}

From 1ccbc245ef01ca0670186ca0b2c9e92f5e4db538 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 20:18:47 +0200
Subject: [PATCH 14/25] forgotten cleanup

---
 .../Backup/Mongodump/MongodumpThread.py                     | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
index 503051f3..d97cc698 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
@@ -7,7 +7,6 @@
 from shutil import rmtree
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from subprocess import Popen, PIPE
-from time import sleep
 
 from mongodb_consistent_backup.Common import is_datetime
 from mongodb_consistent_backup.Oplog import Oplog
@@ -135,11 +134,6 @@ def run(self):
             os.makedirs(self.dump_dir)
             logging.debug("Running mongodump cmd: %s" % mongodump_cmd)
             self._process = Popen(mongodump_cmd, stdin=PIPE, stderr=PIPE)
-            #if self.do_stdin_passwd:
-            #    sleep(5)
-            #    logging.debug("Writing password to stdin")
-            #    self._process.stdin.write(self.password)
-            #    self._process.stdin.flush()
             self.wait()
             self.exit_code = self._process.returncode
             if self.exit_code > 0:

From 803ec9dc7dc7079b9a5eaf1d182ed09e26b28f2b Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Thu, 11 May 2017 20:26:14 +0200
Subject: [PATCH 15/25] Warn users if their password is on the command line

---
 mongodb_consistent_backup/Backup/Mongodump/Mongodump.py       | 2 +-
 mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
index 62a023b0..fc3c99c9 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
@@ -47,7 +47,7 @@ def choose_compression(self):
             if self.compression() == 'auto':
                 logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
                 self.compression('gzip')
-	elif self.compression() == 'gzip':
+        elif self.compression() == 'gzip':
             raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
     def can_gzip(self):
diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
index d97cc698..d748be3a 100644
--- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
+++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py
@@ -116,6 +116,7 @@ def mongodump_cmd(self):
                 mongodump_flags.extend(["-u", self.user, "-p", '""'])
                 self.do_stdin_passwd = True
             else:
+                logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.2.0 to resolve this") 
                 mongodump_flags.extend(["-u", self.user, "-p", self.password])
         mongodump_cmd.extend(mongodump_flags)
         return mongodump_cmd

From 0f6aa82085a0a41bebc6a35086dafa60d5c69ddc Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Fri, 12 May 2017 13:22:07 +0200
Subject: [PATCH 16/25] 'stopped' var should be set to False until stop is ran

---
 mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py b/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
index a4271ff3..c7798904 100644
--- a/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
+++ b/mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py
@@ -21,7 +21,7 @@ def __init__(self, state, uri, tailed_oplog, mongodump_oplog, max_end_ts, compre
 
         self.oplogs  = {}
         self.changes = 0
-        self.stopped = True
+        self.stopped = False
 
     def run(self):
         self.oplogs['backup'] = Oplog(self.mongodump_oplog['file'], self.do_gzip(), 'a+')

From 075a0b3f6e13ff75ee0dcce10329339c16d90025 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Mon, 15 May 2017 12:13:13 +0200
Subject: [PATCH 17/25] Support 3.4.x CSRS-based balancer

---
 mongodb_consistent_backup/Sharding.py | 62 +++++++++++++++++----------
 1 file changed, 40 insertions(+), 22 deletions(-)

diff --git a/mongodb_consistent_backup/Sharding.py b/mongodb_consistent_backup/Sharding.py
index 15d1c16a..848223ee 100644
--- a/mongodb_consistent_backup/Sharding.py
+++ b/mongodb_consistent_backup/Sharding.py
@@ -38,6 +38,9 @@ def close(self):
             self.config_db.close()
         return self.restore_balancer_state()
 
+    def is_gte_34(self):
+        return self.db.server_version() >= tuple("3.4.0".split("."))
+
     def get_start_state(self):
         self._balancer_state_start = self.get_balancer_state()
         logging.info("Began with balancer state running: %s" % str(self._balancer_state_start))
@@ -45,49 +48,64 @@ def get_start_state(self):
 
     def shards(self):
         try:
-            if self.db.is_configsvr() and self.db.server_version() < tuple("3.4.0".split(".")):
-                return self.connection['config'].shards.find()
-            else:
+            if self.is_gte_34():
                 listShards = self.db.admin_command("listShards")
                 if 'shards' in listShards:
                     return listShards['shards']
+            elif self.db.is_configsvr():
+                return self.connection['config'].shards.find()
         except Exception, e:
             raise DBOperationError(e)
 
     def check_balancer_running(self):
         try:
-            config = self.connection['config']
-            lock   = config['locks'].find_one({'_id': 'balancer'})
-            if 'state' in lock and int(lock['state']) == 0:
-                return False
+            if self.is_gte_34():
+                balancerState = self.db.admin_command("balancerStatus")
+                if 'inBalancerRound' in balancerState:
+                    return balancerState['inBalancerRound']
+            else:
+                config = self.connection['config']
+                lock = config['locks'].find_one({'_id': 'balancer'})
+                if 'state' in lock and int(lock['state']) == 0:
+                    return False
             return True
         except Exception, e:
             raise DBOperationError(e)
 
     def get_balancer_state(self):
         try:
-            config = self.connection['config']
-            state  = config['settings'].find_one({'_id': 'balancer'})
-
-            if not state:
-               return True
-            elif 'stopped' in state and state.get('stopped') is True:
-               return False
+            if self.is_gte_34():
+                balancerState = self.db.admin_command("balancerStatus")
+                if 'mode' in balancerState and balancerState['mode'] == 'off':
+                    return False
+                return True
             else:
-               return True
+                config = self.connection['config']
+                state  = config['settings'].find_one({'_id': 'balancer'})
+                if not state:
+                   return True
+                elif 'stopped' in state and state.get('stopped') is True:
+                   return False
+                return True
         except Exception, e:
             raise DBOperationError(e)
 
     def set_balancer(self, value):
         try:
-            if value is True:
-                set_value = False
-            elif value is False:
-                set_value = True
+            if self.is_gte_34():
+                if value is True:
+                    self.db.admin_command("balancerStart")
+                else:
+                    self.db.admin_command("balancerStop")
             else:
-                set_value = True
-            config = self.connection['config']
-            config['settings'].update_one({'_id': 'balancer'}, {'$set': {'stopped': set_value}})
+                if value is True:
+                    set_value = False
+                elif value is False:
+                    set_value = True
+                else:
+                    set_value = True
+                config = self.connection['config']
+                config['settings'].update_one({'_id': 'balancer'}, {'$set': {'stopped': set_value}})
         except Exception, e:
             logging.fatal("Failed to set balancer state! Error: %s" % e)
             raise DBOperationError(e)

From cc0731e694b352909322f41e1a53c9f84a437d5d Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Mon, 15 May 2017 17:20:56 +0200
Subject: [PATCH 18/25] Ensure we use a mongos connection for 3.4 balancer
 commands. Even though the configsvr is now the balancer it doesn't have the
 commands yet

---
 mongodb_consistent_backup/Common/DB.py |  2 +-
 mongodb_consistent_backup/Sharding.py  | 39 +++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py
index 71af001c..d6afabe9 100644
--- a/mongodb_consistent_backup/Common/DB.py
+++ b/mongodb_consistent_backup/Common/DB.py
@@ -45,7 +45,7 @@ def connect(self):
                 conn['admin'].command({"ping":1})
         except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
             logging.error("Unable to connect to %s! Error: %s" % (self.uri, e))
-            raise OperationError(e)
+            raise DBConnectionError(e)
         if conn is not None:
             self._conn = conn
         return self._conn
diff --git a/mongodb_consistent_backup/Sharding.py b/mongodb_consistent_backup/Sharding.py
index 848223ee..5e4ca6da 100644
--- a/mongodb_consistent_backup/Sharding.py
+++ b/mongodb_consistent_backup/Sharding.py
@@ -1,5 +1,6 @@
 import logging
 
+from pymongo import DESCENDING
 from time import sleep
 
 from mongodb_consistent_backup.Common import DB, MongoUri, validate_hostname
@@ -18,6 +19,7 @@ def __init__(self, config, timer, db):
         self.timer_name            = self.__class__.__name__
         self.config_server         = None
         self.config_db             = None
+        self.mongos_db             = None
         self._balancer_state_start = None
         self.restored              = False
 
@@ -36,11 +38,31 @@ def __init__(self, config, timer, db):
     def close(self):
         if self.config_db:
             self.config_db.close()
+        if self.mongos_db:
+            self.mongos_db.close()
         return self.restore_balancer_state()
 
     def is_gte_34(self):
         return self.db.server_version() >= tuple("3.4.0".split("."))
 
+    def get_mongos(self, force=False):
+        if not force and self.mongos_db:
+            return self.mongos_db
+        elif self.db.is_mongos():
+            return self.db
+        else:
+            db = self.connection['config']
+            for doc in db.mongos.find().sort('ping', DESCENDING):
+                try:
+                    mongos_uri = MongoUri(doc['_id'])
+                    logging.debug("Found cluster mongos: %s" % mongos_uri)
+                    self.mongos_db = DB(mongos_uri, self.config, False, 'nearest')
+                    logging.info("Connected to cluster mongos: %s" % mongos_uri)
+                    return self.mongos_db
+                except DBConnectionFailure, e:
+                    logging.debug("Failed to connect to mongos: %s, trying next available mongos" % mongos_uri)
+            raise OperationError('Could not connect to any mongos!')
+
     def get_start_state(self):
         self._balancer_state_start = self.get_balancer_state()
         logging.info("Began with balancer state running: %s" % str(self._balancer_state_start))
@@ -48,19 +70,20 @@ def get_start_state(self):
 
     def shards(self):
         try:
-            if self.is_gte_34():
+            if self.db.is_configsvr() or not self.is_gte_34():
+                return self.connection['config'].shards.find()
+            elif self.is_gte_34():
                 listShards = self.db.admin_command("listShards")
                 if 'shards' in listShards:
                     return listShards['shards']
-            elif self.db.is_configsvr():
-                return self.connection['config'].shards.find()
         except Exception, e:
             raise DBOperationError(e)
 
     def check_balancer_running(self):
         try:
             if self.is_gte_34():
-                balancerState = self.db.admin_command("balancerStatus")
+                # 3.4+ configsvrs dont have balancerStatus, use self.get_mongos() to get a mongos connection for now
+                balancerState = self.get_mongos().admin_command("balancerStatus")
                 if 'inBalancerRound' in balancerState:
                     return balancerState['inBalancerRound']
             else:
@@ -75,7 +98,8 @@ def check_balancer_running(self):
     def get_balancer_state(self):
         try:
             if self.is_gte_34():
-                balancerState = self.db.admin_command("balancerStatus")
+                # 3.4+ configsvrs dont have balancerStatus, use self.get_mongos() to get a mongos connection for now
+                balancerState = self.get_mongos().admin_command("balancerStatus")
                 if 'mode' in balancerState and balancerState['mode'] == 'off':
                     return False
                 return True
@@ -93,10 +117,11 @@ def get_balancer_state(self):
     def set_balancer(self, value):
         try:
             if self.is_gte_34():
+                # 3.4+ configsvrs dont have balancerStart/Stop, even though they're the balancer! Use self.get_mongos() to get a mongos connection for now
                 if value is True:
-                    self.db.admin_command("balancerStart")
+                    self.get_mongos().admin_command("balancerStart")
                 else:
-                    self.db.admin_command("balancerStop")
+                    self.get_mongos().admin_command("balancerStop")
             else:
                 if value is True:
                     set_value = False

From fcc8bb7b2a58c2477cb18a5d6c3e6929a9f200d6 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Mon, 15 May 2017 18:25:50 +0200
Subject: [PATCH 19/25] Make log dir if not exists

---
 mongodb_consistent_backup/Logger.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mongodb_consistent_backup/Logger.py b/mongodb_consistent_backup/Logger.py
index e56789fc..2f87b92a 100644
--- a/mongodb_consistent_backup/Logger.py
+++ b/mongodb_consistent_backup/Logger.py
@@ -35,6 +35,8 @@ def start(self):
     def start_file_logger(self):
         if self.do_file_log:
             try:
+                if os.path.isdir(self.config.log_dir):
+                    os.mkdir(self.config.log_dir)
                 self.current_log_file = os.path.join(self.config.log_dir, "backup.%s.log" % self.backup_name)
                 self.backup_log_file  = os.path.join(self.config.log_dir, "backup.%s.%s.log" % (self.backup_name, self.backup_time))
                 self.file_log = logging.FileHandler(self.backup_log_file)

From 2abc8bb5374b9aea68858662ff1bfc4cdaaed6c8 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Mon, 15 May 2017 18:57:39 +0200
Subject: [PATCH 20/25] Write log if cannot get lock

---
 mongodb_consistent_backup/Logger.py | 27 +++++++++++++--------------
 mongodb_consistent_backup/Main.py   |  4 +++-
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/mongodb_consistent_backup/Logger.py b/mongodb_consistent_backup/Logger.py
index 2f87b92a..ac2a91f0 100644
--- a/mongodb_consistent_backup/Logger.py
+++ b/mongodb_consistent_backup/Logger.py
@@ -16,10 +16,9 @@ def __init__(self, config, backup_time):
 
         self.do_file_log = False
         if self.config.log_dir is not '':
-            if os.path.isdir(self.config.log_dir):
-                self.do_file_log = True
-            else:
-                print("ERROR: Log directory: %s does not exist! Skipping file-based logging" % self.config.log_dir)
+            self.do_file_log = True
+            if not os.path.isdir(self.config.log_dir):
+                os.mkdir(self.config.log_dir)
 
         self.log_format = '[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s'
         self.file_log   = None
@@ -35,15 +34,12 @@ def start(self):
     def start_file_logger(self):
         if self.do_file_log:
             try:
-                if os.path.isdir(self.config.log_dir):
-                    os.mkdir(self.config.log_dir)
                 self.current_log_file = os.path.join(self.config.log_dir, "backup.%s.log" % self.backup_name)
                 self.backup_log_file  = os.path.join(self.config.log_dir, "backup.%s.%s.log" % (self.backup_name, self.backup_time))
                 self.file_log = logging.FileHandler(self.backup_log_file)
                 self.file_log.setLevel(self.log_level)
                 self.file_log.setFormatter(logging.Formatter(self.log_format))
                 logging.getLogger('').addHandler(self.file_log)
-                self.update_symlink()
             except OSError, e:
                 logging.warning("Could not start file log handler, writing to stdout only")
                 pass
@@ -52,18 +48,21 @@ def close(self):
         if self.file_log:
             self.file_log.close()
 
-    def compress(self):
+    def compress(self, current=False):
         gz_log = None
         try:
-            if not os.path.isfile(self.last_log) or self.last_log == self.backup_log_file:
-                return
-            logging.info("Compressing previous log file")
-            gz_file = "%s.gz" % self.last_log
+            compress_file = self.backup_log_file
+            if not current:
+                compress_file = self.last_log
+                if not os.path.isfile(self.last_log) or self.last_log == self.backup_log_file:
+                    return
+            logging.info("Compressing log file: %s" % compress_file)
+            gz_file = "%s.gz" % compress_file
             gz_log  = GzipFile(gz_file, "w+")
-            with open(self.last_log) as f:
+            with open(compress_file) as f:
                 for line in f:
                     gz_log.write(line)
-            os.remove(self.last_log)
+            os.remove(compress_file)
         finally:
             if gz_log:
                 gz_log.close()
diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py
index 479455ff..23f001db 100644
--- a/mongodb_consistent_backup/Main.py
+++ b/mongodb_consistent_backup/Main.py
@@ -55,7 +55,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
             self.setup_logger()
             self.setup_signal_handlers()
             self.get_lock()
-            self.logger.start_file_logger()
+            self.logger.update_symlink()
             self.init()
             self.set_backup_dirs()
             self.get_db_conn()
@@ -74,6 +74,7 @@ def setup_logger(self):
         try:
             self.logger = Logger(self.config, self.backup_time)
             self.logger.start()
+            self.logger.start_file_logger()
         except Exception, e:
             self.exception("Could not start logger: %s" % e, e)
 
@@ -117,6 +118,7 @@ def get_lock(self):
             self.lock = Lock(self.config.lock_file)
         except Exception:
             logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lock_file, self.program_name))
+            self.logger.compress(True)
             sys.exit(1)
 
     def release_lock(self):

From 76ac5b6c0b5dde185bf3943b01ebf86cf289ff64 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Mon, 15 May 2017 19:45:02 +0200
Subject: [PATCH 21/25] version increment to 1.0.2

---
 Dockerfile | 2 +-
 VERSION    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 265adeab..d1a6fc45 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@ FROM centos:centos7
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
 	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all && \
-	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.1/mongodb-consistent-backup.el7.centos.x86_64 && \
+	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.2/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]
diff --git a/VERSION b/VERSION
index 7dea76ed..6d7de6e6 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.0.1
+1.0.2

From 5d05a9a31ca017a9d22aa2beb750339c7805dc10 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Tue, 16 May 2017 13:56:56 +0200
Subject: [PATCH 22/25] Handle TimeoutError from .get() on Popen results

---
 .../Oplog/Resolver/Resolver.py                 | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
index 4b6bfdf2..30278c88 100644
--- a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
+++ b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
@@ -4,7 +4,7 @@
 # noinspection PyPackageRequirements
 from bson.timestamp import Timestamp
 from copy_reg import pickle
-from multiprocessing import Pool
+from multiprocessing import Pool, TimeoutError
 from time import sleep
 from types import MethodType
 
@@ -70,20 +70,24 @@ def done(self, done_uri):
         else:
             raise OperationError("Unexpected response from resolver thread: %s" % done_uri)
 
-    def wait(self):
+    def wait(self, max_wait_secs=6*3600):
         if len(self._pooled) > 0:
+            waited_secs = 0
             self._pool.close()
             while len(self._pooled):
                 logging.debug("Waiting for %i oplog resolver thread(s) to stop" % len(self._pooled))
                 try:
                     for thread_name in self._pooled:
                         thread = self._results[thread_name]
-                        thread.get(1)
-                        sleep(2)
-                except Exception, e:
-                    raise e
+                        thread.get(2)
+                except TimeoutError:
+                    if waited_secs < max_wait_secs:
+                        waited_secs += 2
+                        continue
+                    else:
+                        raise OperationError("Waited more than %i seconds for Oplog resolver! I will assume there is a problem and exit") 
             self._pool.terminate()
-            logging.debug("Stopped all oplog resolve threads")
+            logging.debug("Stopped all oplog resolver threads")
             self.stopped = True
             self.running = False
 

From 0b6bbaa6e07bca380b34394d27c935132e71635a Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Tue, 16 May 2017 13:59:42 +0200
Subject: [PATCH 23/25] Make poll/sleep secs a var

---
 mongodb_consistent_backup/Oplog/Resolver/Resolver.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
index 30278c88..dc81f2fe 100644
--- a/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
+++ b/mongodb_consistent_backup/Oplog/Resolver/Resolver.py
@@ -70,7 +70,7 @@ def done(self, done_uri):
         else:
             raise OperationError("Unexpected response from resolver thread: %s" % done_uri)
 
-    def wait(self, max_wait_secs=6*3600):
+    def wait(self, max_wait_secs=6*3600, poll_secs=2):
         if len(self._pooled) > 0:
             waited_secs = 0
             self._pool.close()
@@ -79,11 +79,10 @@ def wait(self, max_wait_secs=6*3600):
                 try:
                     for thread_name in self._pooled:
                         thread = self._results[thread_name]
-                        thread.get(2)
+                        thread.get(poll_secs)
                 except TimeoutError:
                     if waited_secs < max_wait_secs:
-                        waited_secs += 2
-                        continue
+                        waited_secs += poll_secs
                     else:
                         raise OperationError("Waited more than %i seconds for Oplog resolver! I will assume there is a problem and exit") 
             self._pool.terminate()

From 455e6213d81dae5bb36f81707bf4116158a90a8c Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Tue, 16 May 2017 18:33:49 +0200
Subject: [PATCH 24/25] Add backwards-compatable --user flag

---
 mongodb_consistent_backup/Common/Config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py
index 7b46eb41..bac7e859 100644
--- a/mongodb_consistent_backup/Common/Config.py
+++ b/mongodb_consistent_backup/Common/Config.py
@@ -51,7 +51,7 @@ def makeParser(self):
         parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose output", default=False, action="store_true")
         parser.add_argument("-H", "--host", dest="host", help="MongoDB Hostname, IP address or '<replset>/<host:port>,<host:port>,..' URI (default: localhost)", default="localhost", type=str)
         parser.add_argument("-P", "--port", dest="port", help="MongoDB Port (default: 27017)", default=27017, type=int)
-        parser.add_argument("-u", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
+        parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)

From 9e27e4c111495a65dcc6b52462765ecf16b3ac36 Mon Sep 17 00:00:00 2001
From: Tim Vaillancourt <tim@timvaillancourt.com>
Date: Tue, 16 May 2017 18:36:17 +0200
Subject: [PATCH 25/25] Warn if the log dir was created

---
 mongodb_consistent_backup/Logger.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mongodb_consistent_backup/Logger.py b/mongodb_consistent_backup/Logger.py
index ac2a91f0..84b29d13 100644
--- a/mongodb_consistent_backup/Logger.py
+++ b/mongodb_consistent_backup/Logger.py
@@ -18,6 +18,7 @@ def __init__(self, config, backup_time):
         if self.config.log_dir is not '':
             self.do_file_log = True
             if not os.path.isdir(self.config.log_dir):
+                print "WARNING: Creating logging directory: %s" % self.config.log_dir
                 os.mkdir(self.config.log_dir)
 
         self.log_format = '[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s'