From a7fafe8370cbd002c36d3f348e575ec997616cf2 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:00:31 +0200 Subject: [PATCH 01/22] Allow SSL variables --- conf/mongodb-consistent-backup.example.conf | 7 +++ mongodb_consistent_backup/Common/Config.py | 6 +++ mongodb_consistent_backup/Common/DB.py | 55 ++++++++++++++++----- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index 0e0c8321..e2629a2a 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -4,6 +4,13 @@ production: #username: [auth username] (default: none) #password: [auth password] (default: none) #authdb: [auth database] (default: admin) + #ssl: + # enabled: [true|false] (default: false) + # validate: [true|false] (default: true) + # ca_file: [path] + # crl_file: [path] + # client_cert_file: [path] + # client_key_file: [path] log_dir: /var/log/mongodb-consistent-backup backup: method: mongodump diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ad52b25a..ed09a19f 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -54,6 +54,12 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, type=bool) + parser.add_argument("--ssl.validate", dest="ssl.validate", help="Validate the SSL certificate and hostname of the server (default: true)", default=True, type=bool) + parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file", default=None, type=str) + parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) + parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Key file (for optional client ssl auth)", default=None, type=str) + parser.add_argument("--ssl.client_key_file", dest="ssl.client_key_file", help="Path to Client SSL Certificate file (if not provided by client_cert_file, for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index eec89775..c5a144c0 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -1,4 +1,5 @@ import logging +import ssl from bson.codec_options import CodecOptions from inspect import currentframe, getframeinfo @@ -21,29 +22,59 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.conn_timeout = conn_timeout self.retries = retries + self.ssl_enabled = config.ssl.enabled + self.ssl_validate = config.ssl.validate + self.ssl_ca_file = config.ssl.ca_file + self.ssl_crl_file = config.ssl.crl_file + self.ssl_client_cert_file = config.ssl.client_cert_file + self.ssl_client_key_file = config.ssl.client_key_file + self.replset = None self._conn = None self._is_master = None self.connect() self.auth_if_required() + def do_ssl(self): + if self.ssl_enabled: + return True + return False + def connect(self): try: + opts = { + "connect": self.do_connect, + "host": self.uri.hosts(), + "connectTimeoutMS": self.conn_timeout, + "serverSelectionTimeoutMS": self.conn_timeout, + "maxPoolSize": 1, + } if self.do_replset: self.replset = self.uri.replset - logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s)" % ( - self.uri, self.replset, self.read_pref + opts.update({ + "replicaSet": self.replset, + "readPreference": self.read_pref, + "w": "majority" + }) + if self.do_ssl(): + logging.debug("Enabling SSL security on database connection") + opts.update({ + "ssl": True, + "ssl_ca_certs": self.ssl_ca_file, + "ssl_crlfile": self.ssl_crl_file, + "ssl_certfile": self.ssl_client_cert_file, + "ssl_keyfile": self.ssl_client_key_file, + "ssl_cert_reqs": ssl.CERT_REQUIRED, + }) + if not self.ssl_validate: + opts.update({ + "ssl_cert_reqs": ssl.CERT_NONE + }) + conn = MongoClient(opts) + logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( + self.uri, self.replset, self.read_pref, self.do_ssl() )) - conn = MongoClient( - connect=self.do_connect, - host=self.uri.hosts(), - replicaSet=self.replset, - readPreference=self.read_pref, - connectTimeoutMS=self.conn_timeout, - serverSelectionTimeoutMS=self.conn_timeout, - maxPoolSize=1, - w="majority" - ) + conn = MongoClient(opts) if self.do_connect: conn['admin'].command({"ping": 1}) except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: From 454ef51a240791c788f6eab622fd234bca9ea5f3 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:14:25 +0200 Subject: [PATCH 02/22] Add SSL capabilities to Mongodump backup method, enforce PEM-only for client cert as mongodump requires PEM-only --- .../Backup/Mongodump/MongodumpThread.py | 23 +++++++++++++++---- mongodb_consistent_backup/Common/Config.py | 3 +-- mongodb_consistent_backup/Common/DB.py | 2 -- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 2ac39eed..924db699 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -25,10 +25,15 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.threads = threads self.dump_gzip = dump_gzip - self.user = self.config.username - self.password = self.config.password - self.authdb = self.config.authdb - self.binary = self.config.backup.mongodump.binary + self.user = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_enabled = self.config.ssl.enabled + self.ssl_validate = self.config.ssl.validate + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.binary = self.config.backup.mongodump.binary self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset) self.exit_code = 1 @@ -133,6 +138,16 @@ def mongodump_cmd(self): else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) + if self.ssl_enabled: + mongodump_flags.append("--ssl") + if self.ssl_ca_file: + mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + if self.ssl_crl_file: + mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + if self.client_cert_file: + mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + if not self.ssl_validate: + mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) mongodump_cmd.extend(mongodump_flags) return mongodump_cmd diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ed09a19f..633550f5 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -58,8 +58,7 @@ def makeParser(self): parser.add_argument("--ssl.validate", dest="ssl.validate", help="Validate the SSL certificate and hostname of the server (default: true)", default=True, type=bool) parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) - parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Key file (for optional client ssl auth)", default=None, type=str) - parser.add_argument("--ssl.client_key_file", dest="ssl.client_key_file", help="Path to Client SSL Certificate file (if not provided by client_cert_file, for optional client ssl auth)", default=None, type=str) + parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL PEM file (for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index c5a144c0..1e22f3cd 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -27,7 +27,6 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.ssl_ca_file = config.ssl.ca_file self.ssl_crl_file = config.ssl.crl_file self.ssl_client_cert_file = config.ssl.client_cert_file - self.ssl_client_key_file = config.ssl.client_key_file self.replset = None self._conn = None @@ -63,7 +62,6 @@ def connect(self): "ssl_ca_certs": self.ssl_ca_file, "ssl_crlfile": self.ssl_crl_file, "ssl_certfile": self.ssl_client_cert_file, - "ssl_keyfile": self.ssl_client_key_file, "ssl_cert_reqs": ssl.CERT_REQUIRED, }) if not self.ssl_validate: From c061a4f75ba592a16dc0ae605895f3ac37fbb710 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:15:06 +0200 Subject: [PATCH 03/22] Add SSL capabilities to Mongodump backup method, enforce PEM-only for client cert as mongodump requires PEM-only #2 --- conf/mongodb-consistent-backup.example.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index e2629a2a..c2c170ef 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -10,7 +10,6 @@ production: # ca_file: [path] # crl_file: [path] # client_cert_file: [path] - # client_key_file: [path] log_dir: /var/log/mongodb-consistent-backup backup: method: mongodump From 71ca07c23d2bdfee3e5a581c587fdab83610a90f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:22:46 +0200 Subject: [PATCH 04/22] Make config a class var like the rest of the codebase --- mongodb_consistent_backup/Common/DB.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 1e22f3cd..8891fbb1 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -13,20 +13,21 @@ class DB: def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5): self.uri = uri - self.username = config.username - self.password = config.password - self.authdb = config.authdb + self.config = config self.do_replset = do_replset self.read_pref = read_pref self.do_connect = do_connect self.conn_timeout = conn_timeout self.retries = retries - self.ssl_enabled = config.ssl.enabled - self.ssl_validate = config.ssl.validate - self.ssl_ca_file = config.ssl.ca_file - self.ssl_crl_file = config.ssl.crl_file - self.ssl_client_cert_file = config.ssl.client_cert_file + self.username = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_enabled = self.config.ssl.enabled + self.ssl_validate = self.config.ssl.validate + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file self.replset = None self._conn = None From 40aff32671cd7fa9828d58f61e787e2ed2be65fb Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:24:44 +0200 Subject: [PATCH 05/22] Use class-var bool instead of method --- mongodb_consistent_backup/Common/DB.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 8891fbb1..8f392c74 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -35,11 +35,6 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.connect() self.auth_if_required() - def do_ssl(self): - if self.ssl_enabled: - return True - return False - def connect(self): try: opts = { @@ -56,7 +51,7 @@ def connect(self): "readPreference": self.read_pref, "w": "majority" }) - if self.do_ssl(): + if self.ssl_enabled: logging.debug("Enabling SSL security on database connection") opts.update({ "ssl": True, From 4182b435fbb943a79cd13e2d884ac413b4c5368d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 01:38:15 +0200 Subject: [PATCH 06/22] Move validate to be named 'insecure' --- conf/mongodb-consistent-backup.example.conf | 2 +- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 4 ++-- mongodb_consistent_backup/Common/Config.py | 4 ++-- mongodb_consistent_backup/Common/DB.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index c2c170ef..d21c962c 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -6,7 +6,7 @@ production: #authdb: [auth database] (default: admin) #ssl: # enabled: [true|false] (default: false) - # validate: [true|false] (default: true) + # insecure: [true|false] (default: false) # ca_file: [path] # crl_file: [path] # client_cert_file: [path] diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 924db699..056d07e6 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -29,7 +29,7 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.password = self.config.password self.authdb = self.config.authdb self.ssl_enabled = self.config.ssl.enabled - self.ssl_validate = self.config.ssl.validate + self.ssl_insecure = self.config.ssl.insecure self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file @@ -146,7 +146,7 @@ def mongodump_cmd(self): mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) if self.client_cert_file: mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) - if not self.ssl_validate: + if self.ssl_insecure: mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) mongodump_cmd.extend(mongodump_flags) return mongodump_cmd diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index 633550f5..ad46d10e 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -54,8 +54,8 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) - parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, type=bool) - parser.add_argument("--ssl.validate", dest="ssl.validate", help="Validate the SSL certificate and hostname of the server (default: true)", default=True, type=bool) + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, type=bool, action="store_true") + parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=True, type=bool, action="store_true") parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL PEM file (for optional client ssl auth)", default=None, type=str) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 8f392c74..330f938c 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -24,7 +24,7 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.password = self.config.password self.authdb = self.config.authdb self.ssl_enabled = self.config.ssl.enabled - self.ssl_validate = self.config.ssl.validate + self.ssl_insecure = self.config.ssl.insecure self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file @@ -60,7 +60,7 @@ def connect(self): "ssl_certfile": self.ssl_client_cert_file, "ssl_cert_reqs": ssl.CERT_REQUIRED, }) - if not self.ssl_validate: + if self.ssl_insecure: opts.update({ "ssl_cert_reqs": ssl.CERT_NONE }) From d61f814acd5ea6c4353d510c8ec3498ae2ab6d98 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 02:09:09 +0200 Subject: [PATCH 07/22] Must pass pymongo args dict as kwargs --- mongodb_consistent_backup/Common/Config.py | 4 ++-- mongodb_consistent_backup/Common/DB.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ad46d10e..ac9c96f1 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -54,8 +54,8 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) - parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, type=bool, action="store_true") - parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=True, type=bool, action="store_true") + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL PEM file (for optional client ssl auth)", default=None, type=str) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 330f938c..8b07e64d 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -64,11 +64,10 @@ def connect(self): opts.update({ "ssl_cert_reqs": ssl.CERT_NONE }) - conn = MongoClient(opts) logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( - self.uri, self.replset, self.read_pref, self.do_ssl() + self.uri, self.replset, self.read_pref, self.ssl_enabled )) - conn = MongoClient(opts) + conn = MongoClient(**opts) if self.do_connect: conn['admin'].command({"ping": 1}) except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: From ff2092c49d96ba238de66c21c1cab3fc6690f83d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 02:19:50 +0200 Subject: [PATCH 08/22] Help output cleanup --- mongodb_consistent_backup/Common/Config.py | 4 ++-- mongodb_consistent_backup/Common/DB.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ac9c96f1..10ceeee8 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -56,9 +56,9 @@ def makeParser(self): parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, action="store_true") parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") - parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file", default=None, type=str) + parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) - parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL PEM file (for optional client ssl auth)", default=None, type=str) + parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 8b07e64d..b051c2ce 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -32,6 +32,7 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.replset = None self._conn = None self._is_master = None + self.connect() self.auth_if_required() From de7f1603e5b71ac7cf9b9689e607bb59a619af7a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 02:30:02 +0200 Subject: [PATCH 09/22] Split up options parsing and connect in DB.py --- mongodb_consistent_backup/Common/DB.py | 59 ++++++++++++++------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index b051c2ce..7f6cc55c 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -36,39 +36,42 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.connect() self.auth_if_required() - def connect(self): - try: - opts = { - "connect": self.do_connect, - "host": self.uri.hosts(), - "connectTimeoutMS": self.conn_timeout, - "serverSelectionTimeoutMS": self.conn_timeout, - "maxPoolSize": 1, - } - if self.do_replset: - self.replset = self.uri.replset - opts.update({ - "replicaSet": self.replset, - "readPreference": self.read_pref, - "w": "majority" - }) - if self.ssl_enabled: - logging.debug("Enabling SSL security on database connection") + def client_opts(self): + opts = { + "connect": self.do_connect, + "host": self.uri.hosts(), + "connectTimeoutMS": self.conn_timeout, + "serverSelectionTimeoutMS": self.conn_timeout, + "maxPoolSize": 1, + } + if self.do_replset: + self.replset = self.uri.replset + opts.update({ + "replicaSet": self.replset, + "readPreference": self.read_pref, + "w": "majority" + }) + if self.ssl_enabled: + logging.debug("Enabling SSL security on database connection") + opts.update({ + "ssl": True, + "ssl_ca_certs": self.ssl_ca_file, + "ssl_crlfile": self.ssl_crl_file, + "ssl_certfile": self.ssl_client_cert_file, + "ssl_cert_reqs": ssl.CERT_REQUIRED, + }) + if self.ssl_insecure: opts.update({ - "ssl": True, - "ssl_ca_certs": self.ssl_ca_file, - "ssl_crlfile": self.ssl_crl_file, - "ssl_certfile": self.ssl_client_cert_file, - "ssl_cert_reqs": ssl.CERT_REQUIRED, + "ssl_cert_reqs": ssl.CERT_NONE }) - if self.ssl_insecure: - opts.update({ - "ssl_cert_reqs": ssl.CERT_NONE - }) + return opts + + def connect(self): + try: logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( self.uri, self.replset, self.read_pref, self.ssl_enabled )) - conn = MongoClient(**opts) + conn = MongoClient(**self.client_opts()) if self.do_connect: conn['admin'].command({"ping": 1}) except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: From e7ec74897555e699a431b3ed2df67d8b0fe7edb4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 02:42:29 +0200 Subject: [PATCH 10/22] Only import the 2 x values required from 'ssl' --- mongodb_consistent_backup/Common/DB.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 7f6cc55c..90f8aa33 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -1,10 +1,10 @@ import logging -import ssl from bson.codec_options import CodecOptions from inspect import currentframe, getframeinfo from pymongo import DESCENDING, CursorType, MongoClient from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError +from ssl import CERT_REQUIRED, CERT_NONE from time import sleep from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error @@ -58,11 +58,11 @@ def client_opts(self): "ssl_ca_certs": self.ssl_ca_file, "ssl_crlfile": self.ssl_crl_file, "ssl_certfile": self.ssl_client_cert_file, - "ssl_cert_reqs": ssl.CERT_REQUIRED, + "ssl_cert_reqs": CERT_REQUIRED, }) if self.ssl_insecure: opts.update({ - "ssl_cert_reqs": ssl.CERT_NONE + "ssl_cert_reqs": CERT_NONE }) return opts From f734ebe9ca5ab9893d1c123bf2e4545507d744d3 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 02:45:06 +0200 Subject: [PATCH 11/22] Just set var instead of use dict .update() on a single key/val --- mongodb_consistent_backup/Common/DB.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 90f8aa33..2aca7829 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -61,9 +61,7 @@ def client_opts(self): "ssl_cert_reqs": CERT_REQUIRED, }) if self.ssl_insecure: - opts.update({ - "ssl_cert_reqs": CERT_NONE - }) + opts["ssl_cert_reqs"] = CERT_NONE return opts def connect(self): From dc50128e3510b031dee206a496024757fc706093 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:04:44 +0200 Subject: [PATCH 12/22] Make sure to support config file string 'true' or 'false' --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- mongodb_consistent_backup/Common/DB.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 056d07e6..fbc888d9 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -138,7 +138,7 @@ def mongodump_cmd(self): else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) - if self.ssl_enabled: + if self.ssl_enabled is True or self.ssl_enabled.rstrip().lower() is "true": mongodump_flags.append("--ssl") if self.ssl_ca_file: mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 2aca7829..fc919727 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -51,7 +51,7 @@ def client_opts(self): "readPreference": self.read_pref, "w": "majority" }) - if self.ssl_enabled: + if self.ssl_enabled is True or self.ssl_enabled.rstrip().lower() is "true": logging.debug("Enabling SSL security on database connection") opts.update({ "ssl": True, From 6fc43024b4df6d3fca2e3e31111f6ef156031086 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:27:11 +0200 Subject: [PATCH 13/22] make method to check to enable ssl or not, check if bool or str with isinstance() --- .../Backup/Mongodump/MongodumpThread.py | 9 ++++++++- mongodb_consistent_backup/Common/DB.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index fbc888d9..4fb2a772 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -57,6 +57,13 @@ def close(self, exit_code=None, frame=None): self._command.close() sys.exit(self.exit_code) + def do_ssl(self): + if isinstance(self.ssl_enabled, bool): + return self.ssl_enabled + elif isinstance(self.ssl_enabled, str) and self.ssl_enabled.rstrip().lower() is "true": + return True + return False + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -138,7 +145,7 @@ def mongodump_cmd(self): else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) - if self.ssl_enabled is True or self.ssl_enabled.rstrip().lower() is "true": + if self.do_ssl(): mongodump_flags.append("--ssl") if self.ssl_ca_file: mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index fc919727..1cd64855 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -36,6 +36,13 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.connect() self.auth_if_required() + def do_ssl(self): + if isinstance(self.ssl_enabled, bool): + return self.ssl_enabled + elif isinstance(self.ssl_enabled, str) and self.ssl_enabled.rstrip().lower() is "true": + return True + return False + def client_opts(self): opts = { "connect": self.do_connect, @@ -51,7 +58,7 @@ def client_opts(self): "readPreference": self.read_pref, "w": "majority" }) - if self.ssl_enabled is True or self.ssl_enabled.rstrip().lower() is "true": + if self.do_ssl(): logging.debug("Enabling SSL security on database connection") opts.update({ "ssl": True, From 00caccce9928c110440033da21d6bc296e9c1e1d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:28:01 +0200 Subject: [PATCH 14/22] make method to check to enable ssl or not, check if bool or str with isinstance() #2 --- mongodb_consistent_backup/Common/DB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 1cd64855..a2668c84 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -74,7 +74,7 @@ def client_opts(self): def connect(self): try: logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( - self.uri, self.replset, self.read_pref, self.ssl_enabled + self.uri, self.replset, self.read_pref, self.do_ssl() )) conn = MongoClient(**self.client_opts()) if self.do_connect: From 0a3eb0c296721376cac511b932d99490b22d53e1 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:33:52 +0200 Subject: [PATCH 15/22] Help output clarification --- mongodb_consistent_backup/Common/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index 10ceeee8..0bd1c29b 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -54,7 +54,7 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) - parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use an SSL secured database connection (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true") parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) From 4e9d07afde7a216fdca9672a2aa6ee52613e5f75 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:53:53 +0200 Subject: [PATCH 16/22] Make shared func for parsing config bools --- .../Backup/Mongodump/MongodumpThread.py | 13 ++++++------- mongodb_consistent_backup/Common/Config.py | 12 ++++++++++++ mongodb_consistent_backup/Common/DB.py | 12 ++++++------ mongodb_consistent_backup/Common/__init__.py | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 4fb2a772..a188c18d 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -8,7 +8,7 @@ from signal import signal, SIGINT, SIGTERM, SIG_IGN from subprocess import Popen, PIPE -from mongodb_consistent_backup.Common import is_datetime +from mongodb_consistent_backup.Common import is_datetime, parse_config_bool from mongodb_consistent_backup.Oplog import Oplog @@ -58,11 +58,10 @@ def close(self, exit_code=None, frame=None): sys.exit(self.exit_code) def do_ssl(self): - if isinstance(self.ssl_enabled, bool): - return self.ssl_enabled - elif isinstance(self.ssl_enabled, str) and self.ssl_enabled.rstrip().lower() is "true": - return True - return False + return parse_config_bool(self.ssl_enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.ssl_insecure) def parse_mongodump_line(self, line): try: @@ -153,7 +152,7 @@ def mongodump_cmd(self): mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) if self.client_cert_file: mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) - if self.ssl_insecure: + if self.do_ssl_insecure(): mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) mongodump_cmd.extend(mongodump_flags) return mongodump_cmd diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index 0bd1c29b..64950429 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -8,6 +8,18 @@ from yconf.util import NestedDict +def parse_config_bool(item): + try: + if isinstance(item, bool): + return item + elif isinstance(item, str): + if item.rstrip().lower() is "true": + return True + except: + return False + return False + + class PrintVersions(Action): def __init__(self, option_strings, dest, nargs=0, **kwargs): super(PrintVersions, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, **kwargs) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index a2668c84..601020f8 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -7,6 +7,7 @@ from ssl import CERT_REQUIRED, CERT_NONE from time import sleep +from mongodb_consistent_backup.Common import parse_config_bool from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error @@ -37,11 +38,10 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.auth_if_required() def do_ssl(self): - if isinstance(self.ssl_enabled, bool): - return self.ssl_enabled - elif isinstance(self.ssl_enabled, str) and self.ssl_enabled.rstrip().lower() is "true": - return True - return False + return parse_config_bool(self.ssl_enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.ssl_insecure) def client_opts(self): opts = { @@ -67,7 +67,7 @@ def client_opts(self): "ssl_certfile": self.ssl_client_cert_file, "ssl_cert_reqs": CERT_REQUIRED, }) - if self.ssl_insecure: + if self.do_ssl_insecure(): opts["ssl_cert_reqs"] = CERT_NONE return opts diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py index aa30af72..3104168c 100644 --- a/mongodb_consistent_backup/Common/__init__.py +++ b/mongodb_consistent_backup/Common/__init__.py @@ -1,4 +1,4 @@ -from Config import Config # NOQA +from Config import Config, parse_config_bool # NOQA from DB import DB # NOQA from LocalCommand import LocalCommand # NOQA from Lock import Lock # NOQA From 5e15ae7695ee68982cbc82c417e484d96b6ff209 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 03:56:19 +0200 Subject: [PATCH 17/22] Make shared func for parsing config bools #2 --- .../Backup/Mongodump/MongodumpThread.py | 6 ++---- mongodb_consistent_backup/Common/DB.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index a188c18d..d869d64e 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -28,8 +28,6 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.user = self.config.username self.password = self.config.password self.authdb = self.config.authdb - self.ssl_enabled = self.config.ssl.enabled - self.ssl_insecure = self.config.ssl.insecure self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file @@ -58,10 +56,10 @@ def close(self, exit_code=None, frame=None): sys.exit(self.exit_code) def do_ssl(self): - return parse_config_bool(self.ssl_enabled) + return parse_config_bool(self.config.ssl.enabled) def do_ssl_insecure(self): - return parse_config_bool(self.ssl_insecure) + return parse_config_bool(self.config.ssl.insecure) def parse_mongodump_line(self, line): try: diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 601020f8..e6dd9938 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -24,8 +24,6 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.username = self.config.username self.password = self.config.password self.authdb = self.config.authdb - self.ssl_enabled = self.config.ssl.enabled - self.ssl_insecure = self.config.ssl.insecure self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file @@ -38,10 +36,10 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.auth_if_required() def do_ssl(self): - return parse_config_bool(self.ssl_enabled) + return parse_config_bool(self.config.ssl.enabled) def do_ssl_insecure(self): - return parse_config_bool(self.ssl_insecure) + return parse_config_bool(self.config.ssl.insecure) def client_opts(self): opts = { From 60bbb730441f09f43aa6217ccb61e29e3375e202 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 04:53:48 +0200 Subject: [PATCH 18/22] Make shared func for parsing config bools #3 --- mongodb_consistent_backup/Common/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index 64950429..cc3d5cb2 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -15,9 +15,9 @@ def parse_config_bool(item): elif isinstance(item, str): if item.rstrip().lower() is "true": return True + return False except: return False - return False class PrintVersions(Action): From 023afb5fe40b54a3343b7274ce10954da6d6b26c Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 05:18:04 +0200 Subject: [PATCH 19/22] Improved ssl debug logs --- mongodb_consistent_backup/Common/DB.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index e6dd9938..d7708b3f 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -57,7 +57,12 @@ def client_opts(self): "w": "majority" }) if self.do_ssl(): - logging.debug("Enabling SSL security on database connection") + logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( + self.ssl_ca_file, + self.ssl_client_cert_file, + self.ssl_crl_file, + self.do_ssl_insecure() + )) opts.update({ "ssl": True, "ssl_ca_certs": self.ssl_ca_file, @@ -72,7 +77,10 @@ def client_opts(self): def connect(self): try: logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( - self.uri, self.replset, self.read_pref, self.do_ssl() + self.uri, + self.replset, + self.read_pref, + self.do_ssl(), )) conn = MongoClient(**self.client_opts()) if self.do_connect: From 4b2ba2ca64529dad5c3ebb3475854cbb48843a5c Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 6 Sep 2017 05:22:29 +0200 Subject: [PATCH 20/22] Improve CA help notes --- mongodb_consistent_backup/Common/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index cc3d5cb2..fc04991b 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -68,7 +68,7 @@ def makeParser(self): parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true") parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") - parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format", default=None, type=str) + parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)", default=None, type=str) parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) From d82991df4dd39f2063a19795b3ddfbfd1425f716 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 13 Sep 2017 02:14:44 +0200 Subject: [PATCH 21/22] check SSL is supported (2.6.0+) --- .../Backup/Mongodump/MongodumpThread.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index d869d64e..f1ec043e 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -61,6 +61,12 @@ def do_ssl(self): def do_ssl_insecure(self): return parse_config_bool(self.config.ssl.insecure) + def is_version_gte(self, compare): + if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK): + if tuple(compare.split(".")) <= tuple(self.version.split(".")): + return True + return False + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -127,31 +133,40 @@ def mongodump_cmd(self): mongodump_flags = ["--host", mongodump_uri.host, "--port", str(mongodump_uri.port), "--oplog", "--out", "%s/dump" % self.backup_dir] if self.threads > 0: mongodump_flags.extend(["--numParallelCollections=" + str(self.threads)]) + if self.dump_gzip: mongodump_flags.extend(["--gzip"]) - if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")): + + if self.is_version_gte("3.4.0"): mongodump_flags.extend(["--readPreference=secondary"]) + if self.authdb and self.authdb != "admin": logging.debug("Using database %s for authentication" % self.authdb) mongodump_flags.extend(["--authenticationDatabase", self.authdb]) if self.user and self.password: # >= 3.0.2 supports password input via stdin to mask from ps - if tuple(self.version.split(".")) >= tuple("3.0.2".split(".")): + if self.is_version_gte("3.0.2"): mongodump_flags.extend(["-u", self.user, "-p", '""']) self.do_stdin_passwd = True else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) + if self.do_ssl(): - mongodump_flags.append("--ssl") - if self.ssl_ca_file: - mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) - if self.ssl_crl_file: - mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) - if self.client_cert_file: - mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) - if self.do_ssl_insecure(): - mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) + if self.is_version_gte("2.6.0"): + mongodump_flags.append("--ssl") + if self.ssl_ca_file: + mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + if self.ssl_crl_file: + mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + if self.client_cert_file: + mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + if self.do_ssl_insecure(): + mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) + else: + logging.fatal("Mongodump must be >= 2.6.0 to enable SSL encryption!") + sys.exit(1) + mongodump_cmd.extend(mongodump_flags) return mongodump_cmd From b6172a8e25e93005125503e44ea080450ec76ed4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 14 Sep 2017 19:24:35 +0200 Subject: [PATCH 22/22] README update --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f87a7a1e..cefd49c7 100644 --- a/README.rst +++ b/README.rst @@ -34,6 +34,7 @@ Features - `Nagios NSCA `__ push notification support (*optional*) - Modular backup, archiving, upload and notification components +- Support for MongoDB Authentication and SSL database connections - Multi-threaded, single executable - Auto-scales to number of available CPUs by default @@ -220,7 +221,6 @@ Roadmap - Backup retention/rotation *(eg: delete old backups)* - Support more notification methods *(Prometheus, PagerDuty, etc)* - Support more upload methods *(Rsync, etc)* -- Support SSL MongoDB connections - Documentation for running under Docker with persistent volumes - Python unit tests