From fe862468cd58957e55d0269eacff30522876466b Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 7 Sep 2017 07:25:42 +0200 Subject: [PATCH 01/22] Support read_preference tags with Common/DB.py --- conf/mongodb-consistent-backup.example.conf | 15 +++- .../Backup/Mongodump/MongodumpThread.py | 29 ++++-- mongodb_consistent_backup/Common/Config.py | 17 ++++ mongodb_consistent_backup/Common/DB.py | 88 ++++++++++++++----- mongodb_consistent_backup/Common/__init__.py | 2 +- .../Replication/__init__.py | 4 +- 6 files changed, 120 insertions(+), 35 deletions(-) diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index 0e0c8321..dc51bde1 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -4,6 +4,12 @@ production: #username: [auth username] (default: none) #password: [auth password] (default: none) #authdb: [auth database] (default: admin) + #ssl: + # enabled: [true|false] (default: false) + # insecure: [true|false] (default: false) + # ca_file: [path] + # crl_file: [path] + # client_cert_file: [path] log_dir: /var/log/mongodb-consistent-backup backup: method: mongodump @@ -14,10 +20,11 @@ production: # compression: [auto|none|gzip] (default: auto - enable gzip if supported) # threads: [1-16] (default: auto-generated - shards/cpu) #replication: - # max_lag_secs: [1+] (default: 10) - # min_priority: [0-999] (default: 0) - # max_priority: [2-1000] (default: 1000) - # hidden_only: [true|false] (default: false) + # max_lag_secs: [1+] (default: 10) + # min_priority: [0-999] (default: 0) + # max_priority: [2-1000] (default: 1000) + # hidden_only: [true|false] (default: false) + # read_pref_tags: [key:value,key:value,...] (default: none) #sharding: # balancer: # wait_secs: [1+] (default: 300) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 2ac39eed..d869d64e 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -8,7 +8,7 @@ from signal import signal, SIGINT, SIGTERM, SIG_IGN from subprocess import Popen, PIPE -from mongodb_consistent_backup.Common import is_datetime +from mongodb_consistent_backup.Common import is_datetime, parse_config_bool from mongodb_consistent_backup.Oplog import Oplog @@ -25,10 +25,13 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.threads = threads self.dump_gzip = dump_gzip - self.user = self.config.username - self.password = self.config.password - self.authdb = self.config.authdb - self.binary = self.config.backup.mongodump.binary + self.user = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.binary = self.config.backup.mongodump.binary self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset) self.exit_code = 1 @@ -52,6 +55,12 @@ def close(self, exit_code=None, frame=None): self._command.close() sys.exit(self.exit_code) + def do_ssl(self): + return parse_config_bool(self.config.ssl.enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.config.ssl.insecure) + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -133,6 +142,16 @@ def mongodump_cmd(self): else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) + if self.do_ssl(): + mongodump_flags.append("--ssl") + if self.ssl_ca_file: + mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + if self.ssl_crl_file: + mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + if self.client_cert_file: + mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + if self.do_ssl_insecure(): + mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) mongodump_cmd.extend(mongodump_flags) return mongodump_cmd diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ad52b25a..fc04991b 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -8,6 +8,18 @@ from yconf.util import NestedDict +def parse_config_bool(item): + try: + if isinstance(item, bool): + return item + elif isinstance(item, str): + if item.rstrip().lower() is "true": + return True + return False + except: + return False + + class PrintVersions(Action): def __init__(self, option_strings, dest, nargs=0, **kwargs): super(PrintVersions, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, **kwargs) @@ -54,6 +66,11 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)", default=None, type=str) + parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) + parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index eec89775..0c208bff 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -4,46 +4,88 @@ from inspect import currentframe, getframeinfo from pymongo import DESCENDING, CursorType, MongoClient from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError +from ssl import CERT_REQUIRED, CERT_NONE from time import sleep +from mongodb_consistent_backup.Common import parse_config_bool from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error class DB: def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5): - self.uri = uri - self.username = config.username - self.password = config.password - self.authdb = config.authdb - self.do_replset = do_replset - self.read_pref = read_pref - self.do_connect = do_connect - self.conn_timeout = conn_timeout - self.retries = retries + self.uri = uri + self.config = config + self.do_replset = do_replset + self.read_pref = read_pref + self.do_connect = do_connect + self.conn_timeout = conn_timeout + self.retries = retries + + self.username = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.read_pref_tags = self.config.replication.read_pref_tags.replace(" ", "") self.replset = None self._conn = None self._is_master = None + self.connect() self.auth_if_required() + def do_ssl(self): + return parse_config_bool(self.config.ssl.enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.config.ssl.insecure) + + def client_opts(self): + opts = { + "connect": self.do_connect, + "host": self.uri.hosts(), + "connectTimeoutMS": self.conn_timeout, + "serverSelectionTimeoutMS": self.conn_timeout, + "maxPoolSize": 1, + } + if self.do_replset: + self.replset = self.uri.replset + opts.update({ + "replicaSet": self.replset, + "readPreference": self.read_pref, + "readPreferenceTags": self.read_pref_tags, + "w": "majority" + }) + if self.do_ssl(): + logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( + self.ssl_ca_file, + self.ssl_client_cert_file, + self.ssl_crl_file, + self.do_ssl_insecure() + )) + opts.update({ + "ssl": True, + "ssl_ca_certs": self.ssl_ca_file, + "ssl_crlfile": self.ssl_crl_file, + "ssl_certfile": self.ssl_client_cert_file, + "ssl_cert_reqs": CERT_REQUIRED, + }) + if self.do_ssl_insecure(): + opts["ssl_cert_reqs"] = CERT_NONE + return opts + def connect(self): try: - if self.do_replset: - self.replset = self.uri.replset - logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s)" % ( - self.uri, self.replset, self.read_pref + logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, readPreferenceTags=%s, ssl=%s)" % ( + self.uri, + self.replset, + self.read_pref, + self.read_pref_tags, + self.do_ssl(), )) - conn = MongoClient( - connect=self.do_connect, - host=self.uri.hosts(), - replicaSet=self.replset, - readPreference=self.read_pref, - connectTimeoutMS=self.conn_timeout, - serverSelectionTimeoutMS=self.conn_timeout, - maxPoolSize=1, - w="majority" - ) + conn = MongoClient(**self.client_opts()) if self.do_connect: conn['admin'].command({"ping": 1}) except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py index aa30af72..3104168c 100644 --- a/mongodb_consistent_backup/Common/__init__.py +++ b/mongodb_consistent_backup/Common/__init__.py @@ -1,4 +1,4 @@ -from Config import Config # NOQA +from Config import Config, parse_config_bool # NOQA from DB import DB # NOQA from LocalCommand import LocalCommand # NOQA from Lock import Lock # NOQA diff --git a/mongodb_consistent_backup/Replication/__init__.py b/mongodb_consistent_backup/Replication/__init__.py index e1559a31..6da8aa82 100644 --- a/mongodb_consistent_backup/Replication/__init__.py +++ b/mongodb_consistent_backup/Replication/__init__.py @@ -7,6 +7,6 @@ def config(parser): parser.add_argument("--replication.min_priority", dest="replication.min_priority", help="Min priority of secondary members for backup (default: 0)", default=0, type=int) parser.add_argument("--replication.max_priority", dest="replication.max_priority", help="Max priority of secondary members for backup (default: 1000)", default=1000, type=int) parser.add_argument("--replication.hidden_only", dest="replication.hidden_only", help="Only use hidden secondary members for backup (default: false)", default=False, action="store_true") - # todo: add tag-specific backup option - # parser.add_argument("-replication.use_tag", dest="replication.use_tag", help="Only use secondary members with tag for backup", type=str) + parser.add_argument("--replication.read_pref_tags", dest="replication.read_pref_tags", default=None, type=str, + help="Only use members that match replication tags in comma-separated key:value format (default: none)") return parser From 7c1448642446718f58dbcfa42264675946d9b33e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Sep 2017 22:38:00 +0200 Subject: [PATCH 02/22] only cleanup string if not None --- mongodb_consistent_backup/Common/DB.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 0c208bff..d70607fa 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -27,12 +27,15 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file - self.read_pref_tags = self.config.replication.read_pref_tags.replace(" ", "") + self.read_pref_tags = self.config.replication.read_pref_tags self.replset = None self._conn = None self._is_master = None + if self.read_pref_tags: + self.read_pref_tags = self.read_pref_tags.replace(" ", "") + self.connect() self.auth_if_required() From 3b894c76096d2d3e7dbc539f1a57eb89a99cf3bc Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Sep 2017 23:21:51 +0200 Subject: [PATCH 03/22] Only apply RP tag if not None, pymongo gives 'None not a valid value for readPreferenceTags' --- mongodb_consistent_backup/Common/DB.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index d70607fa..60f8083e 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -56,11 +56,12 @@ def client_opts(self): if self.do_replset: self.replset = self.uri.replset opts.update({ - "replicaSet": self.replset, - "readPreference": self.read_pref, - "readPreferenceTags": self.read_pref_tags, - "w": "majority" + "replicaSet": self.replset, + "readPreference": self.read_pref, + "w": "majority" }) + if self.read_pref_tags: + opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( self.ssl_ca_file, From 21e842bb47ab3cd450edf74fbdc4ec42e0a8575e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Sep 2017 23:25:06 +0200 Subject: [PATCH 04/22] Only apply RP tag if not None, pymongo gives 'None not a valid value for readPreferenceTags' #2 --- mongodb_consistent_backup/Common/DB.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 60f8083e..3c08ec4f 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -33,9 +33,6 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', self._conn = None self._is_master = None - if self.read_pref_tags: - self.read_pref_tags = self.read_pref_tags.replace(" ", "") - self.connect() self.auth_if_required() @@ -61,6 +58,7 @@ def client_opts(self): "w": "majority" }) if self.read_pref_tags: + self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( From 291b78637df95aed181a6fb2b41cbadff53bd0a3 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 06:29:09 +0200 Subject: [PATCH 05/22] Parse mongodump json RP tag string, add function for checking mongodump version, cleaned up some of the mongodump_cmd builder-func --- .../Backup/Mongodump/Mongodump.py | 4 +- .../Backup/Mongodump/MongodumpThread.py | 78 +++++++++++++++---- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 92b2a2ce..e34bd8de 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -17,11 +17,11 @@ class Mongodump(Task): def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, backup_stop=None, sharding=None): super(Mongodump, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir) - self.compression_method = self.config.backup.mongodump.compression - self.binary = self.config.backup.mongodump.binary self.user = self.config.username self.password = self.config.password self.authdb = self.config.authdb + self.compression_method = self.config.backup.mongodump.compression + self.binary = self.config.backup.mongodump.binary self.replsets = replsets self.backup_stop = backup_stop self.sharding = sharding diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index d869d64e..bcf581c1 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -1,3 +1,4 @@ +import json import os import logging import sys @@ -31,6 +32,7 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.ssl_ca_file = self.config.ssl.ca_file self.ssl_crl_file = self.config.ssl.crl_file self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.read_pref_tags = self.config.replication.read_pref_tags self.binary = self.config.backup.mongodump.binary self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset) @@ -61,6 +63,22 @@ def do_ssl(self): def do_ssl_insecure(self): return parse_config_bool(self.config.ssl.insecure) + def is_version_gte(self, compare): + if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK): + if tuple(compare.split(".")) <= tuple(self.version.split(".")): + return True + return False + + def parse_read_pref(self, mode="secondary"): + rp = {"mode": mode} + if self.read_pref_tags: + rp["tags"] = {} + for pair in self.read_pref_tags.replace(" ", "").split(","): + if ":" in pair: + key, value = pair.split(":") + rp["tags"][key] = str(value) + return json.dumps(rp) + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -125,33 +143,59 @@ def mongodump_cmd(self): mongodump_uri = self.uri.get() mongodump_cmd = [self.binary] mongodump_flags = ["--host", mongodump_uri.host, "--port", str(mongodump_uri.port), "--oplog", "--out", "%s/dump" % self.backup_dir] + + # --numParallelCollections if self.threads > 0: - mongodump_flags.extend(["--numParallelCollections=" + str(self.threads)]) + mongodump_flags.append("--numParallelCollections=%s" % str(self.threads)) + + # --gzip if self.dump_gzip: - mongodump_flags.extend(["--gzip"]) - if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")): - mongodump_flags.extend(["--readPreference=secondary"]) + mongodump_flags.append("--gzip") + + # --readPreference + if self.is_version_gte("3.2.0"): + read_pref = self.parse_read_pref() + if read_pref: + mongodump_flags.append("--readPreference='%s'" % read_pref) + elif self.read_pref_tags: + logging.fatal("Mongodump must be >= 3.4.0 to set read preference!") + sys.exit(1) + + # --username/--password/--authdb if self.authdb and self.authdb != "admin": logging.debug("Using database %s for authentication" % self.authdb) - mongodump_flags.extend(["--authenticationDatabase", self.authdb]) + mongodump_flags.append("--authenticationDatabase=%s" % self.authdb) if self.user and self.password: # >= 3.0.2 supports password input via stdin to mask from ps - if tuple(self.version.split(".")) >= tuple("3.0.2".split(".")): - mongodump_flags.extend(["-u", self.user, "-p", '""']) + if self.is_version_gte("3.0.2"): + mongodump_flags.extend([ + "--username=%s" % self.user, + "--password=\"\"" + ]) self.do_stdin_passwd = True else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") - mongodump_flags.extend(["-u", self.user, "-p", self.password]) + mongodump_flags.extend([ + "--username=%s" % self.user, + "--password=%s" % self.password + ]) + + # --ssl if self.do_ssl(): - mongodump_flags.append("--ssl") - if self.ssl_ca_file: - mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) - if self.ssl_crl_file: - mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) - if self.client_cert_file: - mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) - if self.do_ssl_insecure(): - mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) + if self.is_version_gte("2.6.0"): + mongodump_flags.append("--ssl") + if self.ssl_ca_file: + mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + if self.ssl_crl_file: + mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + if self.client_cert_file: + mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + if self.do_ssl_insecure(): + mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) + else: + logging.fatal("Mongodump must be >= 2.6.0 to enable SSL!") + sys.exit(1) + mongodump_cmd.extend(mongodump_flags) return mongodump_cmd From b713d7d85ee7bf80d0051ef092863a870dceb195 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 07:07:55 +0200 Subject: [PATCH 06/22] Fix quotes --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index bcf581c1..a64fae10 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -156,7 +156,7 @@ def mongodump_cmd(self): if self.is_version_gte("3.2.0"): read_pref = self.parse_read_pref() if read_pref: - mongodump_flags.append("--readPreference='%s'" % read_pref) + mongodump_flags.append("--readPreference=%s" % read_pref) elif self.read_pref_tags: logging.fatal("Mongodump must be >= 3.4.0 to set read preference!") sys.exit(1) From 20845cbcf56dda980c83bb72072baed54671a2ef Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 07:09:46 +0200 Subject: [PATCH 07/22] log typo --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index a64fae10..7c05077b 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -158,7 +158,7 @@ def mongodump_cmd(self): if read_pref: mongodump_flags.append("--readPreference=%s" % read_pref) elif self.read_pref_tags: - logging.fatal("Mongodump must be >= 3.4.0 to set read preference!") + logging.fatal("Mongodump must be >= 3.2.0 to set read preference!") sys.exit(1) # --username/--password/--authdb From 6ce36a746436abf8a8b9efc16dda6118014c4277 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 07:15:14 +0200 Subject: [PATCH 08/22] Use long-flag=value for all mongodump flags --- .../Backup/Mongodump/MongodumpThread.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 7c05077b..564936ea 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -142,7 +142,12 @@ def wait(self): def mongodump_cmd(self): mongodump_uri = self.uri.get() mongodump_cmd = [self.binary] - mongodump_flags = ["--host", mongodump_uri.host, "--port", str(mongodump_uri.port), "--oplog", "--out", "%s/dump" % self.backup_dir] + mongodump_flags = [ + "--host=%s" % mongodump_uri.host, + "--port=%s" % str(mongodump_uri.port), + "--oplog", + "--out=%s/dump" % self.backup_dir + ] # --numParallelCollections if self.threads > 0: @@ -185,15 +190,15 @@ def mongodump_cmd(self): if self.is_version_gte("2.6.0"): mongodump_flags.append("--ssl") if self.ssl_ca_file: - mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + mongodump_flags.append("--sslCAFile=%s" % self.ssl_ca_file) if self.ssl_crl_file: - mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + mongodump_flags.append("--sslCRLFile=%s" % self.ssl_crl_file) if self.client_cert_file: - mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + mongodump_flags.append("--sslPEMKeyFile=%s" % self.ssl_cert_file) if self.do_ssl_insecure(): mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) else: - logging.fatal("Mongodump must be >= 2.6.0 to enable SSL!") + logging.fatal("Mongodump must be >= 2.6.0 to enable SSL encryption!") sys.exit(1) mongodump_cmd.extend(mongodump_flags) From 492d39965e14e366ed4f1b61091840a32afc62f7 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 07:32:30 +0200 Subject: [PATCH 09/22] Only set read pref tags if boolean is True, make only the tailing and dumping use them. --- mongodb_consistent_backup/Common/DB.py | 6 ++++-- mongodb_consistent_backup/Oplog/Tailer/TailThread.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 3c08ec4f..cd3f46cb 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -12,11 +12,13 @@ class DB: - def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5): + def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_rp_tags=False, + do_connect=True, conn_timeout=5000, retries=5): self.uri = uri self.config = config self.do_replset = do_replset self.read_pref = read_pref + self.do_rp_tags = do_rp_tags self.do_connect = do_connect self.conn_timeout = conn_timeout self.retries = retries @@ -57,7 +59,7 @@ def client_opts(self): "readPreference": self.read_pref, "w": "majority" }) - if self.read_pref_tags: + if self.do_rp_tags and self.read_pref_tags: self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): diff --git a/mongodb_consistent_backup/Oplog/Tailer/TailThread.py b/mongodb_consistent_backup/Oplog/Tailer/TailThread.py index d97deb99..3a090bdf 100644 --- a/mongodb_consistent_backup/Oplog/Tailer/TailThread.py +++ b/mongodb_consistent_backup/Oplog/Tailer/TailThread.py @@ -88,7 +88,7 @@ def status(self): def connect(self): if not self.db: - self.db = DB(self.uri, self.config, True, 'secondary') + self.db = DB(self.uri, self.config, True, 'secondary', True) return self.db.connection() def run(self): From edb2ad7b1335975c475cda6b71ea5783bcc872e9 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 07:50:44 +0200 Subject: [PATCH 10/22] Fix some Oplog/TailThread.py exception problems found in testing --- mongodb_consistent_backup/Common/DB.py | 3 ++- .../Oplog/Tailer/TailThread.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index cd3f46cb..31b9f61f 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -62,6 +62,7 @@ def client_opts(self): if self.do_rp_tags and self.read_pref_tags: self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags + logging.debug("Using read preference tags: %s" % self.read_pref_tags) if self.do_ssl(): logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( self.ssl_ca_file, @@ -86,7 +87,7 @@ def connect(self): self.uri, self.replset, self.read_pref, - self.read_pref_tags, + self.do_rp_tags, self.do_ssl(), )) conn = MongoClient(**self.client_opts()) diff --git a/mongodb_consistent_backup/Oplog/Tailer/TailThread.py b/mongodb_consistent_backup/Oplog/Tailer/TailThread.py index 3a090bdf..74c4393c 100644 --- a/mongodb_consistent_backup/Oplog/Tailer/TailThread.py +++ b/mongodb_consistent_backup/Oplog/Tailer/TailThread.py @@ -3,7 +3,7 @@ # noinspection PyPackageRequirements from multiprocessing import Process -from pymongo.errors import AutoReconnect, ConnectionFailure, CursorNotFound, ExceededMaxWaiters, ExecutionTimeout, NetworkTimeout, NotMasterError +from pymongo.errors import AutoReconnect, ConnectionFailure, CursorNotFound, ExceededMaxWaiters, ExecutionTimeout, NetworkTimeout, NotMasterError, ServerSelectionTimeoutError from signal import signal, SIGINT, SIGTERM, SIG_IGN from time import sleep, time @@ -144,21 +144,25 @@ def run(self): continue sleep(1) finally: - self._cursor.close() + if self._cursor: + logging.debug("Stopping oplog cursor on %s" % self.uri) + self._cursor.close() except OperationError, e: logging.error("Tailer %s encountered error: %s" % (self.uri, e)) self.exit_code = 1 self.backup_stop.set() raise OperationError(e) + except ServerSelectionTimeoutError, e: + logging.error("Tailer %s could not connect: %s" % (self.uri, e)) + self.exit_code = 1 + self.backup_stop.set() + raise OperationError(e) except Exception, e: logging.error("Tailer %s encountered an unexpected error: %s" % (self.uri, e)) self.exit_code = 1 self.backup_stop.set() raise e finally: - if self._cursor: - logging.debug("Stopping oplog cursor on %s" % self.uri) - self._cursor.close() oplog.flush() oplog.close() self.stopped = True From a5e7c4c1ebbec40f109a77e648086133b75d4bd3 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 08:10:36 +0200 Subject: [PATCH 11/22] Move read_pref string parser to Common/DB.py --- .../Backup/Mongodump/MongodumpThread.py | 16 +++------------- mongodb_consistent_backup/Common/DB.py | 13 ++++++++++++- mongodb_consistent_backup/Common/__init__.py | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 564936ea..f2d7d023 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -9,7 +9,7 @@ from signal import signal, SIGINT, SIGTERM, SIG_IGN from subprocess import Popen, PIPE -from mongodb_consistent_backup.Common import is_datetime, parse_config_bool +from mongodb_consistent_backup.Common import is_datetime, parse_config_bool, parse_read_pref from mongodb_consistent_backup.Oplog import Oplog @@ -69,16 +69,6 @@ def is_version_gte(self, compare): return True return False - def parse_read_pref(self, mode="secondary"): - rp = {"mode": mode} - if self.read_pref_tags: - rp["tags"] = {} - for pair in self.read_pref_tags.replace(" ", "").split(","): - if ":" in pair: - key, value = pair.split(":") - rp["tags"][key] = str(value) - return json.dumps(rp) - def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -159,9 +149,9 @@ def mongodump_cmd(self): # --readPreference if self.is_version_gte("3.2.0"): - read_pref = self.parse_read_pref() + read_pref = parse_read_pref("secondary", self.read_pref_tags) if read_pref: - mongodump_flags.append("--readPreference=%s" % read_pref) + mongodump_flags.append("--readPreference=%s" % json.dumps(read_pref)) elif self.read_pref_tags: logging.fatal("Mongodump must be >= 3.2.0 to set read preference!") sys.exit(1) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 31b9f61f..e3214309 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -11,6 +11,17 @@ from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error +def parse_read_pref(mode="secondary", tags_str=None): + rp = {"mode": mode} + if tags_str: + rp["tags"] = {} + for pair in tags_str.replace(" ", "").split(","): + if ":" in pair: + key, value = pair.split(":") + rp["tags"][key] = str(value) + return rp + + class DB: def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_rp_tags=False, do_connect=True, conn_timeout=5000, retries=5): @@ -60,9 +71,9 @@ def client_opts(self): "w": "majority" }) if self.do_rp_tags and self.read_pref_tags: + logging.debug("Using read preference tags: '%s'" % self.read_pref_tags) self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags - logging.debug("Using read preference tags: %s" % self.read_pref_tags) if self.do_ssl(): logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( self.ssl_ca_file, diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py index 3104168c..4009a45d 100644 --- a/mongodb_consistent_backup/Common/__init__.py +++ b/mongodb_consistent_backup/Common/__init__.py @@ -1,5 +1,5 @@ from Config import Config, parse_config_bool # NOQA -from DB import DB # NOQA +from DB import DB, parse_read_pref # NOQA from LocalCommand import LocalCommand # NOQA from Lock import Lock # NOQA from MongoUri import MongoUri # NOQA From 59798861981ee01defcead69cacbd49956c4df42 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 08:58:17 +0200 Subject: [PATCH 12/22] Only put the tag-parsing part of the parse_read_pref logic in Common/DB.py, move some specific stuff back to Backup/Mongodump/MongodumpThread.py. Added tag-awareness to Replication/Replset.py so we can score/find members properly --- .../Backup/Mongodump/MongodumpThread.py | 10 ++++- mongodb_consistent_backup/Common/DB.py | 16 ++++---- mongodb_consistent_backup/Common/__init__.py | 2 +- .../Replication/Replset.py | 37 ++++++++++++++----- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index f2d7d023..6b90bb38 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -9,7 +9,7 @@ from signal import signal, SIGINT, SIGTERM, SIG_IGN from subprocess import Popen, PIPE -from mongodb_consistent_backup.Common import is_datetime, parse_config_bool, parse_read_pref +from mongodb_consistent_backup.Common import is_datetime, parse_config_bool, parse_read_pref_tags from mongodb_consistent_backup.Oplog import Oplog @@ -69,6 +69,12 @@ def is_version_gte(self, compare): return True return False + def parse_read_pref(self, mode="secondary"): + rp = {"mode": mode} + if self.read_pref_tags: + rp["tags"] = parse_read_pref_tags(self.read_pref_tags) + return json.dumps(rp) + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -149,7 +155,7 @@ def mongodump_cmd(self): # --readPreference if self.is_version_gte("3.2.0"): - read_pref = parse_read_pref("secondary", self.read_pref_tags) + read_pref = self.parse_read_pref() if read_pref: mongodump_flags.append("--readPreference=%s" % json.dumps(read_pref)) elif self.read_pref_tags: diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index e3214309..b697be00 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -11,15 +11,13 @@ from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error -def parse_read_pref(mode="secondary", tags_str=None): - rp = {"mode": mode} - if tags_str: - rp["tags"] = {} - for pair in tags_str.replace(" ", "").split(","): - if ":" in pair: - key, value = pair.split(":") - rp["tags"][key] = str(value) - return rp +def parse_read_pref_tags(tags_str): + tags = {} + for pair in tags_str.replace(" ", "").split(","): + if ":" in pair: + key, value = pair.split(":") + tags[key] = str(value) + return tags class DB: diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py index 4009a45d..246a27f7 100644 --- a/mongodb_consistent_backup/Common/__init__.py +++ b/mongodb_consistent_backup/Common/__init__.py @@ -1,5 +1,5 @@ from Config import Config, parse_config_bool # NOQA -from DB import DB, parse_read_pref # NOQA +from DB import DB, parse_read_pref_tags # NOQA from LocalCommand import LocalCommand # NOQA from Lock import Lock # NOQA from MongoUri import MongoUri # NOQA diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py index 72e1689b..fce9c357 100644 --- a/mongodb_consistent_backup/Replication/Replset.py +++ b/mongodb_consistent_backup/Replication/Replset.py @@ -4,18 +4,19 @@ from math import ceil from time import mktime -from mongodb_consistent_backup.Common import DB, MongoUri +from mongodb_consistent_backup.Common import DB, MongoUri, parse_read_pref_tags from mongodb_consistent_backup.Errors import Error, OperationError class Replset: def __init__(self, config, db): - self.config = config - self.db = db - self.max_lag_secs = self.config.replication.max_lag_secs - self.min_priority = self.config.replication.min_priority - self.max_priority = self.config.replication.max_priority - self.hidden_only = self.config.replication.hidden_only + self.config = config + self.db = db + self.read_pref_tags = self.config.replication.read_pref_tags + self.max_lag_secs = self.config.replication.max_lag_secs + self.min_priority = self.config.replication.min_priority + self.max_priority = self.config.replication.max_priority + self.hidden_only = self.config.replication.hidden_only self.hidden_weight = 0.20 self.pri0_weight = 0.10 @@ -139,6 +140,18 @@ def is_member_electable(self, member): return True return False + def has_read_pref_tags(self, member_config): + if "tags" not in member_config: + raise OperationError("Member config has no 'tags' field!") + tags = parse_read_pref_tags(self.read_pref_tags) + member_tags = member_config["tags"] + for key in tags: + if key not in member_tags: + return False + if member_tags[key] != tags[key]: + return False + return True + def find_primary(self, force=False, quiet=False): if force or not self.primary: rs_status = self.get_rs_status(force, quiet) @@ -170,8 +183,8 @@ def find_secondary(self, force=False, quiet=False): self.get_rs_config(force, quiet) self.get_mongo_config(force, quiet) - quorum = self.get_rs_quorum() - rs_name = rs_status['set'] + quorum = self.get_rs_quorum() + rs_name = rs_status['set'] if self.secondary and not force: return self.secondary @@ -193,6 +206,12 @@ def find_secondary(self, force=False, quiet=False): score = self.max_lag_secs * 10 score_scale = 100 / score priority = 0 + + if self.read_pref_tags: + if not self.has_read_pref_tags(member_config): + logging.info("Found SECONDARY %s without required read preference tags, skipping" % member_uri) + continue + if 'hidden' in member_config and member_config['hidden']: score += (score * self.hidden_weight) log_data['hidden'] = True From 858866926d7496d0e6449563a555471ad4881889 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:05:43 +0200 Subject: [PATCH 13/22] Don't json.dumps() twice --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 6b90bb38..45735f92 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -157,7 +157,7 @@ def mongodump_cmd(self): if self.is_version_gte("3.2.0"): read_pref = self.parse_read_pref() if read_pref: - mongodump_flags.append("--readPreference=%s" % json.dumps(read_pref)) + mongodump_flags.append("--readPreference=%s" % read_pref) elif self.read_pref_tags: logging.fatal("Mongodump must be >= 3.2.0 to set read preference!") sys.exit(1) From 991e479eb9349b0d7dc834e5419008d94bc14dc1 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:22:21 +0200 Subject: [PATCH 14/22] Improved logging of RP tags --- mongodb_consistent_backup/Common/DB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index b697be00..438ee034 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -69,7 +69,7 @@ def client_opts(self): "w": "majority" }) if self.do_rp_tags and self.read_pref_tags: - logging.debug("Using read preference tags: '%s'" % self.read_pref_tags) + logging.debug("Using read preference tags: '%s'" % parse_read_pref_tags(self.read_pref_tags)) self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): From 3e9f7cabcda36a3b8c63774f681ea6ab809fa4f4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:49:14 +0200 Subject: [PATCH 15/22] Improved logging --- .../Backup/Mongodump/MongodumpThread.py | 2 +- mongodb_consistent_backup/Replication/Replset.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 45735f92..f612023b 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -212,7 +212,7 @@ def run(self): if os.path.isdir(self.dump_dir): rmtree(self.dump_dir) os.makedirs(self.dump_dir) - logging.debug("Running mongodump cmd: %s" % mongodump_cmd) + logging.debug("Running mongodump cmd: %s" % " ".join(mongodump_cmd)) self._process = Popen(mongodump_cmd, stdin=PIPE, stderr=PIPE) self.wait() self.exit_code = self._process.returncode diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py index fce9c357..4c67dd2c 100644 --- a/mongodb_consistent_backup/Replication/Replset.py +++ b/mongodb_consistent_backup/Replication/Replset.py @@ -209,7 +209,10 @@ def find_secondary(self, force=False, quiet=False): if self.read_pref_tags: if not self.has_read_pref_tags(member_config): - logging.info("Found SECONDARY %s without required read preference tags, skipping" % member_uri) + logging.info("Found SECONDARY %s without read preference tags: %s, skipping" % ( + member_uri, + parse_read_pref_tags(self.read_pref_tags) + )) continue if 'hidden' in member_config and member_config['hidden']: From b91cc998f9c7994ad1ed723773cd44fdea70ee08 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:55:43 +0200 Subject: [PATCH 16/22] Improved logging #2 --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index f612023b..36b9139e 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -157,7 +157,7 @@ def mongodump_cmd(self): if self.is_version_gte("3.2.0"): read_pref = self.parse_read_pref() if read_pref: - mongodump_flags.append("--readPreference=%s" % read_pref) + mongodump_flags.append("--readPreference='%s'" % read_pref) elif self.read_pref_tags: logging.fatal("Mongodump must be >= 3.2.0 to set read preference!") sys.exit(1) From 86eec18ce4bd1f0266d83f64b44ccd0577af2761 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:56:55 +0200 Subject: [PATCH 17/22] Improved logging #3 --- mongodb_consistent_backup/Common/DB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 438ee034..946cfe2c 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -69,7 +69,7 @@ def client_opts(self): "w": "majority" }) if self.do_rp_tags and self.read_pref_tags: - logging.debug("Using read preference tags: '%s'" % parse_read_pref_tags(self.read_pref_tags)) + logging.debug("Using read preference tags: %s" % parse_read_pref_tags(self.read_pref_tags)) self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): From 6c66e6ee563cac6af897b8ed8974a89b3a3762af Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 09:58:38 +0200 Subject: [PATCH 18/22] Improved logging #4 --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 36b9139e..f612023b 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -157,7 +157,7 @@ def mongodump_cmd(self): if self.is_version_gte("3.2.0"): read_pref = self.parse_read_pref() if read_pref: - mongodump_flags.append("--readPreference='%s'" % read_pref) + mongodump_flags.append("--readPreference=%s" % read_pref) elif self.read_pref_tags: logging.fatal("Mongodump must be >= 3.2.0 to set read preference!") sys.exit(1) From c7c9c97a07ece4040bd986985323045123dd7cd3 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 10:12:42 +0200 Subject: [PATCH 19/22] Improved logging #5 --- mongodb_consistent_backup/Common/DB.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 946cfe2c..bb7408d1 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -69,7 +69,10 @@ def client_opts(self): "w": "majority" }) if self.do_rp_tags and self.read_pref_tags: - logging.debug("Using read preference tags: %s" % parse_read_pref_tags(self.read_pref_tags)) + logging.debug("Using read preference mode: %s, tags: %s" % ( + self.read_pref, + parse_read_pref_tags(self.read_pref_tags) + )) self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags if self.do_ssl(): From 59d7792d25a2fa1288e5ff830d005f80ee94eef5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 12:09:24 +0200 Subject: [PATCH 20/22] code cleanup --- mongodb_consistent_backup/Replication/Replset.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py index 4c67dd2c..c21ab6ab 100644 --- a/mongodb_consistent_backup/Replication/Replset.py +++ b/mongodb_consistent_backup/Replication/Replset.py @@ -207,13 +207,12 @@ def find_secondary(self, force=False, quiet=False): score_scale = 100 / score priority = 0 - if self.read_pref_tags: - if not self.has_read_pref_tags(member_config): - logging.info("Found SECONDARY %s without read preference tags: %s, skipping" % ( - member_uri, - parse_read_pref_tags(self.read_pref_tags) - )) - continue + if self.read_pref_tags and not self.has_read_pref_tags(member_config): + logging.info("Found SECONDARY %s without read preference tags: %s, skipping" % ( + member_uri, + parse_read_pref_tags(self.read_pref_tags) + )) + continue if 'hidden' in member_config and member_config['hidden']: score += (score * self.hidden_weight) From 099759c13020c065cdb26f06cc9b4bbb35402177 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 12 Sep 2017 18:58:07 +0200 Subject: [PATCH 21/22] remove spaces in json --- mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index f612023b..a67e1619 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -73,7 +73,7 @@ def parse_read_pref(self, mode="secondary"): rp = {"mode": mode} if self.read_pref_tags: rp["tags"] = parse_read_pref_tags(self.read_pref_tags) - return json.dumps(rp) + return json.dumps(rp, separators=(',', ':')) def parse_mongodump_line(self, line): try: From be3ce3fbbd6c2d5d7703eabaa196ab55d0c6d32d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 13 Sep 2017 02:19:43 +0200 Subject: [PATCH 22/22] merge ssl branch --- mongodb_consistent_backup/Common/DB.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index bb7408d1..c8454484 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -75,6 +75,7 @@ def client_opts(self): )) self.read_pref_tags = self.read_pref_tags.replace(" ", "") opts["readPreferenceTags"] = self.read_pref_tags + if self.do_ssl(): logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( self.ssl_ca_file,