diff --git a/README.rst b/README.rst index 2dc60c6c..280cb9db 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,7 @@ Features - `Nagios NSCA `__ push notification support (*optional*) - Modular backup, archiving, upload and notification components +- Support for MongoDB Authentication and SSL database connections - Multi-threaded, single executable - Auto-scales to number of available CPUs by default @@ -221,6 +222,7 @@ Roadmap - Upload compatibility for ZBackup archive phase *(upload unsupported today)* - Backup retention/rotation *(eg: delete old backups)* - Support more notification methods *(Prometheus, PagerDuty, etc)* +- Support more upload methods *(Rsync, etc)* - Support SSL MongoDB connections - Documentation for running under Docker with persistent volumes - Python unit tests diff --git a/conf/mongodb-consistent-backup.example.conf b/conf/mongodb-consistent-backup.example.conf index 00fd8fd1..3058648a 100644 --- a/conf/mongodb-consistent-backup.example.conf +++ b/conf/mongodb-consistent-backup.example.conf @@ -4,6 +4,12 @@ production: #username: [auth username] (default: none) #password: [auth password] (default: none) #authdb: [auth database] (default: admin) + #ssl: + # enabled: [true|false] (default: false) + # insecure: [true|false] (default: false) + # ca_file: [path] + # crl_file: [path] + # client_cert_file: [path] log_dir: /var/log/mongodb-consistent-backup backup: method: mongodump diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 2ac39eed..f1ec043e 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -8,7 +8,7 @@ from signal import signal, SIGINT, SIGTERM, SIG_IGN from subprocess import Popen, PIPE -from mongodb_consistent_backup.Common import is_datetime +from mongodb_consistent_backup.Common import is_datetime, parse_config_bool from mongodb_consistent_backup.Oplog import Oplog @@ -25,10 +25,13 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump self.threads = threads self.dump_gzip = dump_gzip - self.user = self.config.username - self.password = self.config.password - self.authdb = self.config.authdb - self.binary = self.config.backup.mongodump.binary + self.user = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.binary = self.config.backup.mongodump.binary self.timer_name = "%s-%s" % (self.__class__.__name__, self.uri.replset) self.exit_code = 1 @@ -52,6 +55,18 @@ def close(self, exit_code=None, frame=None): self._command.close() sys.exit(self.exit_code) + def do_ssl(self): + return parse_config_bool(self.config.ssl.enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.config.ssl.insecure) + + def is_version_gte(self, compare): + if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK): + if tuple(compare.split(".")) <= tuple(self.version.split(".")): + return True + return False + def parse_mongodump_line(self, line): try: line = line.rstrip() @@ -118,21 +133,40 @@ def mongodump_cmd(self): mongodump_flags = ["--host", mongodump_uri.host, "--port", str(mongodump_uri.port), "--oplog", "--out", "%s/dump" % self.backup_dir] if self.threads > 0: mongodump_flags.extend(["--numParallelCollections=" + str(self.threads)]) + if self.dump_gzip: mongodump_flags.extend(["--gzip"]) - if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")): + + if self.is_version_gte("3.4.0"): mongodump_flags.extend(["--readPreference=secondary"]) + if self.authdb and self.authdb != "admin": logging.debug("Using database %s for authentication" % self.authdb) mongodump_flags.extend(["--authenticationDatabase", self.authdb]) if self.user and self.password: # >= 3.0.2 supports password input via stdin to mask from ps - if tuple(self.version.split(".")) >= tuple("3.0.2".split(".")): + if self.is_version_gte("3.0.2"): mongodump_flags.extend(["-u", self.user, "-p", '""']) self.do_stdin_passwd = True else: logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this") mongodump_flags.extend(["-u", self.user, "-p", self.password]) + + if self.do_ssl(): + if self.is_version_gte("2.6.0"): + mongodump_flags.append("--ssl") + if self.ssl_ca_file: + mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file]) + if self.ssl_crl_file: + mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file]) + if self.client_cert_file: + mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file]) + if self.do_ssl_insecure(): + mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"]) + else: + logging.fatal("Mongodump must be >= 2.6.0 to enable SSL encryption!") + sys.exit(1) + mongodump_cmd.extend(mongodump_flags) return mongodump_cmd diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py index ad52b25a..fc04991b 100644 --- a/mongodb_consistent_backup/Common/Config.py +++ b/mongodb_consistent_backup/Common/Config.py @@ -8,6 +8,18 @@ from yconf.util import NestedDict +def parse_config_bool(item): + try: + if isinstance(item, bool): + return item + elif isinstance(item, str): + if item.rstrip().lower() is "true": + return True + return False + except: + return False + + class PrintVersions(Action): def __init__(self, option_strings, dest, nargs=0, **kwargs): super(PrintVersions, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, **kwargs) @@ -54,6 +66,11 @@ def makeParser(self): parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str) parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str) parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str) + parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true") + parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)", default=None, type=str) + parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str) + parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str) parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str) parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str) parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index eec89775..d7708b3f 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -4,46 +4,85 @@ from inspect import currentframe, getframeinfo from pymongo import DESCENDING, CursorType, MongoClient from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError +from ssl import CERT_REQUIRED, CERT_NONE from time import sleep +from mongodb_consistent_backup.Common import parse_config_bool from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error class DB: def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5): self.uri = uri - self.username = config.username - self.password = config.password - self.authdb = config.authdb + self.config = config self.do_replset = do_replset self.read_pref = read_pref self.do_connect = do_connect self.conn_timeout = conn_timeout self.retries = retries + self.username = self.config.username + self.password = self.config.password + self.authdb = self.config.authdb + self.ssl_ca_file = self.config.ssl.ca_file + self.ssl_crl_file = self.config.ssl.crl_file + self.ssl_client_cert_file = self.config.ssl.client_cert_file + self.replset = None self._conn = None self._is_master = None + self.connect() self.auth_if_required() + def do_ssl(self): + return parse_config_bool(self.config.ssl.enabled) + + def do_ssl_insecure(self): + return parse_config_bool(self.config.ssl.insecure) + + def client_opts(self): + opts = { + "connect": self.do_connect, + "host": self.uri.hosts(), + "connectTimeoutMS": self.conn_timeout, + "serverSelectionTimeoutMS": self.conn_timeout, + "maxPoolSize": 1, + } + if self.do_replset: + self.replset = self.uri.replset + opts.update({ + "replicaSet": self.replset, + "readPreference": self.read_pref, + "w": "majority" + }) + if self.do_ssl(): + logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % ( + self.ssl_ca_file, + self.ssl_client_cert_file, + self.ssl_crl_file, + self.do_ssl_insecure() + )) + opts.update({ + "ssl": True, + "ssl_ca_certs": self.ssl_ca_file, + "ssl_crlfile": self.ssl_crl_file, + "ssl_certfile": self.ssl_client_cert_file, + "ssl_cert_reqs": CERT_REQUIRED, + }) + if self.do_ssl_insecure(): + opts["ssl_cert_reqs"] = CERT_NONE + return opts + def connect(self): try: - if self.do_replset: - self.replset = self.uri.replset - logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s)" % ( - self.uri, self.replset, self.read_pref + logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % ( + self.uri, + self.replset, + self.read_pref, + self.do_ssl(), )) - conn = MongoClient( - connect=self.do_connect, - host=self.uri.hosts(), - replicaSet=self.replset, - readPreference=self.read_pref, - connectTimeoutMS=self.conn_timeout, - serverSelectionTimeoutMS=self.conn_timeout, - maxPoolSize=1, - w="majority" - ) + conn = MongoClient(**self.client_opts()) if self.do_connect: conn['admin'].command({"ping": 1}) except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py index 27d38df6..5d65b8b3 100644 --- a/mongodb_consistent_backup/Common/__init__.py +++ b/mongodb_consistent_backup/Common/__init__.py @@ -1,4 +1,4 @@ -from Config import Config # NOQA +from Config import Config, parse_config_bool # NOQA from DB import DB # NOQA from LocalCommand import LocalCommand # NOQA from Lock import Lock # NOQA