From 72976a62d6a0dfacbdbd408027ea27b341305288 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 20 Feb 2018 22:56:31 +0300 Subject: [PATCH 1/3] initial implementation of config stack --- testgres/__init__.py | 8 ++- testgres/cache.py | 35 +++------- testgres/config.py | 149 +++++++++++++++++++++++++++++++++++++---- testgres/consts.py | 1 + testgres/exceptions.py | 6 +- testgres/node.py | 21 +++--- testgres/utils.py | 37 +++++----- tests/test_simple.py | 70 +++++++++---------- 8 files changed, 217 insertions(+), 110 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 29b017a9..7d5a7125 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -1,6 +1,12 @@ from .api import get_new_node from .backup import NodeBackup -from .config import TestgresConfig, configure_testgres + +from .config import \ + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + push_config, \ + pop_config from .connection import \ IsolationLevel, \ diff --git a/testgres/cache.py b/testgres/cache.py index 2fad48f8..d0eaac9c 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,13 +1,11 @@ # coding: utf-8 -import atexit import os import shutil -import tempfile from six import raise_from -from .config import TestgresConfig +from .config import testgres_config from .exceptions import \ InitNodeException, \ @@ -18,44 +16,31 @@ execute_utility -def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): +def cached_initdb(data_dir, logfile=None, params=None): """ Perform initdb or use cached node files. """ - def call_initdb(initdb_dir): + def call_initdb(initdb_dir, log=None): try: _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] - execute_utility(_params + initdb_params, initdb_logfile) + execute_utility(_params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) - def rm_cached_data_dir(cached_data_dir): - shutil.rmtree(cached_data_dir, ignore_errors=True) - - # Call initdb if we have custom params or shouldn't cache it - if initdb_params or not TestgresConfig.cache_initdb: - call_initdb(data_dir) + if params or not testgres_config.cache_initdb: + call_initdb(data_dir, logfile) else: - # Set default temp dir for cached initdb - if TestgresConfig.cached_initdb_dir is None: - - # Create default temp dir - TestgresConfig.cached_initdb_dir = tempfile.mkdtemp() - - # Schedule cleanup - atexit.register(rm_cached_data_dir, - TestgresConfig.cached_initdb_dir) - # Fetch cached initdb dir - cached_data_dir = TestgresConfig.cached_initdb_dir + cached_data_dir = testgres_config.cached_initdb_dir # Initialize cached initdb - if not os.listdir(cached_data_dir): + if not os.path.exists(cached_data_dir) or \ + not os.listdir(cached_data_dir): call_initdb(cached_data_dir) try: # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) except Exception as e: - raise_from(InitNodeException("Failed to copy files"), e) + raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/config.py b/testgres/config.py index 52c5ee58..782f315c 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -1,26 +1,35 @@ # coding: utf-8 +import atexit +import copy +import shutil +import tempfile -class TestgresConfig: +from contextlib import contextmanager + + +class GlobalConfig(object): """ Global config (override default settings). Attributes: - cache_initdb: shall we use cached initdb instance? - cached_initdb_dir: shall we create a temp dir for cached initdb? + cache_initdb: shall we use cached initdb instance? + cached_initdb_dir: shall we create a temp dir for cached initdb? - cache_pg_config: shall we cache pg_config results? + cache_pg_config: shall we cache pg_config results? - use_python_logging: use python logging configuration for all nodes. - error_log_lines: N of log lines to be included into exception (0=inf). + use_python_logging: use python logging configuration for all nodes. + error_log_lines: N of log lines to be shown in exception (0=inf). - node_cleanup_full: shall we remove EVERYTHING (including logs)? + node_cleanup_full: shall we remove EVERYTHING (including logs)? node_cleanup_on_good_exit: remove base_dir on nominal __exit__(). node_cleanup_on_bad_exit: remove base_dir on __exit__() via exception. + + NOTE: attributes must not be callable or begin with __. """ cache_initdb = True - cached_initdb_dir = None + _cached_initdb_dir = None cache_pg_config = True @@ -31,12 +40,128 @@ class TestgresConfig: node_cleanup_on_good_exit = True node_cleanup_on_bad_exit = False + @property + def cached_initdb_dir(self): + return self._cached_initdb_dir + + @cached_initdb_dir.setter + def cached_initdb_dir(self, value): + self._cached_initdb_dir = value + + if value: + cached_initdb_dirs.add(value) + + def __init__(self, **options): + self.update(options) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def __setattr__(self, name, value): + if name not in self.keys(): + raise TypeError('Unknown option {}'.format(name)) + + super(GlobalConfig, self).__setattr__(name, value) + + def keys(self): + keys = [] + + for key in dir(GlobalConfig): + if not key.startswith('__') and not callable(self[key]): + keys.append(key) + + return keys + + def items(self): + return ((key, self[key]) for key in self.keys()) + + def update(self, config): + for key, value in config.items(): + self[key] = value + + return self + + def copy(self): + return copy.copy(self) + + +# cached dirs to be removed +cached_initdb_dirs = set() + +# default config object +testgres_config = GlobalConfig() + +# NOTE: for compatibility +TestgresConfig = testgres_config + +# stack of GlobalConfigs +config_stack = [testgres_config] + + +def rm_cached_initdb_dirs(): + for d in cached_initdb_dirs: + shutil.rmtree(d, ignore_errors=True) + + +def push_config(**options): + """ + Permanently set custom GlobalConfig options + and put previous settings on top of stack. + """ + + # push current config to stack + config_stack.append(testgres_config.copy()) + + return testgres_config.update(options) + + +def pop_config(): + """ + Set previous GlobalConfig options from stack. + """ + + if len(config_stack) <= 1: + raise IndexError('Reached initial config') + + # restore popped config + return testgres_config.update(config_stack.pop()) + + +@contextmanager +def scoped_config(**options): + """ + Temporarily set custom GlobalConfig options for this context. + + Example: + >>> with scoped_config(cache_initdb=False): + ... with get_new_node().init().start() as node: + ... print(node.execute('select 1')) + """ + + # set a new config with options + config = push_config(**options) + try: + # return it + yield config + finally: + # restore previous config + pop_config() + def configure_testgres(**options): """ - Configure testgres. - Look at TestgresConfig to check what can be changed. + Adjust current global options. + Look at GlobalConfig to learn what can be set. """ - for key, option in options.items(): - setattr(TestgresConfig, key, option) + testgres_config.update(options) + + +# NOTE: to be executed at exit() +atexit.register(rm_cached_initdb_dirs) + +# NOTE: assign initial cached dir for initdb +testgres_config.cached_initdb_dir = tempfile.mkdtemp() diff --git a/testgres/consts.py b/testgres/consts.py index fea3dc92..e8dd8c99 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,7 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" diff --git a/testgres/exceptions.py b/testgres/exceptions.py index 69fe43db..6832c788 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -9,11 +9,7 @@ class TestgresException(Exception): @six.python_2_unicode_compatible class ExecUtilException(TestgresException): - def __init__(self, - message=None, - command=None, - exit_code=0, - out=None): + def __init__(self, message=None, command=None, exit_code=0, out=None): super(ExecUtilException, self).__init__(message) self.message = message diff --git a/testgres/node.py b/testgres/node.py index d39eff75..2afab665 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -13,7 +13,7 @@ from .cache import cached_initdb -from .config import TestgresConfig +from .config import testgres_config from .connection import \ NodeConnection, \ @@ -88,8 +88,8 @@ def __init__(self, name=None, port=None, base_dir=None): self.base_dir = base_dir # defaults for __exit__() - self.cleanup_on_good_exit = TestgresConfig.node_cleanup_on_good_exit - self.cleanup_on_bad_exit = TestgresConfig.node_cleanup_on_bad_exit + self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit + self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit self.shutdown_max_attempts = 3 # private @@ -150,7 +150,7 @@ def _try_shutdown(self, max_attempts): self.stop() break # OK except ExecUtilException: - pass # one more time + pass # one more time except Exception: # TODO: probably kill stray instance eprint('cannot stop node {}'.format(self.name)) @@ -204,7 +204,7 @@ def _prepare_dirs(self): os.makedirs(self.logs_dir) def _maybe_start_logger(self): - if TestgresConfig.use_python_logging: + if testgres_config.use_python_logging: # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_name) @@ -223,7 +223,7 @@ def _collect_special_files(self): (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), (os.path.join(self.data_dir, HBA_CONF_FILE), 0), - (self.pg_log_name, TestgresConfig.error_log_lines) + (self.pg_log_name, testgres_config.error_log_lines) ] for f, num_lines in files: @@ -248,7 +248,7 @@ def init(self, fsync=False, unix_sockets=True, allow_streaming=True, - initdb_params=[]): + initdb_params=None): """ Perform initdb for this node. @@ -266,8 +266,9 @@ def init(self, self._prepare_dirs() # initialize this PostgreSQL node - initdb_log = os.path.join(self.logs_dir, "initdb.log") - cached_initdb(self.data_dir, initdb_log, initdb_params) + cached_initdb(data_dir=self.data_dir, + logfile=self.utils_log_name, + params=initdb_params) # initialize default config files self.default_conf(fsync=fsync, @@ -603,7 +604,7 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if TestgresConfig.node_cleanup_full: + if testgres_config.node_cleanup_full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs diff --git a/testgres/utils.py b/testgres/utils.py index b9152aba..01befbc5 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -13,7 +13,7 @@ from distutils.version import LooseVersion -from .config import TestgresConfig +from .config import testgres_config from .exceptions import ExecUtilException # rows returned by PG_CONFIG @@ -68,7 +68,7 @@ def generate_app_name(): return 'testgres-{}'.format(str(uuid.uuid4())) -def execute_utility(args, logfile): +def execute_utility(args, logfile=None): """ Execute utility (pg_ctl, pg_dump etc). @@ -94,27 +94,26 @@ def execute_utility(args, logfile): command = u' '.join(args) # write new log entry if possible - try: - with io.open(logfile, 'a') as file_out: - file_out.write(command) + if logfile: + try: + with io.open(logfile, 'a') as file_out: + file_out.write(command) - if out: - # comment-out lines - lines = ('# ' + l for l in out.splitlines(True)) - file_out.write(u'\n') - file_out.writelines(lines) + if out: + # comment-out lines + lines = ('# ' + l for l in out.splitlines(True)) + file_out.write(u'\n') + file_out.writelines(lines) - file_out.write(u'\n') - except IOError: - pass + file_out.write(u'\n') + except IOError: + pass exit_code = process.returncode if exit_code: message = 'Utility exited with non-zero code' - raise ExecUtilException(message=message, - command=command, - exit_code=exit_code, - out=out) + raise ExecUtilException( + message=message, command=command, exit_code=exit_code, out=out) return out @@ -146,7 +145,7 @@ def get_bin_path(filename): def get_pg_config(): """ Return output of pg_config (provided that it is installed). - NOTE: this fuction caches the result by default (see TestgresConfig). + NOTE: this fuction caches the result by default (see GlobalConfig). """ def cache_pg_config_data(cmd): @@ -166,7 +165,7 @@ def cache_pg_config_data(cmd): return data # drop cache if asked to - if not TestgresConfig.cache_pg_config: + if not testgres_config.cache_pg_config: global _pg_config_data _pg_config_data = {} diff --git a/tests/test_simple.py b/tests/test_simple.py index edbfdf04..98593a7e 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,8 +23,7 @@ TimeoutException from testgres import \ - TestgresConfig, \ - configure_testgres + scoped_config from testgres import \ NodeStatus, \ @@ -458,32 +457,31 @@ def test_logging(self): } logging.config.dictConfig(log_conf) - configure_testgres(use_python_logging=True) - node_name = 'master' - with get_new_node(name=node_name) as master: - master.init().start() + with scoped_config(use_python_logging=True): + node_name = 'master' - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) + with get_new_node(name=node_name) as master: + master.init().start() - # let logging worker do the job - time.sleep(0.1) + # execute a dummy query a few times + for i in range(20): + master.execute('select 1') + time.sleep(0.01) - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + # let logging worker do the job + time.sleep(0.1) - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - self.assertTrue(master._logger.is_alive()) + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + self.assertTrue(any(node_name in s for s in lines)) - configure_testgres(use_python_logging=False) + # test logger after stop/start/restart + master.stop() + master.start() + master.restart() + self.assertTrue(master._logger.is_alive()) @unittest.skipUnless( util_is_executable("pgbench"), "pgbench may be missing") @@ -506,9 +504,6 @@ def test_pgbench(self): self.assertTrue('tps' in out) def test_pg_config(self): - # set global if it wasn't set - configure_testgres(cache_pg_config=True) - # check same instances a = get_pg_config() b = get_pg_config() @@ -517,23 +512,22 @@ def test_pg_config(self): # save right before config change c1 = get_pg_config() - # modify setting - configure_testgres(cache_pg_config=False) - self.assertFalse(TestgresConfig.cache_pg_config) + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: - # save right after config change - c2 = get_pg_config() + # sanity check for value + self.assertFalse(config.cache_pg_config) - # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + # save right after config change + c2 = get_pg_config() - # check different instances - a = get_pg_config() - b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + # check different instances after config change + self.assertNotEqual(id(c1), id(c2)) - # restore setting - configure_testgres(cache_pg_config=True) + # check different instances + a = get_pg_config() + b = get_pg_config() + self.assertNotEqual(id(a), id(b)) def test_unix_sockets(self): with get_new_node() as node: From dd8c5000fae3757248785a275a818a994f042673 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 20 Feb 2018 23:51:06 +0300 Subject: [PATCH 2/3] more tests for config stack --- tests/test_simple.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 98593a7e..60f437c9 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,10 @@ TimeoutException from testgres import \ - scoped_config + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + pop_config from testgres import \ NodeStatus, \ @@ -529,6 +532,29 @@ def test_pg_config(self): b = get_pg_config() self.assertNotEqual(id(a), id(b)) + def test_config_stack(self): + # no such option + with self.assertRaises(TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with self.assertRaises(IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + self.assertEqual(c1.cached_initdb_dir, d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + self.assertEqual(c2.cached_initdb_dir, d2) + + self.assertEqual(c1.cached_initdb_dir, d1) + + self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + def test_unix_sockets(self): with get_new_node() as node: node.init(unix_sockets=False, allow_streaming=True) From 3378f28064c772ab015a0ae0a918a5873ab17c0b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 12:44:16 +0300 Subject: [PATCH 3/3] small fixes, improve tests --- testgres/config.py | 9 +++++---- tests/test_simple.py | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index 782f315c..d192b6d5 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -98,7 +98,7 @@ def copy(self): TestgresConfig = testgres_config # stack of GlobalConfigs -config_stack = [testgres_config] +config_stack = [] def rm_cached_initdb_dirs(): @@ -123,7 +123,7 @@ def pop_config(): Set previous GlobalConfig options from stack. """ - if len(config_stack) <= 1: + if len(config_stack) == 0: raise IndexError('Reached initial config') # restore popped config @@ -141,9 +141,10 @@ def scoped_config(**options): ... print(node.execute('select 1')) """ - # set a new config with options - config = push_config(**options) try: + # set a new config with options + config = push_config(**options) + # return it yield config finally: diff --git a/tests/test_simple.py b/tests/test_simple.py index 60f437c9..6714ee32 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -549,7 +549,16 @@ def test_config_stack(self): self.assertEqual(c1.cached_initdb_dir, d1) with scoped_config(cached_initdb_dir=d2) as c2: + + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with self.assertRaises(TypeError): + with scoped_config(dummy=True): + pass + self.assertEqual(c2.cached_initdb_dir, d2) + self.assertEqual(len(testgres.config.config_stack), stack_size) self.assertEqual(c1.cached_initdb_dir, d1)