diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index e60331f0..f792bd19 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -1,8 +1,8 @@ -from pg_probackup2.gdb import GDBobj -from pg_probackup2.app import ProbackupApp, ProbackupException -from pg_probackup2.init_helpers import init_params -from pg_probackup2.storage.fs_backup import FSTestBackupDir -from pg_probackup2.storage.s3_backup import S3TestBackupDir +from testgres_pg_probackup2.gdb import GDBobj +from testgres_pg_probackup2.app import ProbackupApp, ProbackupException +from testgres_pg_probackup2.init_helpers import init_params +from testgres_pg_probackup2.storage.fs_backup import FSTestBackupDir +from testgres_pg_probackup2.storage.s3_backup import S3TestBackupDir __all__ = [ "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "S3TestBackupDir", "GDBobj" diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md index b62bf24b..d1a18f4d 100644 --- a/testgres/plugins/pg_probackup2/README.md +++ b/testgres/plugins/pg_probackup2/README.md @@ -35,7 +35,7 @@ python my_tests.py ### Examples -Here is an example of what you can do with `testgres-pg_probackup2`: +Here is an example of what you can do with `testgres_pg_probackup2`: ```python # You can see full script here plugins/pg_probackup2/pg_probackup2/tests/basic_test.py diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 371eb078..8ab07daa 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,9 +4,9 @@ from distutils.core import setup setup( - version='0.0.1', + version='0.0.2', name='testgres_pg_probackup2', - packages=['pg_probackup2', 'pg_probackup2.storage'], + packages=['testgres_pg_probackup2', 'testgres_pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', url='https://github.com/postgrespro/testgres', long_description_content_type='text/markdown', @@ -14,5 +14,5 @@ author='Postgres Professional', author_email='testgres@postgrespro.ru', keywords=['pg_probackup', 'testing', 'testgres'], - install_requires=['testgres>=1.9.2'] + install_requires=['testgres>=1.9.3'] ) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/__init__.py similarity index 100% rename from testgres/plugins/pg_probackup2/pg_probackup2/__init__.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/__init__.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/app.py similarity index 93% rename from testgres/plugins/pg_probackup2/pg_probackup2/app.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/app.py index 2c31de51..7b77df11 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/app.py @@ -37,6 +37,17 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +# Local or S3 backup +fs_backup_class = FSTestBackupDir +if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): + root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) + if root not in sys.path: + sys.path.append(root) + from pg_probackup2.storage.s3_backup import S3TestBackupDir + + fs_backup_class = S3TestBackupDir + + class ProbackupApp: def __init__(self, test_class: unittest.TestCase, @@ -283,7 +294,7 @@ def backup_replica_node(self, instance, node, data_dir=False, *, gdb.continue_execution_until_exit() output = self.read_pb_log() - self.unlink_pg_log() + unlink_pg_log() parsed_output = re.compile(r'Backup \S+ completed').search(output) assert parsed_output, f"Expected: `Backup 'backup_id' completed`, but found `{output}`" backup_id = parsed_output[0].split(' ')[1] @@ -698,65 +709,60 @@ def set_archiving( node.set_auto_conf(options) - def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): - """ - Execute pg_switch_wal() in given node - Args: - node: an instance of PostgresNode or NodeConnection class - """ - if isinstance(node, testgres.PostgresNode): - with node.connect('postgres') as con: - if and_tx: - con.execute('select txid_current()') - lsn = con.execute('select pg_switch_wal()')[0][0] - else: - lsn = node.execute('select pg_switch_wal()')[0][0] +def switch_wal_segment(node, sleep_seconds=1, and_tx=False): + """ + Execute pg_switch_wal() in given node - if sleep_seconds > 0: - time.sleep(sleep_seconds) - return lsn + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + lsn = con.execute('select pg_switch_wal()')[0][0] + else: + lsn = node.execute('select pg_switch_wal()')[0][0] - @contextlib.contextmanager - def switch_wal_after(self, node, seconds, and_tx=True): - tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) - tm.start() - try: - yield - finally: - tm.cancel() - tm.join() + if sleep_seconds > 0: + time.sleep(sleep_seconds) + return lsn - def read_pb_log(self): - with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: - return fl.read() - def unlink_pg_log(self): - os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) +@contextlib.contextmanager +def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() - def load_backup_class(fs_type): - fs_type = os.environ.get('PROBACKUP_FS_TYPE') - implementation = f"{__package__}.fs_backup.FSTestBackupDir" - if fs_type: - implementation = fs_type - print("Using ", implementation) - module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) +def read_pb_log(pb_log_path): + with open(os.path.join(pb_log_path, 'pg_probackup.log')) as fl: + return fl.read() - module = importlib.import_module(module_name) - return getattr(module, class_name) +def unlink_pg_log(pb_log_path): + os.unlink(os.path.join(pb_log_path, 'pg_probackup.log')) -# Local or S3 backup -fs_backup_class = FSTestBackupDir -if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): - root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) - if root not in sys.path: - sys.path.append(root) - from pg_probackup2.storage.s3_backup import S3TestBackupDir +def load_backup_class(): + fs_type = os.environ.get('PROBACKUP_FS_TYPE') + implementation = f"{__package__}.fs_backup.FSTestBackupDir" + if fs_type: + implementation = fs_type + + print("Using ", implementation) + module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) + + module = importlib.import_module(module_name) + + return getattr(module, class_name) - fs_backup_class = S3TestBackupDir - def build_backup_dir(self, backup='backup'): - return fs_backup_class(rel_path=self.rel_path, backup=backup) +def build_backup_dir(rel_path, backup='backup'): + return fs_backup_class(rel_path=rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/gdb.py similarity index 100% rename from testgres/plugins/pg_probackup2/pg_probackup2/gdb.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/gdb.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/init_helpers.py similarity index 95% rename from testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/init_helpers.py index 7af21eb6..e06b8aec 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/init_helpers.py @@ -77,6 +77,7 @@ def __init__(self): # Get the directory from which the script was executed self.source_path = os.getcwd() + self.pgdata_path = test_env.get('PGPROBACKUP_PGDATA_DIR') tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') if tmp_path and os.path.isabs(tmp_path): self.tmp_path = tmp_path @@ -200,6 +201,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + self.backup_rel_path = test_env.get('PGPROBACKUP_BACKUPS_TMP_DIR', None) + if self.backup_rel_path and not os.path.isabs(self.backup_rel_path): + raise Exception("PGPROBACKUP_BACKUPS_TMP_DIR should be an absolute path") + def test_env(self): return self._test_env.copy() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/__init__.py similarity index 100% rename from testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/__init__.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/fs_backup.py similarity index 100% rename from testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/fs_backup.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/s3_backup.py similarity index 100% rename from testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/storage/s3_backup.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/tests/basic_test.py similarity index 87% rename from testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py rename to testgres/plugins/pg_probackup2/testgres_pg_probackup2/tests/basic_test.py index f5a82d38..aaf0b4bf 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/testgres_pg_probackup2/tests/basic_test.py @@ -2,9 +2,8 @@ import shutil import unittest import testgres -from pg_probackup2.app import ProbackupApp -from pg_probackup2.init_helpers import Init, init_params -from pg_probackup2.app import build_backup_dir +from testgres_pg_probackup2.app import ProbackupApp, build_backup_dir +from testgres_pg_probackup2.init_helpers import Init, init_params class TestUtils: @@ -36,11 +35,13 @@ def setup_test_environment(self): def setup_test_paths(self): self.rel_path = os.path.join(self.module_name, self.fname) self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) os.makedirs(self.test_path) self.pb_log_path = os.path.join(self.test_path, "pb_log") def setup_backup_dir(self): - self.backup_dir = build_backup_dir(self, 'backup') + self.backup_dir = build_backup_dir(init_params.backup_rel_path or self.rel_path, 'backup') self.backup_dir.cleanup() def setup_probackup(self): @@ -51,6 +52,7 @@ def setup_probackup(self): def tearDown(self): if os.path.exists(self.test_path): shutil.rmtree(self.test_path) + self.backup_dir.cleanup() class BasicTest(ProbackupTest):