[PBCKP-2775] Test test_incr_lsn_long_xact_1 fix

Author: oleg gurev

src/pg_probackup.c

@@ -687,7 +687,7 @@ main(int argc, char *argv[])
 	if (instance_config.pgdata != NULL &&
 	    (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) &&
 		!is_absolute_path(instance_config.pgdata))
-		elog(ERROR, "-D, --pgdata must be an absolute path");
+		elog(ERROR, "-D, --pgdata must be an absolute path: %s", instance_config.pgdata);
 
 #if PG_VERSION_NUM >= 110000
 	/* Check xlog-seg-size option */

tests/archive_test.py

@@ -2437,6 +2437,9 @@ def test_archive_get_prefetch_corruption(self):
 
         prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename)
         restored_line = 'LOG:  restored log file "{0}" from archive'.format(filename)
+
+        self.wait_server_wal_exists(replica.data_dir, wal_dir, filename)
+
         tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log'))
         tailer.wait(contains=prefetch_line)
         tailer.wait(contains=restored_line)
@@ -2483,6 +2486,9 @@ def test_archive_show_partial_files_handling(self):
 
         self.switch_wal_segment(node)
 
+        self.wait_instance_wal_exists(backup_dir, 'node',
+                                f"{filename}.gz")
+
         os.rename(
             os.path.join(wals_dir, filename),
             os.path.join(wals_dir, '{0}.part'.format(filename)))

tests/backup_test.py

@@ -3218,6 +3218,10 @@ def test_missing_replication_permission_1(self):
 
         replica.promote()
 
+        # Wait for replica to catch up with master before promoting
+        # to ensure 'backup' role is replicated
+        self.wait_until_replica_catch_with_master(node, replica)
+
         # PAGE
         output = self.backup_node(
             backup_dir, 'node', replica, backup_type='page',

tests/catchup_test.py

@@ -1586,18 +1586,22 @@ def test_dry_run_catchup_delta(self):
         # Cleanup
         src_pg.stop()
 
+    # Skip test, because it's PGDATA is global variable and has impact for other tests
+    # e.g. test_checkdb_amcheck_only_sanity
+    @unittest.skip("skip")
     def test_pgdata_is_ignored(self):
         """ In catchup we still allow PGDATA to be set either from command line
         or from the env var. This test that PGDATA is actually ignored and
         --source-pgadta is used instead
         """
-        node = self.make_simple_node('node',
+        node = self.make_simple_node(
+            base_dir=os.path.join(self.module_name, self.fname, 'node'),
             set_replication = True
             )
         node.slow_start()
 
         # do full catchup
-        dest = self.make_empty_node('dst')
+        dest = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst'))
         self.catchup_node(
             backup_mode = 'FULL',
             source_pgdata = node.data_dir,
@@ -1612,7 +1616,7 @@ def test_pgdata_is_ignored(self):
 
         os.environ['PGDATA']='xxx'
 
-        dest2 = self.make_empty_node('dst')
+        dest2 = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst'))
         self.catchup_node(
             backup_mode = 'FULL',
             source_pgdata = node.data_dir,

tests/checkdb_test.py

@@ -18,7 +18,7 @@ def test_checkdb_amcheck_only_sanity(self):
 
         backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
         node = self.make_simple_node(
-            base_dir="{0}/{1}/node".format(self.module_name, self.fname),
+            base_dir=os.path.join(self.module_name, self.fname, 'node'),
             set_replication=True,
             initdb_params=['--data-checksums'])
 
@@ -227,7 +227,7 @@ def test_basic_checkdb_amcheck_only_sanity(self):
         """"""
         backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
         node = self.make_simple_node(
-            base_dir="{0}/{1}/node".format(self.module_name, self.fname),
+            base_dir=os.path.join(self.module_name, self.fname, 'node'),
             set_replication=True,
             initdb_params=['--data-checksums'])
 

tests/helpers/ptrack_helpers.py

@@ -6,13 +6,14 @@
 import signal
 import subprocess
 import shutil
+from time import sleep
 import six
 import testgres
 import hashlib
 import re
 import getpass
 import select
-from time import sleep
+import time
 import re
 import json
 import random
@@ -150,8 +151,9 @@ def __str__(self):
 
 class PostgresNodeExtended(testgres.PostgresNode):
 
-    def __init__(self, base_dir=None, *args, **kwargs):
-        super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs)
+    def __init__(self, base_dir=None, port = None, bin_dir=None, *args, **kwargs):
+        assert port is None or type(port) == int
+        super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, port = port, bin_dir=bin_dir, *args, **kwargs)
         self.is_started = False
 
     def slow_start(self, replica=False):
@@ -414,25 +416,28 @@ def is_test_result_ok(test_case):
         #
         # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065
 
-        if not isinstance(test_case, unittest.TestCase):
-            raise AssertionError("test_case is not instance of unittest.TestCase")
-
-        if hasattr(test_case, '_outcome'):  # Python 3.4+
-            if hasattr(test_case._outcome, 'errors'):
-                # Python 3.4 - 3.10  (These two methods have no side effects)
-                result = test_case.defaultTestResult()  # These two methods have no side effects
-                test_case._feedErrorsToResult(result, test_case._outcome.errors)
-            else:
-                # Python 3.11+ and pytest 5.3.5+
-                result = test_case._outcome.result
-                if not hasattr(result, 'errors'):
-                    result.errors = []
-                if not hasattr(result, 'failures'):
-                    result.failures = []
-        else:  # Python 2.7, 3.0-3.3
-            result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups)
+        if hasattr(test_case._outcome, 'errors'):
+            # Python 3.4 - 3.10  (These two methods have no side effects)
+            result = test_case.defaultTestResult()  # These two methods have no side effects
+            test_case._feedErrorsToResult(result, test_case._outcome.errors)
+        else:
+            # Python 3.11+ and pytest 5.3.5+
+            result = test_case._outcome.result
+            if not hasattr(result, 'errors'):
+                result.errors = []
+            if not hasattr(result, 'failures'):
+                result.failures = []
 
         ok = all(test != test_case for test, text in result.errors + result.failures)
+        # check subtests as well
+        ok = ok and all(getattr(test, 'test_case', None) != test_case
+                        for test, text in result.errors + result.failures)
+
+        # for pytest 8+
+        if hasattr(result, '_excinfo'):
+            if result._excinfo is not None and len(result._excinfo) > 0:
+                # if test was successful, _excinfo will be None, else it will be non-empty list
+                ok = False
 
         return ok
 
@@ -475,12 +480,14 @@ def pg_config_version(self):
 
     def make_empty_node(
             self,
-            base_dir=None):
+            base_dir=None,
+            port=None,
+            bin_dir=None):
         real_base_dir = os.path.join(self.tmp_path, base_dir)
         shutil.rmtree(real_base_dir, ignore_errors=True)
         os.makedirs(real_base_dir)
 
-        node = PostgresNodeExtended(base_dir=real_base_dir)
+        node = PostgresNodeExtended(base_dir=real_base_dir, port=port, bin_dir=bin_dir)
         node.should_rm_dirs = True
         self.nodes_to_cleanup.append(node)
 
@@ -489,12 +496,14 @@ def make_empty_node(
     def make_simple_node(
             self,
             base_dir=None,
+            port=None,
+            bin_dir=None,
             set_replication=False,
             ptrack_enable=False,
             initdb_params=[],
             pg_options={}):
 
-        node = self.make_empty_node(base_dir)
+        node = self.make_empty_node(base_dir, port=port, bin_dir=bin_dir)
         node.init(
            initdb_params=initdb_params, allow_streaming=set_replication)
 
@@ -910,6 +919,24 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B):
 
         return filelist_diff
 
+    def wait_instance_wal_exists(self, backup_dir, instance, file, timeout=300):
+        """Wait for WAL segment appeared in the WAL archive"""
+        start = time.time()
+        fl = f'wal/{instance}/{file}'
+        while time.time() - start < timeout:
+            if  os.path.exists(fl):
+                break
+            time.sleep(0.25)
+
+    def wait_server_wal_exists(self, data_dir, wal_dir, file, timeout=300):
+        """Wait for WAL segment appeared in the server WAL dir"""
+        start = time.time()
+        fl = f'{data_dir}/{wal_dir}/{file}'
+        while time.time() - start < timeout:
+            if os.path.exists(fl):
+                return
+            time.sleep(0.25)
+
     # used for partial restore
     def truncate_every_file_in_dir(self, path):
         for file in os.listdir(path):

tests/incr_restore_test.py

@@ -1592,17 +1592,12 @@ def test_incr_checksum_long_xact(self):
                 'select count(*) from t1').decode('utf-8').rstrip(),
             '1')
 
-    # @unittest.skip("skip")
-    # @unittest.expectedFailure
-    # This test will pass with Enterprise
-    # because it has checksums enabled by default
-    @unittest.skipIf(ProbackupTest.enterprise, 'skip')
     def test_incr_lsn_long_xact_1(self):
         """
         """
         node = self.make_simple_node(
             base_dir=os.path.join(self.module_name, self.fname, 'node'),
-            set_replication=True)
+            set_replication=True, initdb_params=['--no-data-checksums'])
 
         backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup')
         self.init_pb(backup_dir)
@@ -2046,8 +2041,9 @@ def test_incremental_partial_restore_exclude_lsn(self):
             base_dir=os.path.join(self.module_name, self.fname, 'node1'))
         node1.cleanup()
 
-        node2 = self.make_simple_node(
-            base_dir=os.path.join(self.module_name, self.fname, 'node2'))
+        node2 = self.make_empty_node(
+            base_dir=os.path.join(self.module_name, self.fname, 'node2'), port=node.port)
+        assert node2.port == node.port
         node2.cleanup()
 
         # restore some data into node2
@@ -2063,7 +2059,7 @@ def test_incremental_partial_restore_exclude_lsn(self):
         pgdata1 = self.pgdata_content(node1.data_dir)
 
         # partial incremental restore backup into node2
-        node2.port = node.port
+        # node2.port = node.port
         node2.slow_start()
         node2.stop()
         self.restore_node(

tests/ptrack_test.py

@@ -1781,7 +1781,9 @@ def test_alter_database_set_tablespace_ptrack(self):
 
         # RESTORE
         node_restored = self.make_simple_node(
-            base_dir=os.path.join(self.module_name, self.fname, 'node_restored'))
+            base_dir=os.path.join(self.module_name, self.fname, 'node_restored'),
+            port = node.port)
+        assert node_restored.port == node.port
         node_restored.cleanup()
         self.restore_node(
             backup_dir, 'node',
@@ -1799,7 +1801,6 @@ def test_alter_database_set_tablespace_ptrack(self):
             self.compare_pgdata(pgdata, pgdata_restored)
 
         # START RESTORED NODE
-        node_restored.port = node.port
         node_restored.slow_start()
 
     # @unittest.skip("skip")

tests/requirements.txt

@@ -5,7 +5,7 @@
 #    git+https://github.com/postgrespro/testgres.git@<git-ref>
 # 3. From a local directory
 #    /path/to/local/directory/testgres
-testgres==1.8.5
+testgres==1.12.0
 allure-pytest
 deprecation
 pexpect

tests/set_backup_test.py

@@ -339,6 +339,9 @@ def test_wal_retention_and_pinning_1(self):
 
         node.pgbench_init(scale=2)
 
+        self.wait_instance_wal_exists(backup_dir, 'node',
+                                "000000010000000000000004")
+
         # Purge backups
         out = self.delete_expired(
             backup_dir, 'node',