[PBCKP-2775] Make auth_test.py multithreaded

Author: Oleg Gurev

tests/auth_test.py

@@ -174,51 +174,36 @@ def test_backup_via_unprivileged_user(self):
 class AuthTest(unittest.TestCase):
     pb = None
     node = None
+    test_path = None
 
-    # TODO move to object scope, replace module_name
-    @classmethod
-    def setUpClass(cls):
+    @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.")
+    def setUp(self):
 
-        super(AuthTest, cls).setUpClass()
+        super().setUp()
 
-        cls.pb = ProbackupTest()
-        cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup')
+        self.pb = ProbackupTest()
+        self.test_path = os.path.join(self.pb.tmp_path, module_name, self._testMethodName)
+        self.backup_dir = os.path.join(self.test_path, 'backup')
 
-        cls.node = cls.pb.make_simple_node(
-            base_dir="{}/node".format(module_name),
+        self.node = self.pb.make_simple_node(
+            base_dir=os.path.join(self.test_path, 'node'),
             set_replication=True,
             initdb_params=['--data-checksums', '--auth-host=md5']
         )
 
-        cls.username = cls.pb.get_username()
+        self.modify_pg_hba(self.node, self.pb.get_username())
 
-        cls.modify_pg_hba(cls.node)
-
-        cls.pb.init_pb(cls.backup_dir)
-        cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node)
-        cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node)
+        self.pb.init_pb(self.backup_dir)
+        self.pb.add_instance(self.backup_dir, self.node.name, self.node)
+        self.pb.set_archiving(self.backup_dir, self.node.name, self.node)
         try:
-            cls.node.slow_start()
+            self.node.slow_start()
         except StartNodeException:
             raise unittest.skip("Node hasn't started")
 
-        if cls.pb.get_version(cls.node) < 100000:
-            cls.node.safe_psql(
-                "postgres",
-                "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; "
-                "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
-                "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; "
-                "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; "
-                "GRANT EXECUTE ON FUNCTION txid_current() TO backup; "
-                "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; "
-                "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;")
-        elif cls.pb.get_version(cls.node) < 150000:
-            cls.node.safe_psql(
+        version = self.pb.get_version(self.node)
+        if version < 150000:
+            self.node.safe_psql(
                 "postgres",
                 "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; "
                 "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
@@ -233,7 +218,7 @@ def setUpClass(cls):
                 "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; "
                 "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;")
         else:
-            cls.node.safe_psql(
+            self.node.safe_psql(
                 "postgres",
                 "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; "
                 "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
@@ -247,16 +232,29 @@ def setUpClass(cls):
                 "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; "
                 "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;")
 
-        cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass')
-
-    # TODO move to object scope, replace module_name
-    @classmethod
-    def tearDownClass(cls):
-        cls.node.cleanup()
-        cls.pb.del_test_dir(module_name, '')
+        if version >= 150000:
+            home_dir = os.path.join(self.test_path, "home")
+            os.makedirs(home_dir, exist_ok=True)
+            self.pb.test_env['HOME'] = home_dir
+            self.pgpass_file = os.path.join(home_dir, '.pgpass')
+            self.pgpass_file_lock = None
+        else:
+            # before PGv15 only true home dir were inspected.
+            # Since we can't have separate file per test, we have to serialize
+            # tests.
+            self.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass')
+            self.pgpass_file_lock = self.pgpass_file + '~probackup_test_lock'
+            # have to lock pgpass by creating file in exclusive mode
+            for i in range(120):
+                try:
+                    open(self.pgpass_file_lock, "x").close()
+                except FileExistsError:
+                    time.sleep(1)
+                else:
+                    break
+            else:
+                raise TimeoutError("can't create ~/.pgpass~probackup_test_lock for 120 seconds")
 
-    @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.")
-    def setUp(self):
         self.pb_cmd = ['backup',
                     '-B', self.backup_dir,
                     '--instance', self.node.name,
@@ -268,6 +266,19 @@ def setUp(self):
                     ]
 
     def tearDown(self):
+        if (self.pgpass_file_lock
+            and hasattr(self, "pgpass_line")
+            and os.path.exists(self.pgpass_file)):
+            with open(self.pgpass_file, 'r', encoding="utf-8") as fl:
+                lines = fl.readlines()
+            if self.pgpass_line in lines:
+                lines.remove(self.pgpass_line)
+            if len(lines) == 0:
+                os.remove(self.pgpass_file)
+            else:
+                with open(self.pgpass_file, 'w', encoding="utf-8") as fl:
+                    fl.writelines(lines)
+
         if "PGPASSWORD" in self.pb.test_env.keys():
             del self.pb.test_env["PGPASSWORD"]
 
@@ -279,6 +290,10 @@ def tearDown(self):
         except OSError:
             pass
 
+        test_path = os.path.join(self.pb.tmp_path, module_name)
+        self.node.cleanup()
+        self.pb.del_test_dir(test_path, self._testMethodName)
+
     def test_empty_password(self):
         """ Test case: PGPB_AUTH03 - zero password length """
         try:
@@ -313,7 +328,7 @@ def test_ctrl_c_event(self):
 
     def test_pgpassfile_env(self):
         """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """
-        path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf')
+        path = os.path.join(self.test_path, 'pgpass.conf')
         line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password'])
         self.create_pgpass(path, line)
         self.pb.test_env["PGPASSFILE"] = path
@@ -367,7 +382,7 @@ def run_pb_with_auth(self, password=None, add_args = [], kill=False):
 
 
     @classmethod
-    def modify_pg_hba(cls, node):
+    def modify_pg_hba(self, node, username):
         """
         Description:
             Add trust authentication for user postgres. Need for add new role and set grant.
@@ -378,11 +393,12 @@ def modify_pg_hba(cls, node):
         with open(hba_conf, 'r+') as fio:
             data = fio.read()
             fio.seek(0)
-            fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data))
+            fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (username, data))
 
 
     def create_pgpass(self, path, line):
+        self.pgpass_line = line+"\n"
         with open(path, 'w') as passfile:
             # host:port:db:username:password
-            passfile.write(line)
+            passfile.write(self.pgpass_line)
         os.chmod(path, 0o600)