Bugfix issue313 (#317)

* Changing some structure

* Removed something

* Implemented most changes related to PXB 8 support with --test_mode for issue #313

* adding new lines to support PXB 8.0

* Some design changes

Vagrantfile

@@ -112,7 +112,7 @@ Vagrant.configure("2") do |config|
      virtualenv -p /usr/bin/python3.5 py_3_5_autoxtrabackup
      cd /vagrant
      sudo /home/vagrant/py_3_5_autoxtrabackup/bin/python setup.py install
-     sudo sed -i "0,/^[ \t]*testpath[ \t]*=.*$/s|^[ \t]*testpath[ \t]*=.*$|testpath=\/home\/vagrant\/XB_TEST\/server_dir|" /etc/bck.conf
+     sudo sed -i "0,/^[ \t]*testpath[ \t]*=.*$/s|^[ \t]*testpath[ \t]*=.*$|testpath=\/home\/vagrant\/XB_TEST\/server_dir|" /home/vagrant/.autoxtrabackup/autoxtrabackup.cnf
      cd /home/vagrant
      sudo chown -R vagrant:vagrant *
      touch python-sudo.sh
@@ -129,4 +129,4 @@ Vagrant.configure("2") do |config|
      /usr/local/bin/bats prepare_env.bats
      chown -R vagrant:vagrant *
 SHELL
-end
+end

autoxtrabackup.py

@@ -218,7 +218,6 @@ def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
                 b = Backup(config=defaults_file)
                 b.show_tags(backup_dir=b.backupdir)
             elif test_mode and defaults_file:
-                # TODO: do staff here to implement all in one things for running test mode
                 logger.warning("Enabled Test Mode!!!")
                 logger.debug("Starting Test Mode")
                 test_obj = RunnerTestMode(config=defaults_file)
@@ -228,6 +227,11 @@ def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
                             test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1)
                         else:
                             test_obj.wipe_backup_prepare_copyback(basedir=basedir)
+                    elif ('8.0' in basedir) and ('8_0_ps_8_0' in defaults_file):
+                        if keyring_vault == 1:
+                            test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1)
+                        else:
+                            test_obj.wipe_backup_prepare_copyback(basedir=basedir)
                     elif ('5.6' in basedir) and ('2_4_ps_5_6' in defaults_file):
                         test_obj.wipe_backup_prepare_copyback(basedir=basedir)
                     elif ('5.6' in basedir) and ('2_3_ps_5_6' in defaults_file):

general_conf/autoxtrabackup.cnf

@@ -90,13 +90,13 @@ chown_command=chown -R mysql:mysql
 
 # Do not touch; this is for --test_mode, which is testing for XtraBackup itself.
 #[TestConf]
-#ps_branches=5.5 5.6 5.7
-#pxb_branches=2.3 2.4
+#ps_branches=5.5 5.6 5.7 8.0
+#pxb_branches=2.3 2.4 8.0
 #gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git
 #pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git
 #testpath=/home/shahriyar.rzaev/XB_TEST/server_dir
 #incremental_count=3
 #make_slaves=1
-#xb_configs=xb_2_4_ps_5_6.cnf xb_2_4_ps_5_7.cnf xb_2_3_ps_5_6.cnf xb_2_3_ps_5_5.cnf xb_2_4_ps_5_5.cnf
+#xb_configs=xb_2_4_ps_5_6.cnf xb_2_4_ps_5_7.cnf xb_2_3_ps_5_6.cnf xb_2_3_ps_5_5.cnf xb_2_4_ps_5_5.cnf xb_8_0_ps_8_0.cnf
 #default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row
 #mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K

general_conf/check_env.py

@@ -14,9 +14,9 @@ class CheckEnv(GeneralClass):
     def __init__(self, config=path_config.config_path_file, full_dir=None, inc_dir=None):
         self.conf = config
         GeneralClass.__init__(self, self.conf)
-        if full_dir is not None:
+        if full_dir:
             self.full_dir = full_dir
-        if inc_dir is not None:
+        if inc_dir:
             self.inc_dir = inc_dir
 
     def check_mysql_uptime(self, options=None):
@@ -48,9 +48,7 @@ def check_mysql_uptime(self, options=None):
 
         logger.debug("Running mysqladmin command -> {}".format(filteredargs))
 
-        #statusargs = shlex.split(statusargs)
         status, output = subprocess.getstatusoutput(statusargs)
-        #myadmin = subprocess.Popen(statusargs, stdout=subprocess.PIPE)
 
         if status == 0:
             logger.debug('OK: Server is Up and running')
@@ -59,13 +57,6 @@ def check_mysql_uptime(self, options=None):
             logger.error('FAILED: Server is NOT Up')
             raise RuntimeError('FAILED: Server is NOT Up')
 
-        # if not ('Uptime' in str(myadmin.stdout.read())):
-        #     logger.error('FAILED: Server is NOT Up')
-        #     raise RuntimeError('FAILED: Server is NOT Up')
-        # else:
-        #     logger.debug('OK: Server is Up and running')
-        #     return True
-
     def check_mysql_conf(self):
         '''
         Method for checking passed MySQL my.cnf defaults file. If it is not passed then skip this check
@@ -74,7 +65,7 @@ def check_mysql_conf(self):
         if self.mycnf is None or self.mycnf == '':
             logger.debug("Skipping my.cnf check, because it is not specified")
             return True
-        elif not os.path.exists(self.mycnf) and (self.mycnf is not None):
+        elif not os.path.exists(self.mycnf) and self.mycnf:
             logger.error('FAILED: MySQL configuration file path does NOT exist')
             raise RuntimeError('FAILED: MySQL configuration file path does NOT exist')
         else:
@@ -86,65 +77,78 @@ def check_mysql_mysql(self):
         Method for checking mysql client path
         :return: True on success, raise RuntimeError on error.
         '''
-        if not os.path.exists(self.mysql):
-            logger.error('FAILED: {} doest NOT exist'.format(self.mysql))
-            raise RuntimeError('FAILED: {} doest NOT exist'.format(self.mysql))
-        else:
+        if os.path.exists(self.mysql):
             logger.debug('OK: {} exists'.format(self.mysql))
             return True
+        else:
+            logger.error('FAILED: {} doest NOT exist'.format(self.mysql))
+            raise RuntimeError('FAILED: {} doest NOT exist'.format(self.mysql))
 
     def check_mysql_mysqladmin(self):
         '''
         Method for checking mysqladmin path
         :return: True on success, raise RuntimeError on error.
         '''
-        if not os.path.exists(self.mysqladmin):
-            logger.error('FAILED: {} does NOT exist'.format(self.mysqladmin))
-            raise RuntimeError('FAILED: {} does NOT exist'.format(self.mysqladmin))
-        else:
+        if os.path.exists(self.mysqladmin):
             logger.debug('OK: {} exists'.format(self.mysqladmin))
             return True
+        else:
+            logger.error('FAILED: {} does NOT exist'.format(self.mysqladmin))
+            raise RuntimeError('FAILED: {} does NOT exist'.format(self.mysqladmin))
 
     def check_mysql_backuptool(self):
-        if not os.path.exists(self.backup_tool):
-            logger.error('FAILED: XtraBackup does NOT exist')
-            raise RuntimeError('FAILED: XtraBackup does NOT exist')
-        else:
+        """
+        Method for checking if given backup tool path is there or not.
+        :return: RuntimeError on failure, True on success
+        """
+        if os.path.exists(self.backup_tool):
             logger.debug('OK: XtraBackup exists')
             return True
+        else:
+            logger.error('FAILED: XtraBackup does NOT exist')
+            raise RuntimeError('FAILED: XtraBackup does NOT exist')
 
     def check_mysql_backupdir(self):
-        '''
+        """
         Check for MySQL backup directory.
         If directory exists already then, return True. If not, try to create it.
-        :return: True on success.
-        '''
-        if not (os.path.exists(self.backupdir)):
-            logger.debug('Main backup directory does not exist')
-            logger.debug('Creating Main Backup folder...')
-            os.makedirs(self.backupdir)
-            logger.debug('OK: Created')
-            return True
-        else:
+        :return: True on success. RuntimeError on failure.
+        """
+        if os.path.exists(self.backupdir):
             logger.debug('OK: Main backup directory exists')
             return True
+        else:
+            logger.debug('Main backup directory does not exist')
+            logger.debug('Creating Main Backup folder...')
+            try:
+                os.makedirs(self.backupdir)
+                logger.debug('OK: Created')
+                return True
+            except Exception as err:
+                logger.error("FAILED: Could not create directory, ", err)
+                raise RuntimeError("FAILED: Could not create directory")
+
 
     def check_mysql_archive_dir(self):
         '''
         Check for archive directory.
-        If archive_dir is given in config file and if it is does not exist, try to create.
-        :return: True on success.
+        If archive_dir is given in config file and if it does not exist, try to create.
+        :return: True on success. RuntimeError on failure.
         '''
         if hasattr(self, 'archive_dir'):
-            if not (os.path.exists(self.archive_dir)):
-                logger.debug('Archive backup directory does not exist')
-                logger.debug('Creating archive folder...')
-                os.makedirs(self.archive_dir)
-                logger.debug('OK: Created')
-                return True
-            else:
+            if os.path.exists(self.archive_dir):
                 logger.debug('OK: Archive folder directory exists')
                 return True
+            else:
+                logger.debug('Archive backup directory does not exist')
+                logger.debug('Creating archive folder...')
+                try:
+                    os.makedirs(self.archive_dir)
+                    logger.debug('OK: Created')
+                    return True
+                except Exception as err:
+                    logger.error("FAILED: Could not create directory, ", err)
+                    raise RuntimeError("FAILED: Could not create directory")
         else:
             return True
 
@@ -154,31 +158,39 @@ def check_mysql_fullbackupdir(self):
         If this path exists return True if not try to create.
         :return: True on success.
         '''
-        if not (os.path.exists(self.full_dir)):
-            logger.debug('Full Backup directory does not exist')
-            logger.debug('Creating full backup directory...')
-            os.makedirs(self.full_dir)
-            logger.debug('OK: Created')
-            return True
-        else:
+        if os.path.exists(self.full_dir):
             logger.debug("OK: Full Backup directory exists")
             return True
+        else:
+            logger.debug('Full Backup directory does not exist')
+            logger.debug('Creating full backup directory...')
+            try:
+                os.makedirs(self.full_dir)
+                logger.debug('OK: Created')
+                return True
+            except Exception as err:
+                logger.error("FAILED: Could not create directory, ", err)
+                raise RuntimeError("FAILED: Could not create directory")
 
     def check_mysql_incbackupdir(self):
         '''
         Check incremental backup directory path.
         If this path exists return True if not try to create.
         :return: True on success.
         '''
-        if not (os.path.exists(self.inc_dir)):
-            logger.debug('Increment directory does not exist')
-            logger.debug('Creating increment backup directory...')
-            os.makedirs(self.inc_dir)
-            logger.debug('OK: Created')
-            return True
-        else:
+        if os.path.exists(self.inc_dir):
             logger.debug('OK: Increment directory exists')
             return True
+        else:
+            logger.debug('Increment directory does not exist')
+            logger.debug('Creating increment backup directory...')
+            try:
+                os.makedirs(self.inc_dir)
+                logger.debug('OK: Created')
+                return True
+            except Exception as err:
+                logger.error("FAILED: Could not create directory, ", err)
+                raise RuntimeError("FAILED: Could not create directory")
 
     def check_all_env(self):
         '''

general_conf/generalops.py

@@ -148,4 +148,4 @@ def __init__(self, config=path_config.config_path_file):
                     self.make_slaves = TEST['make_slaves']
 
         else:
-            logger.critical("Missing config file : /etc/autoxtrabackup.cnf")
+            logger.critical("Missing config file : {}".format(path_config.config_path_file))

prepare_env_test_mode/clone_build_start_server.py

@@ -61,7 +61,7 @@ def clone_ps_server_from_conf(self):
         return True
 
     def clone_pxb(self):
-        # Clone PXB
+        # Clone PXB from github based on branch values from config file.
         pxb_branches = self.pxb_branches.split()
         for branch in pxb_branches:
             clone_cmd = "git clone {} -b {} {}/PXB-{}"
@@ -86,10 +86,7 @@ def build_pxb(self):
         for branch in pxb_branches:
             pxb_path = "{}/PXB-{}".format(self.testpath, branch)
             os.chdir(pxb_path)
-            if '2.3' in branch:
-                build_cmd = "{}/build_{}_pxb.sh {} {}".format(dir_path, '2.3', self.testpath, branch)
-            elif '2.4' in branch:
-                build_cmd = "{}/build_{}_pxb.sh {} {}".format(dir_path, '2.4', self.testpath, branch)
+            build_cmd = "{}/build_pxb.sh {} {}".format(dir_path, self.testpath, branch)
             status, output = subprocess.getstatusoutput(build_cmd)
             if status == 0:
                 logger.debug("PXB build succeeded")
@@ -114,9 +111,11 @@ def build_server(self):
             if '5.5' in branch:
                 # Use same script with 5.5 and 5.6 versions
                 build_cmd = "{}/percona-qa/build_5.x_debug_5.6_for_pxb_tests.sh"
+            elif '8.0' in branch:
+                build_cmd = "{}/percona-qa/build_psms_debug.sh"
             else:
                 build_cmd = "{}/percona-qa/build_5.x_debug_{}_for_pxb_tests.sh"
-            logger.debug("Started to build Percon Server from source...")
+            logger.debug("Started to build Percona Server from source...")
             status, output = subprocess.getstatusoutput(build_cmd.format(self.testpath, branch))
             if status == 0:
                 logger.debug("PS build succeeded")
@@ -187,7 +186,7 @@ def prepare_startup(self, basedir_path):
     @staticmethod
     def prepare_start_dynamic(basedir_path):
         # Method for calling start_dynamic.sh from basedir.
-        # It will generated start_dynamuc executables in basedir.
+        # It will generate start_dynamic executables in basedir.
         saved_path = os.getcwd()
         dir_path = os.path.dirname(os.path.realpath(__file__))
         os.chdir(basedir_path)