xtrabackup plugin revamp

General code cleanup and fixed several  cases where errors were
not being trapped correctly.

plugins/holland.backup.xtrabackup/holland/backup/xtrabackup/mysql.py

@@ -0,0 +1,62 @@
+"""
+holland.backup.xtrabackup.mysql
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Simple mysql client wrapper
+"""
+import MySQLdb
+
+class MySQL(object):
+    MySQLError = MySQLdb.MySQLError
+
+    def __init__(self, *args, **kwargs):
+        self._connection = MySQLdb.connect(*args, **kwargs)
+
+    def execute(self, sql, *args):
+        cursor = self.cursor()
+        try:
+            return cursor.execute(sql, args)
+        finally:
+            cursor.close()
+
+    def scalar(self, sql, *args):
+        cursor = self.cursor()
+        try:
+            if cursor.execute(sql, args):
+                return cursor.fetchone()[0]
+            else:
+                return None
+        finally:
+            cursor.close()
+
+    def first(self, sql, *args):
+        cursor = self.cursor()
+        try:
+            cursor.execute(sql, args)
+            return cursor.fetchone()
+        finally:
+            cursor.close()
+
+    def cursor(self):
+        return self._connection.cursor()
+
+    def from_defaults(cls, defaults_file):
+        return cls(read_default_file=defaults_file)
+    from_defaults = classmethod(from_defaults)
+
+    def var(self, var, scope='SESSION'):
+        scope = scope.upper()
+        if scope not in ('SESSION', 'GLOBAL'):
+            raise BackupError("Invalid variable scope used")
+        var = var.replace('%', '\\%').replace('_', '\\_')
+        sql = "SHOW %s VARIABLES LIKE '%s'" % (scope, var)
+        try:
+            return self.first(sql)[1]
+        except IndexError:
+            return None
+
+    def close(self):
+        try:
+            return self._connection.close()
+        finally:
+            self._connection = None

plugins/holland.backup.xtrabackup/holland/backup/xtrabackup/plugin.py

@@ -1,39 +1,38 @@
-"""holland backup plugin using xtrabackup"""
+"""
+holland.mysql.xtrabackup
+~~~~~~~~~~~~~~~~~~~~~~~
 
-import os, sys
-import shutil
+Xtrabackup backup strategy plugin
+"""
+
+import sys
 import logging
-import tempfile
-from subprocess import list2cmdline, check_call, CalledProcessError, STDOUT
-from holland.core.exceptions import BackupError
+from os.path import join
+from holland.core.backup import BackupError
 from holland.core.util.path import directory_size
 from holland.lib.compression import open_stream
-from holland.lib.mysql.option import build_mysql_config, write_options
-from holland.lib.mysql.client import connect, MySQLError
-from holland.backup.xtrabackup.util import xtrabackup_version, \
-                                           get_stream_method, \
-                                           resolve_template, \
-                                           run_pre_command
+from holland.backup.xtrabackup.mysql import MySQL
+from holland.backup.xtrabackup import util
 
 LOG = logging.getLogger(__name__)
 
 CONFIGSPEC = """
 [xtrabackup]
-global-defaults = string(default='/etc/my.cnf')
-innobackupex    = string(default='innobackupex-1.5.1')
-ibbackup        = string(default=None)
-stream          = option(yes,no,tar,xbstream,default=tar)
-slave-info      = boolean(default=no)
-safe-slave-backup = boolean(default=no)
-no-lock         = boolean(default=no)
-tmpdir          = string(default=None)
-additional-options = force_list(default=list())
-pre-command     = string(default=None)
+global-defaults     = string(default='/etc/my.cnf')
+innobackupex        = string(default='innobackupex-1.5.1')
+ibbackup            = string(default=None)
+stream              = option(yes,no,tar,xbstream,default=tar)
+slave-info          = boolean(default=no)
+safe-slave-backup   = boolean(default=no)
+no-lock             = boolean(default=no)
+tmpdir              = string(default="{backup_directory}")
+additional-options  = force_list(default=list())
+pre-command         = string(default=None)
 
 [compression]
-method          = option('none', 'gzip', 'pigz', 'bzip2', 'pbzip2', 'lzma', 'lzop', default='gzip')
-inline          = boolean(default=yes)
-level           = integer(min=0, max=9, default=1)
+method              = option('none', 'gzip', 'pigz', 'bzip2', 'pbzip2', 'lzma', 'lzop', default=gzip)
+inline              = boolean(default=yes)
+level               = integer(min=0, max=9, default=1)
 
 [mysql:client]
 defaults-extra-file = force_list(default=list('~/.my.cnf'))
@@ -45,9 +44,11 @@
 """.splitlines()
 
 class XtrabackupPlugin(object):
-    """This plugin provides support for backing up a MySQL database using
-    xtrabackup from Percona
-    """
+    #: control connection to mysql server
+    mysql = None
+
+    #: path to the my.cnf generated by this plugin
+    defaults_path = None
 
     def __init__(self, name, config, target_directory, dry_run=False):
         self.name = name
@@ -56,105 +57,110 @@ def __init__(self, name, config, target_directory, dry_run=False):
         self.target_directory = target_directory
         self.dry_run = dry_run
 
+        defaults_path = join(self.target_directory, 'my.cnf')
+        client_opts = self.config['mysql:client']
+        includes = [self.config['xtrabackup']['global-defaults']] + \
+                   client_opts['defaults-extra-file']
+        util.generate_defaults_file(defaults_path, includes, client_opts)
+        self.defaults_path = defaults_path
+
     def estimate_backup_size(self):
-        """Estimate the size of the backup this plugin will produce"""
         try:
-            mysql_config = build_mysql_config(self.config['mysql:client'])
-            client = connect(mysql_config['client'])
-            datadir = client.show_variable('datadir')
-            return directory_size(datadir)
-        except MySQLError, exc:
-            raise BackupError("Failed to lookup the MySQL datadir when "
-                              "estimating backup size: [%d] %s" % exc.args)
-
-    def backup(self):
-        """Run a database backup with xtrabackup"""
-        defaults_file = os.path.join(self.target_directory, 'my.xtrabackup.cnf')
-        args = [
-            self.config['xtrabackup']['innobackupex'],
-            '--defaults-file=%s' % defaults_file,
-        ]
+            client = MySQL.from_defaults(self.defaults_path)
+        except MySQL.MySQLError, exc:
+            raise BackupError('Failed to connect to MySQL [%d] %s' % exc.args)
+        try:
+            try:
+                datadir = client.var('datadir')
+                return directory_size(datadir)
+            except MySQL.MySQLError, exc:
+                raise BackupError("Failed to find mysql datadir: [%d] %s" %
+                                  exc.args)
+            except OSError, exc:
+                raise BackupError('Failed to calculate directory size: [%d] %s'
+                                  % (exc.errno, exc.strerror))
+        finally:
+            client.close()
 
-        if self.config['xtrabackup']['ibbackup']:
-            args.append('--ibbackup=' + self.config['xtrabackup']['ibbackup'])
+    def open_xb_logfile(self):
+        """Open a file object to the log output for xtrabackup"""
+        path = join(self.target_directory, 'xtrabackup.log')
+        try:
+            return open(path, 'a')
+        except IOError, exc:
+            raise BackupError('[%d] %s' % (exc.errno, exc.strerror))
 
+    def open_xb_stdout(self):
+        """Open the stdout output for a streaming xtrabackup run"""
+        config = self.config['xtrabackup']
         backup_directory = self.target_directory
-        stream_method = get_stream_method(self.config['xtrabackup']['stream'])
-        if stream_method:
-            args.append('--stream=' + stream_method)
+        if config['stream'] in ('tar', 'tar4ibd', 'xbstream'):
+            # XXX: bounce through compression
+            if 'tar' in config['stream']:
+                archive_path = join(backup_directory, 'backup.tar')
+                zconfig = self.config['compression']
+                return open_stream(archive_path, 'w',
+                                        method=zconfig['method'],
+                                        level=zconfig['level'])
+            elif 'xbstream' in config['stream']:
+                archive_path = join(backup_directory, 'backup.xb')
+                return open(archive_path, 'w')
         else:
-            backup_directory = os.path.join(backup_directory, 'data')
-            args.append('--no-timestamp')
-        if self.config['xtrabackup']['tmpdir']:
-            args.append('--tmpdir=' + self.config['xtrabackup']['tmpdir'])
-        if self.config['xtrabackup']['slave-info']:
-            args.append('--slave-info')
-        if self.config['xtrabackup']['safe-slave-backup']:
-            args.append('--safe-slave-backup')
-        if self.config['xtrabackup']['no-lock']:
-            args.append('--no-lock')
-        if self.config['xtrabackup']['additional-options']:
-            args.extend(self.config['xtrabackup']['additional-options'])
-        args.append(backup_directory)
-
-
-        if self.config['xtrabackup']['pre-command']:
-            cmd = resolve_template(self.config['xtrabackup']['pre-command'],
-                                   backupdir=self.target_directory)
-            if self.dry_run:
-                LOG.info("Skipping pre-command in dry-run mode: %s", cmd)
-            else:
-                run_pre_command(cmd)
-
-        LOG.info("%s", list2cmdline(args))
-        if self.dry_run:
-            return
+            return open('/dev/null', 'w')
 
 
-        config = build_mysql_config(self.config['mysql:client'])
-        write_options(config, defaults_file)
-        shutil.copyfileobj(open(self.config['xtrabackup']['global-defaults'], 'r'),
-                           open(defaults_file, 'a'))
-
-        if stream_method:
-            stdout_path = os.path.join(self.target_directory, 'backup.tar')
-            stdout = open_stream(stdout_path, 'w',
-                                 **self.config['compression'])
-            stderr_path = os.path.join(self.target_directory, 'xtrabackup.log')
-            stderr = open(stderr_path, 'wb')
-        else:
-            stdout_path = os.path.join(self.target_directory, 'xtrabackup.log')
-            stderr_path = stdout_path
-            stdout = open(stdout_path, 'wb')
-            stderr = STDOUT
-
+    def dryrun(self):
+        from subprocess import Popen, list2cmdline, PIPE, STDOUT
+        xb_cfg = self.config['xtrabackup']
+        args = util.build_xb_args(xb_cfg, self.target_directory,
+                self.defaults_path)
+        LOG.info("* xtrabackup command: %s", list2cmdline(args))
+        args = [
+            'xtrabackup',
+            '--defaults-file=' + self.defaults_path,
+            '--help'
+        ]
+        cmdline = list2cmdline(args)
+        LOG.info("* Running xtrabackup --help to verify %s is valid",
+                self.defaults_path)
         try:
+            process = Popen(args, stdout=PIPE, stderr=STDOUT, close_fds=True)
+        except OSError, exc:
+            raise BackupError("Failed to find xtrabackup binary")
+        stdout = process.stdout.read()
+        process.wait()
+        # Note: xtrabackup --help will exit with 1 usually
+        if process.returncode != 1:
+            LOG.error("! %s failed. Output follows below.", cmdline)
+            for line in stdout.splitlines():
+                LOG.error("! %s", line)
+            raise BackupError("%s exited with failure status [%d]" %
+                              (cmdline, process.returncode))
+    def backup(self):
+        if self.dry_run:
+            self.dryrun()
+            return
+        xb_cfg = self.config['xtrabackup']
+        backup_directory = self.target_directory
+        args = util.build_xb_args(xb_cfg, backup_directory, self.defaults_path)
+        util.execute_pre_command(xb_cfg['pre-command'])
+        stderr = self.open_xb_logfile()
+        try:
+            stdout = self.open_xb_stdout()
+            exc = None
             try:
-                check_call(args,
-                           stdout=stdout,
-                           stderr=stderr,
-                           close_fds=True)
-            except OSError, exc:
-                LOG.info("Command not found: %s", args[0])
-                raise BackupError("%s not found. Is xtrabackup installed?" %
-                                  args[0])
-            except CalledProcessError, exc:
-                LOG.info("%s failed", list2cmdline(exc.cmd))
-                for line in open(stderr_path, 'rb'):
-                    if line.startswith('>>'):
-                        continue
-                    LOG.error("%s", line.rstrip())
-                raise BackupError("%s failed" % exc.cmd[0])
+                try:
+                    util.run_xtrabackup(args, stdout, stderr)
+                except Exception, exc:
+                    raise
+            finally:
+                try:
+                    stdout.close()
+                except IOError, e:
+                    LOG.error("Error when closing %s: %s", stdout.name, e)
+                    if exc is None:
+                        raise
         finally:
-            exc_info = sys.exc_info()[1]
-            try:
-                stdout.close()
-                if stderr != STDOUT:
-                    stderr.close()
-            except IOError, exc:
-                if not exc_info:
-                    raise BackupError(str(exc))
-
-    def info(self):
-        """Provide information about the backup this plugin produced"""
-        return ""
+            stderr.close()
+        util.apply_xtrabackup_logfile(xb_cfg, args[-1])
+