Permalink
Browse files

Lots of refactoring.

Let each subclass handle its own config processing.  Moving towards chainable post processors for things like file renaming with timestamp, file copy, bzip output, copy to S3, etc.
  • Loading branch information...
1 parent e5fc365 commit 4d1714b71c5dc461cac66504deeb3e1f5689e3ff @robhudson committed Apr 7, 2009
Showing with 182 additions and 95 deletions.
  1. +182 −95 dumpy/dumper.py
View
@@ -1,4 +1,5 @@
import os
+import ConfigParser
import tempfile
from boto.s3.key import Key
@@ -8,59 +9,195 @@ class MySQLDumpError(Exception):
pass
-class Backup(object):
+class DumpyBase(object):
+ """
+ Overall base class for BackupBase and PostProcessBase.
+
+ Provides a few utility methods to subclasses.
+ """
+ def _get_option_value(self, config, section, option, type=None):
+ """
+ Tries to get the section and option from config, returning None
+ if not found. Convert to type if given.
+ """
+ if not type or type not in ['boolean', 'float', 'int', 'string']:
+ type = 'string'
+
+ value = None
+ try:
+ if type == 'boolean':
+ return config.getboolean(section, option)
+ elif type == 'float':
+ return config.getfloat(section, option)
+ elif type == 'int':
+ return config.getint(section, option)
+ elif type == 'string':
+ return config.get(section, option)
+ else:
+ return None
+ except ConfigParser.NoSectionError:
+ pass
+ except ConfigParser.NoOptionError:
+ pass
+
+ return value
+
+ def parse_config(self):
+ """
+ Subclasses parse their own config files since each will only need a
+ subsection of the config.
+
+ Example::
+
+ super(SubClass, self).parse_config()
+ options = {}
+ try:
+ for k, v in config.items('section'):
+ options[k] = v
+ except ConfigParser.NoSectionError:
+ pass # No section
+
+ Or using the _get_option_value method::
+
+ config = ConfigParser.SafeConfigParser()
+ config.read(os.path.expanduser('~/.dumpy.cfg'))
+
+ option1 = _get_option_value(config, 'section', 'option1')
+ option2 = _get_option_value(config, 'section', 'option1', 'boolean')
+
+ """
+ self.config = ConfigParser.SafeConfigParser()
+ self.config.read(os.path.expanduser('~/.dumpy.cfg'))
+
+class BackupBase(DumpyBase):
+ """
+ Base class for database backups.
+ """
def backup(self):
raise NotImplementedError
-class MysqlBackup(Backup):
- def __init__(self, database, username, password, host=None, binary=None, flags=None, bzip=False):
+class PostProcessBase(DumpyBase):
+ """
+ Base class for post processing routines.
+ """
+ def process(self):
+ raise NotImplementedError
+
+class DatabaseBackup(BackupBase):
+ """
+ This classes loads the config's type and passes of backup to the type's
+ class. (e.g. type == 'mysql' calls MysqlBackup().backup().)
+ """
+ def __init__(self, database):
self.database = database
- self.username = username
- self.password = password
- self.host = host and host or 'localhost'
- if binary:
- self.binary = binary
- else:
- pass # Search for mysqldump?
- self.flags = flags
- self.bzip = bzip
+
+ def parse_config(self):
+ super(DatabaseBackup, self).parse_config()
+
+ section = 'database %s' % (self.database)
+ self.type = self._get_option_value(self.config, section, 'type')
def backup(self):
+ """
+ A sort of proxy method to call the appropriate database type's backup
+ method.
+ """
+ self.parse_config()
+ if self.type == 'mysql':
+ return MysqlBackup(self.database).backup()
+
+class MysqlBackup(BackupBase):
+
+ def __init__(self, database):
+ self.database = database
+
+ def parse_config(self):
+ super(MysqlBackup, self).parse_config()
+
+ section = 'database %s' % (self.database)
+ self.user = self._get_option_value(self.config, section, 'user')
+ self.password = self._get_option_value(self.config, section, 'password')
+ self.host = self._get_option_value(self.config, section, 'host')
+ self.port = self._get_option_value(self.config, section, 'port', 'int')
+
+ self.binary = self._get_option_value(self.config, 'mysqldump options', 'path')
+ self.flags = self._get_option_value(self.config, 'mysqldump options', 'flags')
+
+ def get_flags(self):
+ flags = '%s' % (self.flags)
+ if self.user:
+ flags += ' -u %s' % (self.user)
+ if self.password:
+ flags += ' -p%s' % (self.password)
+ if self.host:
+ flags += ' -h %s' % (self.host)
+ if self.port:
+ flags += ' -P %d' % (self.port)
+ return flags
+
+ def backup(self):
+ self.parse_config()
tmp_file = tempfile.NamedTemporaryFile()
# try:
- cmd = '%(binary)s %(flags)s %(database)s %(bzip)s > %(file)s' % ({
+ cmd = '%(binary)s %(flags)s %(database)s > %(file)s' % ({
'binary': self.binary,
'flags': self.get_flags(),
'database': self.database,
'file': tmp_file.name,
- 'bzip': self.bzip and '|bzip2' or '',
})
- print cmd
+ print cmd #FIXME
os.system(cmd)
-# except:
-# print cmd
-# raise MySQLDumpError, "Mysqldump command failed!"
-
- # Return temp file?
- # Then we can copy it somewhere? or push it up to S3?
- # Close it? Can OS read it if it's already opened?
- # Maybe just return name to be opened by other methods?
+
return tmp_file
- def get_flags(self):
- flags = '%s' % (self.flags)
- flags += ' -u %s' % (self.username)
- flags += ' -p%s' % (self.password)
- flags += ' -h %s' % (self.host)
- return flags
+class PostProcess(PostProcessBase):
+ """
+ This classes loads the specified database `postprocessing` config option
+ and passes off handling to each post processor.
+ """
+ def __init__(self, database):
+ self.database = database
+
+ def parse_config(self):
+ super(PostProcess, self).parse_config()
+
+ self.processors = self._get_option_value(self.config, 'database %s' % (self.database), 'postprocessing')
+
+ def process(self, file):
+ self.parse_config()
+ processors = [p.strip() for p in self.processors.split(',')]
+
+ for processor in processors:
+ print processor
+
+class Bzip(PostProcessBase):
+ """
+ A post processor that bzips the given file and returns it.
+ """
+ def __init__(self, file):
+ self.file = file
-class S3Backup(Backup):
- def __init__(self, access_key, secret_key, bucket, filename):
- self.access_key = access_key
- self.secret_key = secret_key
- self.bucket = bucket
- self.filename = filename
- self.is_connected = False
+ def parse_config(self):
+ super(Bzip, self).parse_config()
+
+ self.path = self._get_option_value(self.config, 'Bzip options', 'path')
+
+ def process(self):
+ cmd = "%(path)s %(file)s"
+ file = open('%s.bz2' % (self.file.name))
+ self.file.close()
+ return file
+
+class SystemFileCopy(PostProcessBase):
+ """
+ A post processor that copies the file to a specified path.
+ """
+ pass
+
+class S3Copy(PostProcessBase):
+ """
+ A post processor that copies the given file to S3.
+ """
def s3_connect(self):
try:
@@ -91,63 +228,13 @@ def backup(self):
return
if __name__ == '__main__':
- # If called as a script directly, parse config and run...
- import ConfigParser
-
- config = ConfigParser.SafeConfigParser()
- config.read(os.path.expanduser('~/.dumpy.conf'))
-
- aws_s3 = {}
- try:
- for k, v in config.items('aws_s3'):
- aws_s3[k] = v
- except ConfigParser.NoSectionError:
- pass # No AWS S3 options set
-
- databases = {}
- try:
- for k, v in config.items('databases'):
- databases[k] = [db.strip() for db in v.split(',')]
- except ConfigParser.NoSectionError:
- pass # No databases
-
- for db_type, db_list in databases.iteritems():
-
- if db_type == 'mysql':
- mysql_options = {}
- try:
- for k, v in config.items('mysqldump options'):
- mysql_options[k] = v
- # Convert bzip to boolean
- mysql_options['bzip'] = config.getboolean('mysqldump options', 'bzip')
- except ConfigParser.NoSectionError:
- pass
-
- for db in db_list:
-
- print "Performing %s dump for: %s" % (db_type, db)
-
- section = 'mysql %s' % (db)
- host = None
- s3_copy = False
- try:
- username = config.get(section, 'user')
- password = config.get(section, 'pass')
- if config.has_option(section, 'host'):
- host = config.get(section, 'host')
- if config.has_option(section, 's3_copy'):
- s3_copy = config.getboolean(section, 's3_copy')
- except ConfigParser.NoOptionError:
- pass
-
- backup = MysqlBackup(db, username, password, host,
- binary=mysql_options['path'],
- flags=mysql_options['flags'],
- bzip=mysql_options['bzip']
- )
- tmp_file = backup.backup()
- if s3_copy and aws_s3:
- print "Copying file '%s' to S3" % (tmp_file.name)
- #s3backup = S3Backup(aws_s3['key'], aws_s3['secret'], aws_s3['bucket'], tmp_file.name)
- #s3backup.backup()
+
+ # Process options:
+ # dumpy --database [database name]
+
+ # Call DatabaseBackup first
+ file = DatabaseBackup('test_db').backup()
+
+ # Then call post processors, in the given order
+ PostProcess('test_db').process(file)

0 comments on commit 4d1714b

Please sign in to comment.