Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Major refactoring.

  • Loading branch information...
commit 5a56fe9347b4c28972da15d6bc6a6cfe75a41a2e 1 parent 00d89ca
@robhudson authored
View
166 dumpy/base.py
@@ -0,0 +1,166 @@
+import ConfigParser
+import os
+
+from dumpy.importlib import import_module
+
+class ProcessorException(Exception):
+ pass
+
+class DumpyBase(object):
+ """
+ Overall base class for BackupBase and PostProcessBase.
+
+ Provides a few utility methods to subclasses.
+ """
+ def _get_option_value(self, config, section, option, type=None):
+ """
+ Tries to get the section and option from config, returning None
+ if not found. Convert to type if given.
+ """
+ if not type or type not in ['boolean', 'float', 'int', 'string']:
+ type = 'string'
+
+ value = None
+ try:
+ if type == 'boolean':
+ return config.getboolean(section, option)
+ elif type == 'float':
+ return config.getfloat(section, option)
+ elif type == 'int':
+ return config.getint(section, option)
+ elif type == 'string':
+ return config.get(section, option)
+ else:
+ return None
+ except ConfigParser.NoSectionError:
+ pass
+ except ConfigParser.NoOptionError:
+ pass
+
+ return value
+
+ def parse_config(self):
+ """
+ Subclasses parse their own config files since each will only need a
+ subsection of the config.
+
+ Example::
+
+ super(SubClass, self).parse_config()
+ options = {}
+ try:
+ for k, v in config.items('section'):
+ options[k] = v
+ except ConfigParser.NoSectionError:
+ pass # No section
+
+ Or using the _get_option_value method::
+
+ config = ConfigParser.SafeConfigParser()
+ config.read(os.path.expanduser('~/.dumpy.cfg'))
+
+ option1 = _get_option_value(config, 'section', 'option1')
+ option2 = _get_option_value(config, 'section', 'option1', 'boolean')
+
+ """
+ self.config = ConfigParser.SafeConfigParser()
+ self.config.read(os.path.expanduser('~/.dumpy.cfg'))
+
+class BackupBase(DumpyBase):
+ """
+ Base class for database backups.
+
+ Any subclass of BackupBase needs to implement the backup() method and
+ return a file-like object.
+ """
+ def __init__(self, db):
+ self.db = db
+
+ def backup(self):
+ raise NotImplementedError
+
+class PostProcessBase(DumpyBase):
+ """
+ Base class for post processing routines.
+
+ Any subclass of PostProcessBase needs to implement the process(file)
+ method, taking in a file-like object and returning a file-like object. The
+ process(file) method should return the file object passed in if unchanged.
+ """
+ def process(self, file):
+ raise NotImplementedError
+
+class DatabaseBackup(BackupBase):
+ """
+ This classes loads the config's type and passes of backup to the type's
+ class. (e.g. type == 'mysql' calls MysqlBackup().backup().)
+ """
+ def __init__(self, database):
+ self.database = database
+
+ def parse_config(self):
+ super(DatabaseBackup, self).parse_config()
+
+ section = 'database %s' % (self.database)
+ self.type = self._get_option_value(self.config, section, 'type')
+
+ def backup(self):
+ """
+ A sort of proxy method to call the appropriate database type's backup
+ method.
+ """
+ self.parse_config()
+ if self.type == 'mysql':
+ from dumpy.database import mysql
+ return mysql.MysqlBackup(self.database).backup()
+ elif self.type == 'postgresql':
+ from dumpy.database import postgresql
+ return postgresql.PostgresqlBackup(self.database).backup()
+
+class PostProcess(PostProcessBase):
+ """
+ This classes loads the specified database `postprocessing` config option
+ and passes off handling to each post processor.
+ """
+ def __init__(self, db):
+ self.db = db
+ self.builtin_processors = {
+ 'Bzip': 'dumpy.postprocessor.bzip.Bzip',
+ 'TimestampRename': 'dumpy.postprocessor.timestamp.TimestampRename',
+ 'FileSystemCopy': 'dumpy.postprocessor.fscopy.FileSystemCopy',
+ 'S3Copy': 'dumpy.postprocessor.s3copy.S3Copy',
+ }
+
+ def parse_config(self):
+ super(PostProcess, self).parse_config()
+
+ self.processors = self._get_option_value(self.config, 'database %s' % (self.db,), 'postprocessing')
+
+ def process(self, file):
+ self.parse_config()
+
+ if self.processors:
+ processors = [p.strip() for p in self.processors.split(',')]
+
+ for processor_path in processors:
+ if processor_path in self.builtin_processors.keys():
+ processor_path = self.builtin_processors.get(processor_path)
+
+ try:
+ dot = processor_path.rindex('.')
+ except ValueError:
+ raise ProcessorException, '%s isn\'t a processor module' % processor_path
+ pp_module, pp_classname = processor_path[:dot], processor_path[dot+1:]
+ try:
+ mod = import_module(pp_module)
+ except ImportError, e:
+ raise ProcessorException, 'Error importing processor %s: "%s"' % (pp_module, e)
+ try:
+ pp_class = getattr(mod, pp_classname)
+ except AttributeError:
+ raise ProcessorException, 'Processor module "%s" does not define a "%s" class' % (pp_module, pp_classname)
+
+ processor = pp_class(self.db)
+
+ file = processor.process(file)
+
View
0  dumpy/database/__init__.py
No changes.
View
53 dumpy/database/mysql.py
@@ -0,0 +1,53 @@
+import logging
+import os
+import tempfile
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class MySQLDumpError(Exception):
+ pass
+
+class MysqlBackup(dumpy.base.BackupBase):
+
+ def parse_config(self):
+ super(MysqlBackup, self).parse_config()
+
+ section = 'database %s' % (self.db)
+ self.name = self._get_option_value(self.config, section, 'name')
+ self.user = self._get_option_value(self.config, section, 'user')
+ self.password = self._get_option_value(self.config, section, 'password')
+ self.host = self._get_option_value(self.config, section, 'host')
+ self.port = self._get_option_value(self.config, section, 'port', 'int')
+
+ self.binary = self._get_option_value(self.config, 'mysqldump options', 'path')
+ self.flags = self._get_option_value(self.config, 'mysqldump options', 'flags')
+
+ def get_flags(self):
+ flags = '%s' % (self.flags)
+ if self.user:
+ flags += ' -u %s' % (self.user)
+ if self.password:
+ flags += ' -p%s' % (self.password)
+ if self.host:
+ flags += ' -h %s' % (self.host)
+ if self.port:
+ flags += ' -P %d' % (self.port)
+ return flags
+
+ def backup(self):
+ self.parse_config()
+ tmp_file = tempfile.NamedTemporaryFile()
+# try:
+ cmd = '%(binary)s %(flags)s %(database)s > %(file)s' % ({
+ 'binary': self.binary,
+ 'flags': self.get_flags(),
+ 'database': self.name,
+ 'file': tmp_file.name,
+ })
+ logger.info('%s - Command: %s' % (self.db, cmd))
+ os.system(cmd)
+
+ return tmp_file
+
View
49 dumpy/database/postgresql.py
@@ -0,0 +1,49 @@
+import logging
+import os
+import tempfile
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class PostgresqlBackup(dumpy.base.BackupBase):
+
+ def parse_config(self):
+ super(PostgresqlBackup, self).parse_config()
+
+ section = 'database %s' % self.db
+ self.name = self._get_option_value(self.config, section, 'name')
+ self.user = self._get_option_value(self.config, section, 'user')
+ self.host = self._get_option_value(self.config, section, 'host')
+ self.port = self._get_option_value(self.config, section, 'port', 'int')
+
+ self.binary = self._get_option_value(self.config, 'pg_dump options', 'path')
+ self.flags = self._get_option_value(self.config, 'pg_dump options', 'flags')
+
+ def get_flags(self):
+ # @@@ ugh. i don't like this.
+ if self.flags is None:
+ flags = ''
+ else:
+ flags = '%s' % self.flags
+ if self.user:
+ flags += ' -U %s' % self.user
+ if self.host:
+ flags += ' -h %s' % self.host
+ if self.port:
+ flags += ' -p %d' % self.port
+ return flags
+
+ def backup(self):
+ self.parse_config()
+ tmp_file = tempfile.NamedTemporaryFile()
+ cmd = '%(binary)s %(flags)s %(database)s > %(file)s' % {
+ 'binary': self.binary,
+ 'flags': self.get_flags(),
+ 'database': self.name,
+ 'file': tmp_file.name,
+ }
+ logger.info('%s - Command: %s' % (self.db, cmd))
+ os.system(cmd)
+ return tmp_file
+
View
442 dumpy/dumper.py
@@ -1,401 +1,49 @@
#!/usr/bin/env python
-import os
import ConfigParser
-import datetime
-import shutil
-import tempfile
-
-try:
- import boto
-except ImportError:
- boto = None
+import logging
+import optparse
+import os
+import sys
+
+from dumpy import base
+
+parser = optparse.OptionParser()
+parser.add_option("-D", "--database", dest="database",
+ help="Which database would you like to dump?", default='db1')
+parser.add_option("-v", "--verbose",
+ action="store_true", dest="verbose", default=False,
+ help="Output debugging information")
+parser.add_option("-a", "--all-databases",
+ action="store_true", dest="all", default=False,
+ help="Dump all databases")
+
+(options, args) = parser.parse_args()
+
+logger = logging.getLogger("dumper")
+logger.setLevel(logging.ERROR)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+if options.verbose:
+ logger.setLevel(logging.DEBUG)
+
+dbs_to_dump = []
+
+if options.all:
+ config = ConfigParser.SafeConfigParser()
+ config.read(os.path.expanduser('~/.dumpy.cfg'))
+ sections = config.sections()
+ for db in sections:
+ if db.startswith('database '):
+ dbname = db.replace('database ', '')
+ dbs_to_dump.append(dbname)
else:
- from boto.s3.key import Key
- from boto.s3.connection import S3Connection
-
-class MySQLDumpError(Exception):
- pass
-
-class DumpyBase(object):
- """
- Overall base class for BackupBase and PostProcessBase.
-
- Provides a few utility methods to subclasses.
- """
- def _get_option_value(self, config, section, option, type=None):
- """
- Tries to get the section and option from config, returning None
- if not found. Convert to type if given.
- """
- if not type or type not in ['boolean', 'float', 'int', 'string']:
- type = 'string'
-
- value = None
- try:
- if type == 'boolean':
- return config.getboolean(section, option)
- elif type == 'float':
- return config.getfloat(section, option)
- elif type == 'int':
- return config.getint(section, option)
- elif type == 'string':
- return config.get(section, option)
- else:
- return None
- except ConfigParser.NoSectionError:
- pass
- except ConfigParser.NoOptionError:
- pass
-
- return value
-
- def parse_config(self):
- """
- Subclasses parse their own config files since each will only need a
- subsection of the config.
-
- Example::
-
- super(SubClass, self).parse_config()
- options = {}
- try:
- for k, v in config.items('section'):
- options[k] = v
- except ConfigParser.NoSectionError:
- pass # No section
-
- Or using the _get_option_value method::
-
- config = ConfigParser.SafeConfigParser()
- config.read(os.path.expanduser('~/.dumpy.cfg'))
-
- option1 = _get_option_value(config, 'section', 'option1')
- option2 = _get_option_value(config, 'section', 'option1', 'boolean')
-
- """
- self.config = ConfigParser.SafeConfigParser()
- self.config.read(os.path.expanduser('~/.dumpy.cfg'))
-
-class BackupBase(DumpyBase):
- """
- Base class for database backups.
-
- Any subclass of BackupBase needs to implement the backup() method and
- return a file-like object.
- """
- def __init__(self, db):
- self.db = db
-
- def backup(self):
- raise NotImplementedError
-
-class PostProcessBase(DumpyBase):
- """
- Base class for post processing routines.
-
- Any subclass of PostProcessBase needs to implement the process(file)
- method, taking in a file-like object and returning a file-like object. The
- process(file) method should return the file object passed in if unchanged.
- """
- def process(self, file):
- raise NotImplementedError
-
-class DatabaseBackup(BackupBase):
- """
- This classes loads the config's type and passes of backup to the type's
- class. (e.g. type == 'mysql' calls MysqlBackup().backup().)
- """
- def __init__(self, database):
- self.database = database
-
- def parse_config(self):
- super(DatabaseBackup, self).parse_config()
-
- section = 'database %s' % (self.database)
- self.type = self._get_option_value(self.config, section, 'type')
-
- def backup(self):
- """
- A sort of proxy method to call the appropriate database type's backup
- method.
- """
- self.parse_config()
- if self.type == 'mysql':
- return MysqlBackup(self.database).backup()
- elif self.type == 'postgresql':
- return PostgresqlBackup(self.database).backup()
-
-class MysqlBackup(BackupBase):
-
- def parse_config(self):
- super(MysqlBackup, self).parse_config()
-
- section = 'database %s' % (self.db)
- self.name = self._get_option_value(self.config, section, 'name')
- self.user = self._get_option_value(self.config, section, 'user')
- self.password = self._get_option_value(self.config, section, 'password')
- self.host = self._get_option_value(self.config, section, 'host')
- self.port = self._get_option_value(self.config, section, 'port', 'int')
-
- self.binary = self._get_option_value(self.config, 'mysqldump options', 'path')
- self.flags = self._get_option_value(self.config, 'mysqldump options', 'flags')
-
- def get_flags(self):
- flags = '%s' % (self.flags)
- if self.user:
- flags += ' -u %s' % (self.user)
- if self.password:
- flags += ' -p%s' % (self.password)
- if self.host:
- flags += ' -h %s' % (self.host)
- if self.port:
- flags += ' -P %d' % (self.port)
- return flags
-
- def backup(self):
- self.parse_config()
- tmp_file = tempfile.NamedTemporaryFile()
-# try:
- cmd = '%(binary)s %(flags)s %(database)s > %(file)s' % ({
- 'binary': self.binary,
- 'flags': self.get_flags(),
- 'database': self.name,
- 'file': tmp_file.name,
- })
- logger.info('%s - Command: %s' % (self.db, cmd))
- os.system(cmd)
-
- return tmp_file
-
-class PostgresqlBackup(BackupBase):
-
- def parse_config(self):
- super(PostgresqlBackup, self).parse_config()
-
- section = 'database %s' % self.db
- self.name = self._get_option_value(self.config, section, 'name')
- self.user = self._get_option_value(self.config, section, 'user')
- self.host = self._get_option_value(self.config, section, 'host')
- self.port = self._get_option_value(self.config, section, 'port', 'int')
-
- self.binary = self._get_option_value(self.config, 'pg_dump options', 'path')
- self.flags = self._get_option_value(self.config, 'pg_dump options', 'flags')
-
- def get_flags(self):
- # @@@ ugh. i don't like this.
- if self.flags is None:
- flags = ''
- else:
- flags = '%s' % self.flags
- if self.user:
- flags += ' -U %s' % self.user
- if self.host:
- flags += ' -h %s' % self.host
- if self.port:
- flags += ' -p %d' % self.port
- return flags
-
- def backup(self):
- self.parse_config()
- tmp_file = tempfile.NamedTemporaryFile()
- cmd = '%(binary)s %(flags)s %(database)s > %(file)s' % {
- 'binary': self.binary,
- 'flags': self.get_flags(),
- 'database': self.name,
- 'file': tmp_file.name,
- }
- logger.info('%s - Command: %s' % (self.db, cmd))
- os.system(cmd)
- return tmp_file
-
-class PostProcess(PostProcessBase):
- """
- This classes loads the specified database `postprocessing` config option
- and passes off handling to each post processor.
- """
- def __init__(self, db):
- self.db = db
-
- def parse_config(self):
- super(PostProcess, self).parse_config()
-
- self.processors = self._get_option_value(self.config, 'database %s' % (self.db,), 'postprocessing')
-
- def process(self, file):
- self.parse_config()
-
- if self.processors:
- processors = [p.strip() for p in self.processors.split(',')]
-
- for processor in processors:
- file = globals()[processor](self.db).process(file)
-
-class Bzip(PostProcessBase):
- """
- A post processor that bzips the given file and returns it.
- """
- def __init__(self, db):
- self.db = db
-
- def parse_config(self):
- super(Bzip, self).parse_config()
- self.path = self._get_option_value(self.config, 'Bzip options', 'path')
-
- def process(self, file):
-
- self.parse_config()
-
- cmd = "%(path)s -f '%(file)s'" % ({'path': self.path, 'file': file.name})
- logger.info('%s - %s - Command: %s' % (self.db, self.__class__.__name__, cmd))
- os.system(cmd)
- new_file = open('%s.bz2' % (file.name))
- file.close()
- return new_file
-
-class TimestampRename(PostProcessBase):
- """
- A post procesor that renames the file using timestamp format.
- """
- def __init__(self, db):
- self.db = db
-
- def parse_config(self):
- super(TimestampRename, self).parse_config()
- self.format = self._get_option_value(self.config, 'TimestampRename options', 'format')
- self.insert_db_name = self._get_option_value(self.config, 'database %s' % (self.db), 'insert_db_name', 'boolean')
-
- def process(self, file):
-
- self.parse_config()
-
- dir = os.path.dirname(file.name)
- base, ext = os.path.splitext(os.path.basename(file.name))
- # @@@ Clint: Put DB name into renamed name, I dunno how i like this in here
- file_name_format = '%s/%s%s'
- if self.insert_db_name:
- file_name_format = '%s/'+self.db+'-%s%s'
- new_file_name = file_name_format % (dir, datetime.datetime.now().strftime(self.format), ext)
-
- shutil.copy(file.name, new_file_name)
- logger.info('%s - %s - Copying %s to %s' % (self.db, self.__class__.__name__, file.name, new_file_name))
- new_file = open(new_file_name)
- file.close()
- return new_file
-
-
-class SystemFileCopy(PostProcessBase):
- """
- A post processor that copies the file to a specified path.
- """
- def __init__(self, db):
- self.db = db
-
- def parse_config(self):
- super(SystemFileCopy, self).parse_config()
- self.dir = self._get_option_value(self.config, 'SystemFileCopy options', 'directory')
- override = self._get_option_value(self.config, 'database %s' % (self.db), 'SystemFileCopy directory')
- if override:
- self.dir = override
-
- def process(self, file):
- self.parse_config()
-
- dir = os.path.dirname(file.name)
- base, ext = os.path.splitext(os.path.basename(file.name))
- if self.dir.endswith('/'):
- self.dir = self.dir[0:-1]
- new_file_name = '%s/%s%s' % (self.dir, base, ext)
-
- shutil.copy(file.name, new_file_name)
- logger.info('%s - %s - Copying %s to %s' % (self.db, self.__class__.__name__, file.name, new_file_name))
- new_file = open(new_file_name)
- # TODO:
- # This should probably not return the new file but return the original.
- # In that way, any copy post processing results in a dead end but the
- # original file can still be used in more post processing.
- file.close()
- return new_file
-
-class S3Copy(PostProcessBase):
- """
- A post processor that copies the given file to S3.
- """
- def __init__(self, db):
- self.db = db
-
- def parse_config(self):
- super(S3Copy, self).parse_config()
- self.access_key = self._get_option_value(self.config, 'S3Copy options', 'access_key')
- self.secret_key = self._get_option_value(self.config, 'S3Copy options', 'secret_key')
- self.bucket = self._get_option_value(self.config, 'S3Copy options', 'bucket')
- self.prefix = self._get_option_value(self.config, 'S3Copy options', 'prefix')
-
- def process(self, file):
- if boto is None:
- raise Exception("You must have boto installed before using S3 support.")
-
- self.parse_config()
-
- conn = S3Connection(self.access_key, self.secret_key)
- bucket = conn.create_bucket(self.bucket)
- k = Key(bucket)
- if self.prefix:
- keyname = '%s%s%s' % (
- self.prefix,
- self.prefix.endswith('/') and '' or '/',
- os.path.basename(file.name)
- )
- else:
- keyname = os.path.basename(file.name)
- k.key = keyname
- k.set_contents_from_file(file)
-
- logger.info('%s - %s - Copying to S3 with key name: %s' % (self.db, self.__class__.__name__, keyname))
-
- return file
-
-if __name__ == '__main__':
-
- import logging
- import optparse
-
- parser = optparse.OptionParser()
- parser.add_option("-D", "--database", dest="database",
- help="Which database would you like to dump?", default='db1')
- parser.add_option("-v", "--verbose",
- action="store_true", dest="verbose", default=False,
- help="Output debugging information")
- parser.add_option("-a", "--all-databases",
- action="store_true", dest="all", default=False,
- help="Dump all databases")
-
- (options, args) = parser.parse_args()
-
- logger = logging.getLogger("dumper")
- logger.setLevel(logging.ERROR)
- ch = logging.StreamHandler()
- ch.setLevel(logging.DEBUG)
- formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
- ch.setFormatter(formatter)
- logger.addHandler(ch)
-
- if options.verbose:
- logger.setLevel(logging.DEBUG)
-
- dbs_to_dump = []
-
- if options.all:
- config = ConfigParser.SafeConfigParser()
- config.read(os.path.expanduser('~/.dumpy.cfg'))
- sections = config.sections()
- for db in sections:
- if db.startswith('database '):
- dbname = db.replace('database ', '')
- dbs_to_dump.append(dbname)
- else:
- dbs_to_dump.append(options.database)
+ dbs_to_dump.append(options.database)
- for db in dbs_to_dump:
- file = DatabaseBackup(db).backup()
- # Then call post processors, in the given order
- file = PostProcess(db).process(file)
+for db in dbs_to_dump:
+ file = base.DatabaseBackup(db).backup()
+ # Then call post processors, in the given order
+ file = base.PostProcess(db).process(file)
View
35 dumpy/importlib.py
@@ -0,0 +1,35 @@
+import sys
+
+def _resolve_name(name, package, level):
+ """Return the absolute name of the module to be imported."""
+ if not hasattr(package, 'rindex'):
+ raise ValueError("'package' not set to a string")
+ dot = len(package)
+ for x in xrange(level, 1, -1):
+ try:
+ dot = package.rindex('.', 0, dot)
+ except ValueError:
+ raise ValueError("attempted relative import beyond top-level "
+ "package")
+ return "%s.%s" % (package[:dot], name)
+
+
+def import_module(name, package=None):
+ """Import a module.
+
+ The 'package' argument is required when performing a relative import. It
+ specifies the package to use as the anchor point from which to resolve the
+ relative import to an absolute import.
+
+ """
+ if name.startswith('.'):
+ if not package:
+ raise TypeError("relative imports require the 'package' argument")
+ level = 0
+ for character in name:
+ if character != '.':
+ break
+ level += 1
+ name = _resolve_name(name[level:], package, level)
+ __import__(name)
+ return sys.modules[name]
View
0  dumpy/postprocessor/__init__.py
No changes.
View
29 dumpy/postprocessor/bzip.py
@@ -0,0 +1,29 @@
+import logging
+import os
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class Bzip(dumpy.base.PostProcessBase):
+ """
+ A post processor that bzips the given file and returns it.
+ """
+ def __init__(self, db):
+ self.db = db
+
+ def parse_config(self):
+ super(Bzip, self).parse_config()
+ self.path = self._get_option_value(self.config, 'Bzip options', 'path')
+
+ def process(self, file):
+
+ self.parse_config()
+
+ cmd = "%(path)s -f '%(file)s'" % ({'path': self.path, 'file': file.name})
+ logger.info('%s - %s - Command: %s' % (self.db, self.__class__.__name__, cmd))
+ os.system(cmd)
+ new_file = open('%s.bz2' % (file.name))
+ file.close()
+ return new_file
+
View
41 dumpy/postprocessor/fscopy.py
@@ -0,0 +1,41 @@
+import logging
+import os
+import shutil
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class FileSystemCopy(dumpy.base.PostProcessBase):
+ """
+ A post processor that copies the file to a specified path.
+ """
+ def __init__(self, db):
+ self.db = db
+
+ def parse_config(self):
+ super(FileSystemCopy, self).parse_config()
+ self.dir = self._get_option_value(self.config, 'FileSystemCopy options', 'directory')
+ override = self._get_option_value(self.config, 'database %s' % (self.db), 'FileSystemCopy directory')
+ if override:
+ self.dir = override
+
+ def process(self, file):
+ self.parse_config()
+
+ dir = os.path.dirname(file.name)
+ base, ext = os.path.splitext(os.path.basename(file.name))
+ if self.dir.endswith('/'):
+ self.dir = self.dir[0:-1]
+ new_file_name = '%s/%s%s' % (self.dir, base, ext)
+
+ shutil.copy(file.name, new_file_name)
+ logger.info('%s - %s - Copying %s to %s' % (self.db, self.__class__.__name__, file.name, new_file_name))
+ new_file = open(new_file_name)
+ # TODO:
+ # This should probably not return the new file but return the original.
+ # In that way, any copy post processing results in a dead end but the
+ # original file can still be used in more post processing.
+ file.close()
+ return new_file
+
View
53 dumpy/postprocessor/s3copy.py
@@ -0,0 +1,53 @@
+import logging
+import os
+
+try:
+ import boto
+except ImportError:
+ boto = None
+else:
+ from boto.s3.key import Key
+ from boto.s3.connection import S3Connection
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class S3Copy(dumpy.base.PostProcessBase):
+ """
+ A post processor that copies the given file to S3.
+ """
+ def __init__(self, db):
+ self.db = db
+
+ def parse_config(self):
+ super(S3Copy, self).parse_config()
+ self.access_key = self._get_option_value(self.config, 'S3Copy options', 'access_key')
+ self.secret_key = self._get_option_value(self.config, 'S3Copy options', 'secret_key')
+ self.bucket = self._get_option_value(self.config, 'S3Copy options', 'bucket')
+ self.prefix = self._get_option_value(self.config, 'S3Copy options', 'prefix')
+
+ def process(self, file):
+ if boto is None:
+ raise Exception("You must have boto installed before using S3 support.")
+
+ self.parse_config()
+
+ conn = S3Connection(self.access_key, self.secret_key)
+ bucket = conn.create_bucket(self.bucket)
+ k = Key(bucket)
+ if self.prefix:
+ keyname = '%s%s%s' % (
+ self.prefix,
+ self.prefix.endswith('/') and '' or '/',
+ os.path.basename(file.name)
+ )
+ else:
+ keyname = os.path.basename(file.name)
+ k.key = keyname
+ k.set_contents_from_file(file)
+
+ logger.info('%s - %s - Copying to S3 with key name: %s' % (self.db, self.__class__.__name__, keyname))
+
+ return file
+
View
39 dumpy/postprocessor/timestamp.py
@@ -0,0 +1,39 @@
+import datetime
+import logging
+import os
+import shutil
+
+import dumpy
+
+logger = logging.getLogger("dumper")
+
+class TimestampRename(dumpy.base.PostProcessBase):
+ """
+ A post procesor that renames the file using timestamp format.
+ """
+ def __init__(self, db):
+ self.db = db
+
+ def parse_config(self):
+ super(TimestampRename, self).parse_config()
+ self.format = self._get_option_value(self.config, 'TimestampRename options', 'format')
+ self.insert_db_name = self._get_option_value(self.config, 'database %s' % (self.db), 'insert_db_name', 'boolean')
+
+ def process(self, file):
+
+ self.parse_config()
+
+ dir = os.path.dirname(file.name)
+ base, ext = os.path.splitext(os.path.basename(file.name))
+ # @@@ Clint: Put DB name into renamed name, I dunno how i like this in here
+ file_name_format = '%s/%s%s'
+ if self.insert_db_name:
+ file_name_format = '%s/'+self.db+'-%s%s'
+ new_file_name = file_name_format % (dir, datetime.datetime.now().strftime(self.format), ext)
+
+ shutil.copy(file.name, new_file_name)
+ logger.info('%s - %s - Copying %s to %s' % (self.db, self.__class__.__name__, file.name, new_file_name))
+ new_file = open(new_file_name)
+ file.close()
+ return new_file
+
Please sign in to comment.
Something went wrong with that request. Please try again.