Skip to content
Browse files

File compression is now optional for postgres dumps

The script will also create dump filename which reflect the dump format
by extension (.tar, .sql or .c)
  • Loading branch information...
1 parent bd32e83 commit 2b1e6fb9937917fd7cf4a398bfe77aecf2fbf7de @exhuma committed May 11, 2011
Showing with 88 additions and 14 deletions.
  1. +2 −2 docs/source/conf.py
  2. +84 −10 pickup/generator_profile/postgres.py
  3. +1 −1 pickup/pickup.py
  4. +1 −1 setup.py
View
4 docs/source/conf.py
@@ -50,9 +50,9 @@
# built documents.
#
# The short X.Y version.
-version = '1.3'
+version = '1.4'
# The full version, including alpha/beta/rc tags.
-release = '1.3'
+release = '1.4'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
View
94 pickup/generator_profile/postgres.py
@@ -22,14 +22,27 @@
The port on which the database is running
**database**
- Tihs can be either a list of database names to backup, or simply one
+ This can be either a list of database names to backup, or simply one
database name to backup. This can also be ``'*'`` to backup all databases
(excluding ``template0``, ``template1`` and ``postgres``)
.. note:: In order for the wildcard ``"*"`` to work in the config file,
the user must be able to connect to "template1" and must have
read access to the system table "pg_database".
+ **compress_command**
+ If specified and non-empty, this command is used to compress the data.
+ The command will receive the data as standard input via a pipe. So it
+ must support this (gzip and bzip2 come to mind...).
+
+ As the command is used as first parameter to Popen, it must be specified
+ as list!
+
+ Examples:
+
+ * ``['gzip']``
+ * ``['gzip', '-5']``
+
**ignore_dbs**
A list of databases to ignore (mostly useful when using ``'*'`` as
database source.
@@ -70,6 +83,7 @@
host = 'localhost',
user = 'backup',
database = '*', # using '*' will dump all dbs
+ compress_command = ['gzip'],
ignore_dbs = ['my_test_db'],
port = 5432,
pg_dump_params = "-Ft -c",
@@ -127,10 +141,15 @@
CONFIG = {}
SOURCE = {}
+FORMAT_PLAIN = 0
+FORMAT_TAR = 1
+FORMAT_CUSTOM = 2
+
def init(source):
CONFIG.update(source['config'])
SOURCE.update(source)
CONFIG.setdefault('ignore_dbs', [])
+ CONFIG.setdefault('compress_command', [])
LOG.debug("Initialised '%s' with %r" % ( __name__, CONFIG))
@@ -168,24 +187,79 @@ def get_params(command):
out.extend( shlex.split(CONFIG[key]) )
return out
+def get_format_type(command):
+ """
+ Try to guess the dump format by inspecting the command elements
+ """
+ format_string = ""
+
+ for i,element in enumerate(command):
+
+ # the format was specified as a sepearate element
+ if element == '-F' or element == '--format':
+ format_string = command[i+1]
+ break
+
+ # the format was specified using the abbreviated form (one element)
+ if element.startswith('-F') and len(element) == 3:
+ format_string = element[-1]
+ break
+
+ if format_string in ('c', 'custom'):
+ return FORMAT_CUSTOM
+ elif format_string in ('t', 'tar'):
+ return FORMAT_TAR
+ else:
+ return FORMAT_PLAIN
+
def dump_one_db(staging_area, dbname):
LOG.info("Dumping %s" % dbname)
command = [ 'pg_dump', '-w' ]
command.extend( get_params("pg_dump") )
command.append( dbname )
- p1 = Popen( command, stdout=PIPE, stderr=PIPE )
- p2 = Popen( "gzip", stdin=p1.stdout, stdout=open(
- join(staging_area, "%s.gz" % dbname), "wb"), stderr=PIPE )
+ # change dump file suffix depending on dump type
+ dump_format = get_format_type(command)
+ if dump_format == FORMAT_TAR:
+ file_suffix = 'tar'
+ elif dump_format == FORMAT_CUSTOM:
+ file_suffix = 'c'
+ else:
+ file_suffix = 'sql'
+
+ filename = "%s.%s" % (dbname, file_suffix)
+
+ if CONFIG['compress_command']:
+
+ if CONFIG['compress_command'][0] == 'gzip':
+ compress_suffix = 'gz'
+ elif CONFIG['compress_command'][0] == 'bzip2':
+ compress_suffix = 'bz2'
+ elif CONFIG['compress_command'][0] == 'compress':
+ compress_suffix = 'z'
+ else:
+ compress_suffix = CONFIG['compress_command'][0]
- p1.wait()
- p2.wait()
+ p1 = Popen( command, stdout=PIPE, stderr=PIPE )
+ p2 = Popen( CONFIG['compress_command'], stdin=p1.stdout, stdout=open(
+ join(staging_area, "%s.%s" % (filename, compress_suffix)), "wb"),
+ stderr=PIPE )
- if p1.returncode != 0:
- LOG.error("Error while running pg_dump: %s" % p1.stderr.read())
+ p1.wait()
+ p2.wait()
- if p2.returncode != 0:
- LOG.error("Error while running gzip: %s" % p2.stderr.read())
+ if p1.returncode != 0:
+ LOG.error("Error while running pg_dump: %s" % p1.stderr.read())
+
+ if p2.returncode != 0:
+ LOG.error("Error while running gzip: %s" % p2.stderr.read())
+
+ else:
+ target_file = join(staging_area, "%s" % filename)
+ p1 = Popen( command + ['-f', target_file] )
+ stdout, stderr = p1.communicate()
+ if p1.returncode != 0:
+ LOG.error("Error while running pg_dump: %s" % stderr)
def dump_globals(staging_area):
LOG.info("Dumping posgtres globals")
View
2 pickup/pickup.py
@@ -41,7 +41,7 @@
#-----------------------------------------------------------------------------
-EXPECTED_CONFIG_VERSION = (2,1)
+EXPECTED_CONFIG_VERSION = (2,2)
TERM = TerminalController()
class ReverseLevelFilter(logging.Filter):
View
2 setup.py
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
setup(
name = "pickup",
- version = "1.3",
+ version = "1.4",
packages = find_packages(),
entry_points = { 'console_scripts': ['pickup = pickup.pickup:main'] },
install_requires = [

0 comments on commit 2b1e6fb

Please sign in to comment.
Something went wrong with that request. Please try again.