Permalink
Browse files

Initial commit

Signed-off-by: Bas Westerbaan <bas@westerbaan.name>
  • Loading branch information...
0 parents commit 5b8208ed2cedefdcaf57e84ceb40a896b1e54c6c @bwesterb committed Feb 4, 2012
Showing with 973 additions and 0 deletions.
  1. +8 −0 .gitignore
  2. +674 −0 LICENSE
  3. +2 −0 MANIFEST.in
  4. +105 −0 get_git_version.py
  5. 0 pachy/__init__.py
  6. +160 −0 pachy/main.py
  7. +24 −0 setup.py
@@ -0,0 +1,8 @@
+*.pyo
+*.pyc
+*.swp
+/*.egg-info
+/build
+/dist
+MANIFEST
+RELEASE-VERSION
Oops, something went wrong.
@@ -0,0 +1,2 @@
+include RELEASE-VERSION
+include get_get_version.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Author: Douglas Creager <dcreager@dcreager.net>
+# This file is placed into the public domain.
+
+# Calculates the current version number. If possible, this is the
+# output of “git describe”, modified to conform to the versioning
+# scheme that setuptools uses. If “git describe” returns an error
+# (most likely because we're in an unpacked copy of a release tarball,
+# rather than in a git working copy), then we fall back on reading the
+# contents of the RELEASE-VERSION file.
+#
+# To use this script, simply import it your setup.py file, and use the
+# results of get_git_version() as your package version:
+#
+# from version import *
+#
+# setup(
+# version=get_git_version(),
+# .
+# .
+# .
+# )
+#
+# This will automatically update the RELEASE-VERSION file, if
+# necessary. Note that the RELEASE-VERSION file should *not* be
+# checked into git; please add it to your top-level .gitignore file.
+#
+# You'll probably want to distribute the RELEASE-VERSION file in your
+# sdist tarballs; to do this, just create a MANIFEST.in file that
+# contains the following line:
+#
+# include RELEASE-VERSION
+
+__all__ = ("get_git_version")
+
+from subprocess import Popen, PIPE
+
+
+def call_git_describe(abbrev=4):
+ try:
+ p = Popen(['git', 'describe', '--abbrev=%d' % abbrev],
+ stdout=PIPE, stderr=PIPE)
+ p.stderr.close()
+ line = p.stdout.readlines()[0]
+ return line.strip()
+
+ except:
+ return None
+
+
+def read_release_version():
+ try:
+ f = open("RELEASE-VERSION", "r")
+
+ try:
+ version = f.readlines()[0]
+ return version.strip()
+
+ finally:
+ f.close()
+
+ except:
+ return None
+
+
+def write_release_version(version):
+ f = open("RELEASE-VERSION", "w")
+ f.write("%s\n" % version)
+ f.close()
+
+
+def get_git_version(abbrev=4):
+ # Read in the version that's currently in RELEASE-VERSION.
+
+ release_version = read_release_version()
+
+ # First try to get the current version using “git describe”.
+
+ version = call_git_describe(abbrev)
+
+ # If that doesn't work, fall back on the value that's in
+ # RELEASE-VERSION.
+
+ if version is None:
+ version = release_version
+
+ # If we still don't have anything, that's an error.
+
+ if version is None:
+ raise ValueError("Cannot find the version number!")
+
+ # If the current version is different from what's in the
+ # RELEASE-VERSION file, update the file to be current.
+
+ if version != release_version:
+ write_release_version(version)
+
+ # Finally, return the current version.
+
+ return version
+
+
+if __name__ == "__main__":
+ print get_git_version()
+
No changes.
@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# vim: et:sta:bs=2:sw=4:
+
+# pachy (short for pachyderm referring to elephants) is a backup tool.
+#
+# Bas Westerbaan <bas@westerbaan.name>
+# Licensed under the GPLv3
+
+import sys
+import logging
+import os.path
+import argparse
+import datetime
+import subprocess
+
+class Pachy(object):
+ def parse_cmdLine_args(self):
+ parser = argparse.ArgumentParser(
+ description='pachy does incremental backups')
+ parser.add_argument('source', metavar='SRC',
+ help='source directory')
+ parser.add_argument('dest', metavar='DEST',
+ help='destination directory')
+ self.args = parser.parse_args()
+ # Ensure source has a trailing /
+ self.source_arg = self.args.source
+ if not self.source_arg.endswith('/'):
+ self.source_arg += '/'
+ # Set some convenience variables
+ self.mirror_dir = os.path.abspath(os.path.join(
+ self.args.dest, 'mirror'))
+ self.deltas_dir = os.path.abspath(os.path.join(
+ self.args.dest, 'deltas'))
+ self.work_dir = os.path.abspath(os.path.join(
+ self.args.dest, 'work'))
+ self.pile_dir = os.path.join(self.work_dir, 'pile')
+
+ def main(self):
+ self.parse_cmdLine_args()
+ logging.info("0. Checking set-up")
+ self.check_setup()
+ logging.info("1. Running rsync")
+ self.run_rsync()
+ logging.info("2. Checking for changes")
+ self.find_changed()
+ logging.info("3. Creating archive")
+ self.create_archive()
+ logging.info("4. Cleaning up")
+ self.cleanup()
+
+ def check_setup(self):
+ # Does the destination directory exist?
+ if not os.path.exists(self.args.dest):
+ # TODO add an option to create the directory
+ logging.error('Destination directory does not exist')
+ sys.exit(1)
+ # Do the mirror, deltas and work subdirectories exist?
+ for d in (self.mirror_dir, self.deltas_dir, self.work_dir):
+ if not os.path.exists(d):
+ os.mkdir(d)
+ # Is the work directory empty?
+ if os.listdir(self.work_dir):
+ # TODO add an option to clean the directory
+ logging.error('The work directory is not empty')
+ sys.exit(2)
+ os.mkdir(os.path.join(self.work_dir, 'pile'))
+ os.mkdir(os.path.join(self.work_dir, 'changed'))
+ os.mkdir(os.path.join(self.work_dir, 'deleted'))
+
+ def run_rsync(self):
+ ret = subprocess.call([
+ 'rsync', '--archive', # we want to preserve metadata
+ '--backup', # do not override files
+ '--delete', # delete extraneous files
+ self.source_arg,
+ self.mirror_dir,
+ '--backup-dir='+self.pile_dir,
+ '--filter=dir-merge /.pachy-filter',
+ '--verbose'])
+ if ret != 0:
+ logging.error('rsync failed with error code %s', ret)
+ sys.exit(3)
+
+ def find_changed(self):
+ # walk the work directory
+ stack = ['.']
+ while stack:
+ d = stack.pop()
+ d_pile = os.path.join(self.pile_dir, d)
+ d_mirror = os.path.join(self.mirror_dir, d)
+ for c in os.listdir(d_pile):
+ c_pile = os.path.join(d_pile, c)
+ if os.path.isdir(c_pile):
+ stack.append(os.path.join(d, c))
+ continue
+ c_mirror = os.path.join(d_mirror, c)
+ # c is a file in the work directory.
+ if not os.path.exists(c_mirror):
+ # it was apparently deleted. Move to deleted.
+ d_deleted = os.path.join(self.work_dir, 'deleted', d)
+ # TODO cache this to limit syscalls
+ if not os.path.exists(d_deleted):
+ os.makedirs(d_deleted)
+ os.rename(c_pile, os.path.join(d_deleted, c))
+ continue
+ # c was changed. Create a xdelta
+ d_changed = os.path.join(self.work_dir, 'changed', d)
+ # TODO cache this to limit syscalls
+ if not os.path.exists(d_changed):
+ os.makedirs(d_changed)
+ self.create_delta(os.path.join(d, c))
+
+ def create_delta(self, f):
+ f_pile = os.path.join(self.pile_dir, f)
+ f_mirror = os.path.join(self.mirror_dir, f)
+ f_changed = os.path.join(self.work_dir, 'changed', f) + '.xdelta3'
+ ret = subprocess.call([
+ 'xdelta3',
+ '-s', f_pile, # source
+ f_mirror, # target
+ f_changed]) # out
+ if ret != 0:
+ logging.error('xdelta3 failed with errorcode %s', ret)
+ sys.exit(4)
+
+ def create_archive(self):
+ archive_path = os.path.join(self.deltas_dir,
+ datetime.datetime.now().strftime('%Y-%m-%d@%Hh%M.%S.tar'))
+ ret = subprocess.call([
+ 'tar',
+ '-cf', # create a file
+ archive_path,
+ 'changed',
+ 'deleted'],
+ cwd=self.work_dir)
+ if ret != 0:
+ logging.error('tar failed with errorcode %s', ret)
+ sys.exit(5)
+ ret = subprocess.call([
+ 'xz',
+ '-9',
+ archive_path])
+ if ret != 0:
+ logging.error('xz failed with errorcode %s', ret)
+ sys.exit(6)
+ def cleanup(self):
+ ret = subprocess.call([
+ 'rm',
+ '-r',
+ self.work_dir])
+ if ret != 0:
+ logging.error('rm failed with errorcode %s', ret)
+ sys.exit(7)
+
+def main():
+ logging.basicConfig(level=logging.DEBUG)
+ Pachy().main()
+
+if __name__ == '__main__':
+ main()
@@ -0,0 +1,24 @@
+# vim: et:sta:bs=2:sw=4:
+#!/usr/bin/env python
+
+from setuptools import setup
+from get_git_version import get_git_version
+
+setup(name='pachy',
+ version=get_git_version(),
+ description='Simple incremental backups with rsync and xdelta3',
+ author='Bas Westerbaan',
+ author_email='bas@westerbaan.name',
+ url='http://github.com/bwesterb/pachy/',
+ packages=['pachy'],
+ zip_safe=True,
+ install_requires = ['docutils>=0.3'],
+ entry_points = {
+ 'console_scripts': [
+ 'pachy = pachy.main:main',
+ ]
+ },
+ classifiers=[
+ "Topic :: System :: Archiving :: Backup",
+ "License :: OSI Approved :: GNU General Public License (GPL)",
+ "Development Status :: 4 - Beta"])

0 comments on commit 5b8208e

Please sign in to comment.