Initial commit

Signed-off-by: Bas Westerbaan <bas@westerbaan.name>

.gitignore

@@ -0,0 +1,8 @@
+*.pyo
+*.pyc
+*.swp
+/*.egg-info
+/build
+/dist
+MANIFEST
+RELEASE-VERSION

MANIFEST.in

@@ -0,0 +1,2 @@
+include RELEASE-VERSION
+include get_get_version.py

get_git_version.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Author: Douglas Creager <dcreager@dcreager.net>
+# This file is placed into the public domain.
+
+# Calculates the current version number.  If possible, this is the
+# output of “git describe”, modified to conform to the versioning
+# scheme that setuptools uses.  If “git describe” returns an error
+# (most likely because we're in an unpacked copy of a release tarball,
+# rather than in a git working copy), then we fall back on reading the
+# contents of the RELEASE-VERSION file.
+#
+# To use this script, simply import it your setup.py file, and use the
+# results of get_git_version() as your package version:
+#
+# from version import *
+#
+# setup(
+#     version=get_git_version(),
+#     .
+#     .
+#     .
+# )
+#
+# This will automatically update the RELEASE-VERSION file, if
+# necessary.  Note that the RELEASE-VERSION file should *not* be
+# checked into git; please add it to your top-level .gitignore file.
+#
+# You'll probably want to distribute the RELEASE-VERSION file in your
+# sdist tarballs; to do this, just create a MANIFEST.in file that
+# contains the following line:
+#
+#   include RELEASE-VERSION
+
+__all__ = ("get_git_version")
+
+from subprocess import Popen, PIPE
+
+
+def call_git_describe(abbrev=4):
+    try:
+        p = Popen(['git', 'describe', '--abbrev=%d' % abbrev],
+                  stdout=PIPE, stderr=PIPE)
+        p.stderr.close()
+        line = p.stdout.readlines()[0]
+        return line.strip()
+
+    except:
+        return None
+
+
+def read_release_version():
+    try:
+        f = open("RELEASE-VERSION", "r")
+
+        try:
+            version = f.readlines()[0]
+            return version.strip()
+
+        finally:
+            f.close()
+
+    except:
+        return None
+
+
+def write_release_version(version):
+    f = open("RELEASE-VERSION", "w")
+    f.write("%s\n" % version)
+    f.close()
+
+
+def get_git_version(abbrev=4):
+    # Read in the version that's currently in RELEASE-VERSION.
+
+    release_version = read_release_version()
+
+    # First try to get the current version using “git describe”.
+
+    version = call_git_describe(abbrev)
+
+    # If that doesn't work, fall back on the value that's in
+    # RELEASE-VERSION.
+
+    if version is None:
+        version = release_version
+
+    # If we still don't have anything, that's an error.
+
+    if version is None:
+        raise ValueError("Cannot find the version number!")
+
+    # If the current version is different from what's in the
+    # RELEASE-VERSION file, update the file to be current.
+
+    if version != release_version:
+        write_release_version(version)
+
+    # Finally, return the current version.
+
+    return version
+
+
+if __name__ == "__main__":
+    print get_git_version()
+

pachy/main.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# vim: et:sta:bs=2:sw=4:
+
+# pachy (short for pachyderm referring to elephants) is a backup tool.
+#
+# Bas Westerbaan <bas@westerbaan.name>
+# Licensed under the GPLv3
+
+import sys
+import logging
+import os.path
+import argparse
+import datetime
+import subprocess
+
+class Pachy(object):
+    def parse_cmdLine_args(self):
+        parser = argparse.ArgumentParser(
+                        description='pachy does incremental backups')
+        parser.add_argument('source', metavar='SRC',
+                                help='source directory')
+        parser.add_argument('dest', metavar='DEST',
+                                help='destination directory')
+        self.args = parser.parse_args()
+        # Ensure source has a trailing /
+        self.source_arg = self.args.source
+        if not self.source_arg.endswith('/'):
+            self.source_arg += '/'
+        # Set some convenience variables
+        self.mirror_dir = os.path.abspath(os.path.join(
+                                self.args.dest, 'mirror'))
+        self.deltas_dir = os.path.abspath(os.path.join(
+                                self.args.dest, 'deltas'))
+        self.work_dir = os.path.abspath(os.path.join(
+                                self.args.dest, 'work'))
+        self.pile_dir = os.path.join(self.work_dir, 'pile')
+
+    def main(self):
+        self.parse_cmdLine_args()
+        logging.info("0. Checking set-up")
+        self.check_setup()
+        logging.info("1. Running rsync")
+        self.run_rsync()
+        logging.info("2. Checking for changes")
+        self.find_changed()
+        logging.info("3. Creating archive")
+        self.create_archive()
+        logging.info("4. Cleaning up")
+        self.cleanup()
+
+    def check_setup(self):
+        # Does the destination directory exist?
+        if not os.path.exists(self.args.dest):
+            # TODO add an option to create the directory
+            logging.error('Destination directory does not exist')
+            sys.exit(1)
+        # Do the mirror, deltas and work subdirectories exist?
+        for d in (self.mirror_dir, self.deltas_dir, self.work_dir):
+            if not os.path.exists(d):
+                os.mkdir(d)
+        # Is the work directory empty?
+        if os.listdir(self.work_dir):
+            # TODO add an option to clean the directory
+            logging.error('The work directory is not empty')
+            sys.exit(2)
+        os.mkdir(os.path.join(self.work_dir, 'pile'))
+        os.mkdir(os.path.join(self.work_dir, 'changed'))
+        os.mkdir(os.path.join(self.work_dir, 'deleted'))
+
+    def run_rsync(self):
+        ret = subprocess.call([
+                'rsync', '--archive', # we want to preserve metadata
+                         '--backup',  # do not override files
+                         '--delete',  # delete extraneous files
+                         self.source_arg,
+                         self.mirror_dir,
+                         '--backup-dir='+self.pile_dir,
+                         '--filter=dir-merge /.pachy-filter',
+                         '--verbose'])
+        if ret != 0:
+            logging.error('rsync failed with error code %s', ret)
+            sys.exit(3)
+
+    def find_changed(self):
+        # walk the work directory
+        stack = ['.']
+        while stack:
+            d = stack.pop()
+            d_pile = os.path.join(self.pile_dir, d)
+            d_mirror = os.path.join(self.mirror_dir, d)
+            for c in os.listdir(d_pile):
+                c_pile = os.path.join(d_pile, c)
+                if os.path.isdir(c_pile):
+                    stack.append(os.path.join(d, c))
+                    continue
+                c_mirror = os.path.join(d_mirror, c)
+                # c is a file in the work directory.
+                if not os.path.exists(c_mirror):
+                    # it was apparently deleted. Move to deleted.
+                    d_deleted = os.path.join(self.work_dir, 'deleted', d)
+                    # TODO cache this to limit syscalls
+                    if not os.path.exists(d_deleted):
+                        os.makedirs(d_deleted)
+                    os.rename(c_pile, os.path.join(d_deleted, c))
+                    continue
+                # c was changed.  Create a xdelta
+                d_changed = os.path.join(self.work_dir, 'changed', d)
+                # TODO cache this to limit syscalls
+                if not os.path.exists(d_changed):
+                    os.makedirs(d_changed)
+                self.create_delta(os.path.join(d, c))
+
+    def create_delta(self, f):
+        f_pile = os.path.join(self.pile_dir, f)
+        f_mirror = os.path.join(self.mirror_dir, f)
+        f_changed = os.path.join(self.work_dir, 'changed', f) + '.xdelta3'
+        ret = subprocess.call([
+                    'xdelta3',
+                    '-s', f_pile, # source
+                    f_mirror,     # target
+                    f_changed])   # out
+        if ret != 0:
+            logging.error('xdelta3 failed with errorcode %s', ret)
+            sys.exit(4)
+
+    def create_archive(self):
+        archive_path = os.path.join(self.deltas_dir,
+                    datetime.datetime.now().strftime('%Y-%m-%d@%Hh%M.%S.tar'))
+        ret = subprocess.call([
+                    'tar',
+                    '-cf',        # create a file
+                    archive_path,
+                    'changed',
+                    'deleted'],
+                        cwd=self.work_dir)
+        if ret != 0:
+            logging.error('tar failed with errorcode %s', ret)
+            sys.exit(5)
+        ret = subprocess.call([
+                    'xz',
+                    '-9',
+                    archive_path])
+        if ret != 0:
+            logging.error('xz failed with errorcode %s', ret)
+            sys.exit(6)
+    def cleanup(self):
+        ret = subprocess.call([
+                    'rm',
+                    '-r',
+                    self.work_dir])
+        if ret != 0:
+            logging.error('rm failed with errorcode %s', ret)
+            sys.exit(7)
+
+def main():
+    logging.basicConfig(level=logging.DEBUG)
+    Pachy().main()
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,24 @@
+# vim: et:sta:bs=2:sw=4:
+#!/usr/bin/env python
+
+from setuptools import setup
+from get_git_version import get_git_version
+
+setup(name='pachy',
+      version=get_git_version(),
+      description='Simple incremental backups with rsync and xdelta3',
+      author='Bas Westerbaan',
+      author_email='bas@westerbaan.name',
+      url='http://github.com/bwesterb/pachy/',
+      packages=['pachy'],
+      zip_safe=True,
+      install_requires = ['docutils>=0.3'],
+      entry_points = {
+          'console_scripts': [
+              'pachy = pachy.main:main',
+              ]
+          },
+      classifiers=[
+              "Topic :: System :: Archiving :: Backup",
+              "License :: OSI Approved :: GNU General Public License (GPL)",
+              "Development Status :: 4 - Beta"])