rsync-repos

#!/usr/bin/env python
#
# Use rsync to intelligently sync OSTree repositories.  This
# avoids a few race conditions and issues that could otherwise
# occur if one simply does the whole repository in a single run.
#
# Known issues:
#   - Ignores the fact that detached metadata (e.g. GPG signatures)
#     can be updated asynchronously.
#
# Copyright 2016 Colin Walters <walters@verbum.org>
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)

import os, sys, subprocess, argparse, shlex

def fatal(msg):
    print >>sys.stderr, msg
    sys.exit(1)

parser = argparse.ArgumentParser()
parser.add_argument("--src", help="Source path",
                    action='store', required=True)
parser.add_argument("--dest", help="Destination path",
                    action='store', required=True)
parser.add_argument("--rsync-opts", help="Additional rsync options (parsed via shell)",
                    action='store', default=None)
parser.add_argument("--rsync-opt", help="Additional (single) rsync option",
                    action='append', default=[])

args = parser.parse_args()

OBJECTS_AND_DELTAS = ['/objects', '/objects/**', '/deltas', '/deltas/**']
# We rsync in reverse data dependence order - the summary and refs
# point to objects + deltas.  Our first pass over the objects doesn't
# perform any deletions, as that would create race conditions.  We
# do handle deletions for refs and summary.
# Finally, we handle any deletions for objects and deltas.
RSYNC_ORDERING = [(OBJECTS_AND_DELTAS, ['--ignore-existing']),
                  (['/refs', '/refs/**', '/summary', '/summary.sig'], ['--delete', '--ignore-missing-args']),
                  (OBJECTS_AND_DELTAS, ['--ignore-existing', '--delete'])]

for (paths, opts) in RSYNC_ORDERING:
    argv = ['rsync', '-rlpt']
    for path in paths:
        argv.append('--include=' + path)
    argv.extend(['--exclude=*', args.src, args.dest])
    argv.extend(opts)
    if args.rsync_opts is not None:
        argv.extend(shlex.split(args.rsync_opts))
    argv.extend(args.rsync_opt)
    print("Executing: {}".format(subprocess.list2cmdline(argv)))
    subprocess.check_call(argv)