Merge branches 'gf/ls', 'gf/tag', 'zz/import-rsnapshot' and 'bl/selfi…

…ndex'

* gf/ls:
  ls-cmd: hide files with a leading dot by default

* gf/tag:
  Refuse branch/tag names that start with a dot
  tag-cmd: Some fixups

* zz/import-rsnapshot:
  Adds a testcase for import-rsnapshot.
  Makes import-rsnapshot use save's -f option.
  Adds -f option to save to use a given indexfile.
  Makefile: handle shell commands (cmd/*-cmd.sh)
  Adds documentation for bup-import-rsnapshot
  Adds import-rsnapshot command.
  Adds documentation for save's strip option.
  Adds testcases for --strip and --strip-path.
  Adds a strip and strip-path option to bup save.

* bl/selfindex:
  Rename receive-objects command to receive-objects-v2.
  Write idxs directly rather than using git-index-pack.
  Send SHAs from the client to reduce server load
  Use chunkyreader() instead of manually reading multiple blocks.

Documentation/bup-import-rsnapshot.md

@@ -0,0 +1,33 @@
+% bup-import-rsnapshot(1) Bup %BUP_VERSION%
+% Zoran Zaric <zz@zoranzaric.de>
+% %BUP_DATE%
+
+# NAME
+
+bup-import-rsnapshot - import a rsnapshot archive
+
+# SYNOPSIS
+
+bup import-rsnapshot [-n] <path to snapshot_root> [<backuptarget>]
+
+# SYNOPSIS
+
+`bup import-rsnapshot` imports a rsnapshot archive. The
+timestamps for the backups are preserved and the path to
+the rsnapshot archive is stripped from the paths.
+
+`bup import-rsnapshot` either imports the whole archive
+or only imports all backups for a given backuptarget.
+
+# OPTIONS
+
+-n,--dry-rung
+:   don't do anything just print out what would be done
+
+# EXAMPLE
+
+    $ bup import-rsnapshot /.snapshots
+
+# BUP
+
+Part of the `bup`(1) suite.

Documentation/bup-save.md

@@ -8,8 +8,8 @@ bup-save - create a new bup backup set
 
 # SYNOPSIS
 
-bup save [-r *host*:*path*] <-t|-c|-n *name*> [-v] [-q]
-  [--smaller=*maxsize*] <paths...>
+bup save [-r *host*:*path*] <-t|-c|-n *name*> [-f *indexfile*]
+[-v] [-q] [--smaller=*maxsize*] <paths...>
 
 # DESCRIPTION
 
@@ -45,6 +45,10 @@ for `bup-index`(1).
     the same name, and later view the history of that
     backup set to see how files have changed over time.)
 
+-f, --indexfile=*indexfile*
+:   use a different index filename instead of
+    `~/.bup/bupindex`.
+
 -v, --verbose
 :   increase verbosity (can be used more than once).  With
     one -v, prints every directory name as it gets backed up.  With
@@ -70,6 +74,21 @@ for `bup-index`(1).
     like k, M, or G to specify multiples of 1024,
     1024*1024, 1024*1024*1024 respectively.
 
+--strip
+:   strips the path that is given from all files and directories.
+
+    A directory */root/chroot/etc* saved with
+    "bup save -n chroot --strip /root/chroot" would be saved
+    as */etc*.
+
+--strip-prefix=*path-prefix*
+:   strips the given path-prefix *path-prefix* from all
+    files and directories.
+
+    A directory */root/chroots/webserver* saved with
+    "bup save -n webserver --strip-path=/root/chroots" would
+    be saved as */webserver/etc*
+
 
 # EXAMPLE
 

Makefile

@@ -103,7 +103,9 @@ bup: main.py
 	rm -f $@
 	ln -s $< $@
 
-cmds: $(patsubst cmd/%-cmd.py,cmd/bup-%,$(wildcard cmd/*-cmd.py))
+cmds: \
+    $(patsubst cmd/%-cmd.py,cmd/bup-%,$(wildcard cmd/*-cmd.py)) \
+    $(patsubst cmd/%-cmd.sh,cmd/bup-%,$(wildcard cmd/*-cmd.sh))
 
 cmd/bup-%: cmd/%-cmd.py
 	rm -f $@
@@ -117,6 +119,10 @@ bup-%: cmd-%.sh
 	rm -f $@
 	ln -s $< $@
 
+cmd/bup-%: cmd/%-cmd.sh
+	rm -f $@
+	ln -s $*-cmd.sh $@
+
 %.o: %.c
 	gcc -c -o $@ $< $(CPPFLAGS) $(CFLAGS)
 

cmd/import-rsnapshot-cmd.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+# bup-import-rsnapshot.sh
+
+# Does an import of a rsnapshot archive.
+
+usage() {
+    echo "Usage: bup import-rsnapshot [-n]" \
+        "<path to snapshot_root> [<backuptarget>]"
+    echo "-n,--dry-rung: don't do anything just print out what would be done"
+    exit -1
+}
+
+if [ "$1" = "-n" -o "$1" = "--dry-run" ]; then
+    bup()
+    {
+        echo bup "$@" >&2
+    }
+    shift 1
+elif [ -n "$BUP_MAIN_EXE" ]; then
+    bup()
+    {
+        "$BUP_MAIN_EXE" "$@"
+    }
+else
+    bup()
+    {
+        bup "$@"
+    }
+fi
+
+[ "$#" -eq 1 ] || [ "$#" -eq 2 ] || usage
+
+if [ ! -e "$1/." ]; then
+    echo "$1 isn't a directory!"
+    exit -1
+fi
+
+TARGET=
+[ "$#" -eq 2 ] && TARGET="$2"
+
+
+ABSPATH=`readlink -f "$1"`
+
+for SNAPSHOT in "$ABSPATH/"*; do
+    if [ -e "$SNAPSHOT/." ]; then
+        for BRANCH_PATH in "$SNAPSHOT/"*; do
+            if [ -e "$BRANCH_PATH/." ]; then
+                # Get the snapshot's ctime
+                DATE=`stat -c %Z "$BRANCH_PATH"`
+                BRANCH=`basename "$BRANCH_PATH"`
+                TMPIDX="/tmp/$BRANCH"
+
+                if [ "$TARGET" == "" ] || [ "$TARGET" == "$BRANCH" ]; then
+                    bup index -ux \
+                        -f $TMPIDX \
+                        $BRANCH_PATH/
+                    bup save \
+                        --strip \
+                        --date=$DATE \
+                        -f $TMPIDX \
+                        -n $BRANCH \
+                        $BRANCH_PATH/
+
+                    if [ -e "$TMPIDX" ]; then
+                        rm "$TMPIDX"
+                    fi
+                fi
+            fi
+        done
+    fi
+done
+

cmd/ls-cmd.py

@@ -19,6 +19,7 @@ def print_node(text, n):
 bup ls <dirs...>
 --
 s,hash   show hash for each file
+a,all    show hidden files
 """
 o = options.Options('bup ls', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -35,9 +36,11 @@ def print_node(text, n):
         n = top.lresolve(d)
         if stat.S_ISDIR(n.mode):
             for sub in n:
-                print_node(sub.name, sub)
+                if opt.all or not sub.name.startswith('.'):
+                    print_node(sub.name, sub)
         else:
-            print_node(d, n)
+            if opt.all or not sub.name.startswith('.'):
+                print_node(d, n)
     except vfs.NodeError, e:
         log('error: %s\n' % e)
         ret = 1

cmd/save-cmd.py

@@ -16,6 +16,9 @@
 q,quiet    don't show progress meter
 smaller=   only back up files smaller than n bytes
 bwlimit=   maximum bytes/sec to transmit to server
+f,indexfile=  the name of the index file (normally BUP_DIR/bupindex)
+strip      strips the path to every filename given
+strip-path= path-prefix to be stripped when saving
 """
 o = options.Options('bup save', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -36,10 +39,15 @@
 else:
     date = time.time()
 
+if opt.strip and opt.strip_path:
+    o.fatal("--strip is incompatible with --strip-path")
+
 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
 if is_reverse and opt.remote:
     o.fatal("don't use -r in reverse mode; it's automatic")
 
+if opt.name and opt.name.startswith('.'):
+    o.fatal("'%s' is not a valid branch name" % opt.name)
 refname = opt.name and 'refs/heads/%s' % opt.name or None
 if opt.remote or is_reverse:
     cli = client.Client(opt.remote)
@@ -129,7 +137,9 @@ def vlog(s):
     log(s)
 
 
-r = index.Reader(git.repo('bupindex'))
+indexfile = opt.indexfile or git.repo('bupindex')
+print indexfile
+r = index.Reader(indexfile)
 
 def already_saved(ent):
     return ent.is_valid() and w.exists(ent.sha) and ent.sha
@@ -195,7 +205,13 @@ def wantrecurse_during(ent):
         continue
 
     assert(dir.startswith('/'))
-    dirp = dir.split('/')
+    if opt.strip:
+        stripped_base_path = strip_base_path(dir, extra)
+        dirp = stripped_base_path.split('/')
+    elif opt.strip_path:
+        dirp = strip_path(opt.strip_path, dir).split('/')
+    else:
+        dirp = dir.split('/')
     while parts > dirp:
         _pop(force_tree = None)
     if dir != '/':
@@ -283,7 +299,6 @@ def wantrecurse_during(ent):
     print tree.encode('hex')
 if opt.commit or opt.name:
     msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
-    ref = opt.name and ('refs/heads/%s' % opt.name) or None
     commit = w.new_commit(oldref, tree, date, msg)
     if opt.commit:
         print commit.encode('hex')

cmd/server-cmd.py

@@ -36,7 +36,7 @@ def send_index(conn, name):
     conn.ok()
 
 
-def receive_objects(conn, junk):
+def receive_objects_v2(conn, junk):
     global suspended_w
     git.check_repo_or_die()
     suggested = {}
@@ -67,15 +67,16 @@ def receive_objects(conn, junk):
             conn.ok()
             return
 
+        shar = conn.read(20)
+        crcr = struct.unpack('!I', conn.read(4))[0]
+        n -= 20 + 4
         buf = conn.read(n)  # object sizes in bup are reasonably small
         #debug2('read %d bytes\n' % n)
         if len(buf) < n:
             w.abort()
             raise Exception('object read: expected %d bytes, got %d\n'
                             % (n, len(buf)))
-        (type, content) = git._decode_packobj(buf)
-        sha = git.calc_hash(type, content)
-        oldpack = w.exists(sha)
+        oldpack = w.exists(shar)
         # FIXME: we only suggest a single index per cycle, because the client
         # is currently too dumb to download more than one per cycle anyway.
         # Actually we should fix the client, but this is a minor optimization
@@ -88,7 +89,7 @@ def receive_objects(conn, junk):
             # fix that deficiency of midx files eventually, although it'll
             # make the files bigger.  This method is certainly not very
             # efficient.
-            oldpack = w.objcache.packname_containing(sha)
+            oldpack = w.objcache.packname_containing(shar)
             debug2('new suggestion: %r\n' % oldpack)
             assert(oldpack)
             assert(oldpack != True)
@@ -102,8 +103,16 @@ def receive_objects(conn, junk):
                 conn.write('index %s\n' % name)
                 suggested[name] = 1
         else:
-            w._raw_write([buf])
+            nw, crc = w._raw_write([buf], sha=shar)
+            _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
+            _check(w, n, nw, 'object read: expected %d bytes, got %d\n')
     # NOTREACHED
+
+
+def _check(w, expected, actual, msg):
+    if expected != actual:
+        w.abort()
+        raise Exception(msg % (expected, actual))
 
 
 def read_ref(conn, refname):
@@ -156,7 +165,7 @@ def cat(conn, id):
     'set-dir': set_dir,
     'list-indexes': list_indexes,
     'send-index': send_index,
-    'receive-objects': receive_objects,
+    'receive-objects-v2': receive_objects_v2,
     'read-ref': read_ref,
     'update-ref': update_ref,
     'cat': cat,

cmd/split-cmd.py

@@ -78,6 +78,8 @@ def prog(filenum, nbytes):
     o.fatal("don't use -r in reverse mode; it's automatic")
 start_time = time.time()
 
+if opt.name and opt.name.startswith('.'):
+    o.fatal("'%s' is not a valid branch name." % opt.name)
 refname = opt.name and 'refs/heads/%s' % opt.name or None
 if opt.noop or opt.copy:
     cli = pack_writer = oldref = None

cmd/tag-cmd.py

@@ -35,33 +35,38 @@
     try:
         os.unlink(tag_file)
     except OSError, e:
-        log("bup: error: unable to delete tag: %s" % e)
+        log("bup: error: unable to delete tag '%s': %s" % (opt.delete, e))
         sys.exit(1)
 
     sys.exit(0)
 
-tags = []
-for (t, dummy) in git.list_refs():
-    if t.startswith('refs/tags/'):
-        tags.append(t[10:])
+tags = [t for sublist in git.tags().values() for t in sublist]
 
 if not extra:
     for t in tags:
-        log("%s\n" % t)
+        print t
     sys.exit(0)
-elif len(extra) != 2:
-    log('bup: error: no ref or hash given.')
-    sys.exit(1)
+elif len(extra) < 2:
+    o.fatal('no commit ref or hash given.')
 
-tag_name = extra[0]
-commit = extra[1]
-debug1("from args: tag name = %s; commit = %s\n" % (tag_name, commit))
+(tag_name, commit) = extra[:2]
+if not tag_name:
+    o.fatal("tag name must not be empty.")
+debug1("args: tag name = %s; commit = %s\n" % (tag_name, commit))
 
 if tag_name in tags:
     log("bup: error: tag '%s' already exists" % tag_name)
     sys.exit(1)
 
-hash = git.rev_parse(commit)
+if tag_name.startswith('.'):
+    o.fatal("'%s' is not a valid tag name." % tag_name)
+
+try:
+    hash = git.rev_parse(commit)
+except git.GitError, e:
+    log("bup: error: %s" % e)
+    sys.exit(2)
+
 if not hash:
     log("bup: error: commit %s not found." % commit)
     sys.exit(2)
@@ -75,7 +80,7 @@
 try:
     tag = file(tag_file, 'w')
 except OSError, e:
-    log('bup: error: could not create tag %s: %s' % (tag_name, e))
+    log("bup: error: could not create tag '%s': %s" % (tag_name, e))
     sys.exit(3)
 
 tag.write(hash.encode('hex'))