Extremely basic 'bup server' support.

It's enough to send a pack to the remote end with 'bup split', though 'bup
save' doesn't support it yet, and we're not smart enough to do incremental
backups, which means we generate the gigantic pack every single time.

.gitignore

@@ -1,8 +1,5 @@
 bup
-bup-split
-bup-join
-bup-save
-bup-init
+bup-*
 randomgen
 *.o
 *.so

Makefile

@@ -2,7 +2,7 @@ CFLAGS=-Wall -g -O2 -Werror -I/usr/include/python2.5 -g -fPIC
 
 default: all
 
-all: bup-split bup-join bup-save bup-init bup randomgen chashsplit.so
+all: bup-split bup-join bup-save bup-init bup-server bup randomgen chashsplit.so
 
 randomgen: randomgen.o
 
@@ -38,5 +38,6 @@ bup-%: cmd-%.sh
 
 clean:
 	rm -f *.o *.so *~ .*~ *.pyc */*.pyc */*~ \
-		bup bup-split bup-join bup-save bup-init randomgen \
-		out[12] out2[tc] tags[12] tags2[tc] *.tmp
+		bup bup-* randomgen \
+		out[12] out2[tc] tags[12] tags2[tc]
+	rm -rf *.tmp

bup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import sys, os
+import sys, os, git
 
 argv = sys.argv
 exe = argv[0]
@@ -16,7 +16,6 @@ def usage():
             log('\t%s\n' % c[4:])
     exit(99)
 
-
 if len(argv) < 2 or not argv[1] or argv[1][0] == '-':
     usage()
 

cmd-save.py

@@ -105,6 +105,7 @@ def gen_tree(self, w):
 o = options.Options('bup save', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 
+git.check_repo_or_die()
 if not (opt.tree or opt.commit or opt.name):
     log("bup save: use one or more of -t, -c, -n\n")
     o.usage()

cmd-server.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+import sys, struct
+import options, git
+from helpers import *
+
+
+def receive_objects(f):
+    w = git.PackWriter()
+    while 1:
+        ns = f.read(4)
+        if not ns:
+            w.abort()
+            raise Exception('object read: expected length header, got EOF\n')
+        n = struct.unpack('!I', ns)[0]
+        #log('expecting %d bytes\n' % n)
+        if not n:
+            w.close()
+            return
+        buf = f.read(n)
+        #log('read %d bytes\n' % n)
+        if len(buf) < n:
+            w.abort()
+            raise Exception('object read: expected %d bytes, got %d\n'
+                            % (n, len(buf)))
+        w._raw_write(buf)
+    w.close()
+
+
+optspec = """
+bup server
+"""
+o = options.Options('bup server', optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+    log('bup server: no arguments expected\n')
+    o.usage()
+
+log('bup server: reading from stdin.\n')
+
+f = sys.stdin
+lr = linereader(f)
+for _line in lr:
+    line = _line.strip()
+    if not line:
+        continue
+    log('bup server: command: %r\n' % line)
+    if line == 'quit':
+        break
+    elif line == 'set-dir':
+        git.repodir = lr.next()
+        git.check_repo_or_die()
+        log('bup server: bupdir is %r\n' % git.repodir)
+    elif line == 'receive-objects':
+        git.check_repo_or_die()
+        receive_objects(f)
+    else:
+        raise Exception('unknown server command: %r\n' % line)
+
+log('bup server: done\n')

cmd-split.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
-import sys, time
+import sys, time, re
 import hashsplit, git, options
 from helpers import *
 
 optspec = """
 bup split [-tcb] [-n name] [--bench] [filenames...]
 --
+r,remote=  remote repository path
 b,blobs    output a series of blob ids
 t,tree     output a tree id
 c,commit   output a commit id
@@ -16,6 +17,7 @@
 o = options.Options('bup split', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 
+git.check_repo_or_die()
 if not (opt.blobs or opt.tree or opt.commit or opt.name):
     log("bup split: use one or more of -b, -t, -c, -n\n")
     o.usage()
@@ -27,7 +29,26 @@
 
 start_time = time.time()
 
-w = git.PackWriter()
+def server_connect(remote):
+    rs = remote.split(':', 1)
+    if len(rs) == 1:
+        p = subprocess.Popen(['bup', 'server', '-d', opt.remote],
+                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    else:
+        (host, dir) = rs
+        p = subprocess.Popen(['ssh', host, '--', 'bup', 'server'],
+                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        dir = re.sub(r'[\r\n]', ' ', dir)
+        p.stdin.write('set-dir\n%s\n' % dir)
+    return p
+
+if opt.remote:
+    p = server_connect(opt.remote)
+    p.stdin.write('receive-objects\n')
+    w = git.PackWriter_Remote(p.stdin)
+else:
+    w = git.PackWriter()
+
 (shalist,tree) = hashsplit.split_to_tree(w, hashsplit.autofiles(extra))
 
 if opt.verbose:
@@ -44,6 +65,11 @@
     if opt.commit:
         print commit.encode('hex')
 
+if opt.remote:
+    w.close()
+    p.stdin.write('quit\n')
+    p.wait()
+
 secs = time.time() - start_time
 size = hashsplit.total_split
 if opt.bench:

git.py

@@ -2,9 +2,10 @@
 from helpers import *
 
 verbose = 0
+repodir = os.environ.get('BUP_DIR', '.git')
 
-def repodir(sub = ''):
-    return os.path.join(os.environ.get('BUP_DIR', '.git'), sub)
+def repo(sub = ''):
+    return os.path.join(repodir, sub)
 
 
 class PackIndex:
@@ -102,48 +103,52 @@ def _shalist_sort_key(ent):
 
 _typemap = dict(blob=3, tree=2, commit=1, tag=8)
 class PackWriter:
-    def __init__(self):
+    def __init__(self, objcache=None):
         self.count = 0
-        self.binlist = []
-        self.objcache = MultiPackIndex(repodir('objects/pack'))
         self.filename = None
         self.file = None
+        self.objcache = objcache or MultiPackIndex(repo('objects/pack'))
 
     def __del__(self):
         self.close()
 
     def _open(self):
         assert(not self.file)
         self.objcache.zap_also()
-        self.filename = repodir('objects/bup%d' % os.getpid())
+        self.filename = repo('objects/bup%d' % os.getpid())
         self.file = open(self.filename + '.pack', 'w+')
         self.file.write('PACK\0\0\0\2\0\0\0\0')
 
-    def _write(self, bin, type, content):
+    def _raw_write(self, datalist):
         if not self.file:
             self._open()
         f = self.file
+        for d in datalist:
+            f.write(d)
+        self.count += 1
 
+    def _write(self, bin, type, content):
         if verbose:
             log('>')
-
+
+        out = []
+
         sz = len(content)
         szbits = (sz & 0x0f) | (_typemap[type]<<4)
         sz >>= 4
         while 1:
             if sz: szbits |= 0x80
-            f.write(chr(szbits))
+            out.append(chr(szbits))
             if not sz:
                 break
             szbits = sz & 0x7f
             sz >>= 7
-        
+
         z = zlib.compressobj(1)
-        f.write(z.compress(content))
-        f.write(z.flush())
+        out.append(z.compress(content))
+        out.append(z.flush())
 
-        self.count += 1
-        self.binlist.append(bin)
+        self._raw_write(out)
         return bin
 
     def write(self, type, content):
@@ -223,18 +228,37 @@ def close(self):
         out = p.stdout.read().strip()
         if p.wait() or not out:
             raise Exception('git index-pack returned an error')
-        nameprefix = repodir('objects/pack/%s' % out)
+        nameprefix = repo('objects/pack/%s' % out)
         os.rename(self.filename + '.pack', nameprefix + '.pack')
         os.rename(self.filename + '.idx', nameprefix + '.idx')
         return nameprefix
 
 
+class PackWriter_Remote(PackWriter):
+    def __init__(self, file, objcache=None):
+        PackWriter.__init__(self, objcache)
+        self.file = file
+        self.filename = 'remote socket'
+
+    def close(self):
+        if self.file:
+            self.file.write('\0\0\0\0')
+            self.file.flush()
+        self.file = None
+
+    def _raw_write(self, datalist):
+        assert(self.file)
+        data = ''.join(datalist)
+        assert(len(data))
+        self.file.write(struct.pack('!I', len(data)) + data)
+
+
 def _git_date(date):
     return time.strftime('%s %z', time.localtime(date))
 
 
 def _gitenv():
-    os.environ['GIT_DIR'] = os.path.abspath(repodir())
+    os.environ['GIT_DIR'] = os.path.abspath(repo())
 
 
 def _read_ref(refname):
@@ -259,9 +283,15 @@ def _update_ref(refname, newval, oldval):
 
 
 def init_repo():
-    d = repodir()
+    d = repo()
     if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
         raise Exception('"%d" exists but is not a directory\n' % d)
     p = subprocess.Popen(['git', 'init', '--bare'],
                          preexec_fn = _gitenv)
     return p.wait()
+
+
+def check_repo_or_die():
+    if not os.path.isdir(repo('objects/pack/.')):
+        log('error: %r is not a git repository\n' % repo())
+        exit(15)

helpers.py

@@ -45,3 +45,11 @@ def hostname():
         except OSError:
             pass
     return _hostname or 'localhost'
+
+
+def linereader(f):
+    while 1:
+        line = f.readline()
+        if not line:
+            break
+        yield line[:-1]