Merge commit '6f02181'

* commit '6f02181':
  helpers: separately determine if stdout and stderr are ttys.
  cmd/newliner: restrict progress lines to the screen width.
  hashsplit: use shorter offset-filenames inside trees.
  Replace 040000 and 0100644 constants with GIT_MODE_{TREE,FILE}
  git.py: rename treeparse to tree_decode() and add tree_encode().
  hashsplit.py: remove PackWriter-specific knowledge.
  cmd/split: fixup progress message, and print -b output incrementally.
  hashsplit.py: convert from 'bits' to 'level' earlier in the sequence.
  hashsplit.py: okay, *really* fix BLOB_MAX.
  hashsplit.py: simplify code and fix BLOB_MAX handling.
  options.py: o.fatal(): print error after, not before, usage message.
  options.py: make --usage just print the usage message.

cmd/fsck-cmd.py

@@ -36,7 +36,7 @@ def par2_setup():
 
 def parv(lvl):
     if opt.verbose >= lvl:
-        if istty:
+        if istty2:
             return []
         else:
             return ['-q']
@@ -203,6 +203,6 @@ def do_pack(base, last):
     if not opt.verbose:
         progress('fsck (%d/%d)\r' % (count, len(extra)))
 
-if not opt.verbose and istty:
+if not opt.verbose and istty2:
     log('fsck done.           \n')
 sys.exit(code)

cmd/index-cmd.py

@@ -2,7 +2,7 @@
 import sys, stat, time, os
 from bup import options, git, index, drecurse
 from bup.helpers import *
-
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
 
 class IterHelper:
     def __init__(self, l):
@@ -57,7 +57,7 @@ def update_index(top, excluded_paths):
     hashgen = None
     if opt.fake_valid:
         def hashgen(name):
-            return (0100644, index.FAKE_SHA)
+            return (GIT_MODE_FILE, index.FAKE_SHA)
 
     total = 0
     bup_dir = os.path.abspath(git.repo())

cmd/newliner-cmd.py

@@ -15,6 +15,7 @@
 r = re.compile(r'([\r\n])')
 lastlen = 0
 all = ''
+width = options._tty_width() or 78
 while 1:
     l = r.split(all, 1)
     if len(l) <= 1:
@@ -32,6 +33,8 @@
     else:
         assert(len(l) == 3)
         (line, splitchar, all) = l
+        if splitchar == '\r':
+            line = line[:width]
         sys.stdout.write('%-*s%s' % (lastlen, line, splitchar))
         if splitchar == '\r':
             lastlen = len(line)

cmd/save-cmd.py

@@ -2,6 +2,7 @@
 import sys, stat, time, math
 from bup import hashsplit, git, options, index, client
 from bup.helpers import *
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
 
 
 optspec = """
@@ -30,7 +31,7 @@
 if not extra:
     o.fatal("no filenames given")
 
-opt.progress = (istty and not opt.quiet)
+opt.progress = (istty2 and not opt.quiet)
 opt.smaller = parse_num(opt.smaller or 0)
 if opt.bwlimit:
     client.bwlimit = parse_num(opt.bwlimit)
@@ -99,8 +100,9 @@ def _pop(force_tree):
     shalist = shalists.pop()
     tree = force_tree or w.new_tree(shalist)
     if shalists:
-        shalists[-1].append(('40000',
-                             git.mangle_name(part, 040000, 40000),
+        shalists[-1].append((GIT_MODE_TREE,
+                             git.mangle_name(part,
+                                             GIT_MODE_TREE, GIT_MODE_TREE),
                              tree))
     else:  # this was the toplevel, so put it back for sanity
         shalists.append(shalist)
@@ -237,7 +239,7 @@ def wantrecurse_during(ent):
             if lastskip_name and lastskip_name.startswith(ent.name):
                 ent.invalidate()
             else:
-                ent.validate(040000, newtree)
+                ent.validate(GIT_MODE_TREE, newtree)
             ent.repack()
         if exists and wasmissing:
             count += oldsize
@@ -246,9 +248,8 @@ def wantrecurse_during(ent):
     # it's not a directory
     id = None
     if hashvalid:
-        mode = '%o' % ent.gitmode
         id = ent.sha
-        shalists[-1].append((mode, 
+        shalists[-1].append((ent.gitmode, 
                              git.mangle_name(file, ent.mode, ent.gitmode),
                              id))
     else:
@@ -263,7 +264,8 @@ def wantrecurse_during(ent):
                 lastskip_name = ent.name
             else:
                 try:
-                    (mode, id) = hashsplit.split_to_blob_or_tree(w, [f],
+                    (mode, id) = hashsplit.split_to_blob_or_tree(
+                                            w.new_blob, w.new_tree, [f],
                                             keep_boundaries=False)
                 except IOError, e:
                     add_error('%s: %s' % (ent.name, e))
@@ -286,7 +288,7 @@ def wantrecurse_during(ent):
                 add_error(Exception('skipping special file "%s"' % ent.name))
                 lastskip_name = ent.name
         if id:
-            ent.validate(int(mode, 8), id)
+            ent.validate(mode, id)
             ent.repack()
             shalists[-1].append((mode,
                                  git.mangle_name(file, ent.mode, ent.gitmode),

cmd/split-cmd.py

@@ -5,20 +5,22 @@
 
 
 optspec = """
-bup split [-tcb] [-n name] [--bench] [filenames...]
+bup split <-t|-c|-b|-n name|--copy|--noop> [--bench] [filenames...]
 --
-r,remote=  remote repository path
+ Modes:
 b,blobs    output a series of blob ids
 t,tree     output a tree id
 c,commit   output a commit id
-n,name=    name of backup set to update (if any)
+n,name=    save the result under the given name
+noop       split the input, but throw away the result
+copy       split the input, copy it to stdout, don't save to repo
+ Options:
+r,remote=  remote repository path
 d,date=    date for the commit (seconds since the epoch)
 q,quiet    don't print progress messages
 v,verbose  increase log output (can be used more than once)
 git-ids    read a list of git object ids from stdin and split their contents
 keep-boundaries  don't let one chunk span two input files
-noop       don't actually save the data anywhere
-copy       just copy input to output, hashsplitting along the way
 bench      print benchmark timings to stderr
 max-pack-size=  maximum bytes in a single pack
 max-pack-objects=  maximum number of objects in a single pack
@@ -36,16 +38,18 @@
 if (opt.noop or opt.copy) and (opt.blobs or opt.tree or 
                                opt.commit or opt.name):
     o.fatal('-N and --copy are incompatible with -b, -t, -c, -n')
+if opt.blobs and (opt.tree or opt.commit or opt.name):
+    o.fatal('-b is incompatible with -t, -c, -n')
 if extra and opt.git_ids:
     o.fatal("don't provide filenames when using --git-ids")
 
 if opt.verbose >= 2:
     git.verbose = opt.verbose - 1
     opt.bench = 1
 if opt.max_pack_size:
-    hashsplit.max_pack_size = parse_num(opt.max_pack_size)
+    git.max_pack_size = parse_num(opt.max_pack_size)
 if opt.max_pack_objects:
-    hashsplit.max_pack_objects = parse_num(opt.max_pack_objects)
+    git.max_pack_objects = parse_num(opt.max_pack_objects)
 if opt.fanout:
     hashsplit.fanout = parse_num(opt.fanout)
 if opt.blobs:
@@ -123,30 +127,35 @@ def read_ids():
     # the input either comes from a series of files or from stdin.
     files = extra and (open(fn) for fn in extra) or [sys.stdin]
 
-if pack_writer:
-    shalist = hashsplit.split_to_shalist(pack_writer, files,
+if pack_writer and opt.blobs:
+    shalist = hashsplit.split_to_blobs(pack_writer.new_blob, files,
+                                       keep_boundaries=opt.keep_boundaries,
+                                       progress=prog)
+    for (sha, size, level) in shalist:
+        print sha.encode('hex')
+        reprogress()
+elif pack_writer:  # tree or commit or name
+    shalist = hashsplit.split_to_shalist(pack_writer.new_blob,
+                                         pack_writer.new_tree,
+                                         files,
                                          keep_boundaries=opt.keep_boundaries,
                                          progress=prog)
     tree = pack_writer.new_tree(shalist)
 else:
     last = 0
-    for (blob, bits) in hashsplit.hashsplit_iter(files,
-                                    keep_boundaries=opt.keep_boundaries,
-                                    progress=prog):
+    it = hashsplit.hashsplit_iter(files,
+                                  keep_boundaries=opt.keep_boundaries,
+                                  progress=prog)
+    for (blob, level) in it:
         hashsplit.total_split += len(blob)
         if opt.copy:
             sys.stdout.write(str(blob))
         megs = hashsplit.total_split/1024/1024
         if not opt.quiet and last != megs:
-            progress('%d Mbytes read\r' % megs)
             last = megs
-    progress('%d Mbytes read, done.\n' % megs)
 
 if opt.verbose:
     log('\n')
-if opt.blobs:
-    for (mode,name,bin) in shalist:
-        print bin.encode('hex')
 if opt.tree:
     print tree.encode('hex')
 if opt.commit or opt.name:
@@ -171,7 +180,7 @@ def read_ids():
 secs = time.time() - start_time
 size = hashsplit.total_split
 if opt.bench:
-    log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
+    log('bup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
         % (size/1024., secs, size/1024./secs))
 
 if saved_errors:

lib/bup/_helpers.c

@@ -9,7 +9,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-static int istty = 0;
+static int istty2 = 0;
 
 // Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
 #if __WIN32__ || __CYGWIN__
@@ -87,6 +87,7 @@ static PyObject *splitbuf(PyObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "t#", &buf, &len))
 	return NULL;
     out = bupsplit_find_ofs(buf, len, &bits);
+    if (out) assert(bits >= BUP_BLOBBITS);
     return Py_BuildValue("ii", out, bits);
 }
 
@@ -399,7 +400,7 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
     {
 	struct idx *idx;
 	uint32_t new_prefix;
-	if (count % 102424 == 0 && istty)
+	if (count % 102424 == 0 && istty2)
 	    fprintf(stderr, "midx: writing %.2f%% (%d/%d)\r",
 		    count*100.0/total, count, total);
 	idx = idxs[last_i];
@@ -654,7 +655,8 @@ static PyMethodDef faster_methods[] = {
 
 PyMODINIT_FUNC init_helpers(void)
 {
+    char *e = getenv("BUP_FORCE_TTY");
     Py_InitModule("_helpers", faster_methods);
-    istty = isatty(2) || getenv("BUP_FORCE_TTY");
+    istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
     unpythonize_argv();
 }

lib/bup/client.py

@@ -177,7 +177,6 @@ def sync_indexes(self):
             self.sync_index(idx)
         git.auto_midx(self.cachedir)
 
-
     def sync_index(self, name):
         #debug1('requesting %r\n' % name)
         self.check_busy()