Permalink
Browse files

Merge commit '6f02181'

* commit '6f02181':
  helpers: separately determine if stdout and stderr are ttys.
  cmd/newliner: restrict progress lines to the screen width.
  hashsplit: use shorter offset-filenames inside trees.
  Replace 040000 and 0100644 constants with GIT_MODE_{TREE,FILE}
  git.py: rename treeparse to tree_decode() and add tree_encode().
  hashsplit.py: remove PackWriter-specific knowledge.
  cmd/split: fixup progress message, and print -b output incrementally.
  hashsplit.py: convert from 'bits' to 'level' earlier in the sequence.
  hashsplit.py: okay, *really* fix BLOB_MAX.
  hashsplit.py: simplify code and fix BLOB_MAX handling.
  options.py: o.fatal(): print error after, not before, usage message.
  options.py: make --usage just print the usage message.
  • Loading branch information...
2 parents 1baade3 + 6f02181 commit f3e3307ebee6b88b118a447b0f999232eb948696 @apenwarr apenwarr committed Feb 26, 2011
Showing with 211 additions and 171 deletions.
  1. +2 −2 cmd/fsck-cmd.py
  2. +2 −2 cmd/index-cmd.py
  3. +3 −0 cmd/newliner-cmd.py
  4. +10 −8 cmd/save-cmd.py
  5. +27 −18 cmd/split-cmd.py
  6. +5 −3 lib/bup/_helpers.c
  7. +0 −1 lib/bup/client.py
  8. +54 −41 lib/bup/git.py
  9. +61 −69 lib/bup/hashsplit.py
  10. +18 −3 lib/bup/helpers.py
  11. +1 −0 lib/bup/index.py
  12. +6 −5 lib/bup/options.py
  13. +11 −11 lib/bup/vfs.py
  14. +6 −7 main.py
  15. +5 −1 t/test.sh
View
@@ -36,7 +36,7 @@ def par2_setup():
def parv(lvl):
if opt.verbose >= lvl:
- if istty:
+ if istty2:
return []
else:
return ['-q']
@@ -203,6 +203,6 @@ def do_pack(base, last):
if not opt.verbose:
progress('fsck (%d/%d)\r' % (count, len(extra)))
-if not opt.verbose and istty:
+if not opt.verbose and istty2:
log('fsck done. \n')
sys.exit(code)
View
@@ -2,7 +2,7 @@
import sys, stat, time, os
from bup import options, git, index, drecurse
from bup.helpers import *
-
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
class IterHelper:
def __init__(self, l):
@@ -57,7 +57,7 @@ def update_index(top, excluded_paths):
hashgen = None
if opt.fake_valid:
def hashgen(name):
- return (0100644, index.FAKE_SHA)
+ return (GIT_MODE_FILE, index.FAKE_SHA)
total = 0
bup_dir = os.path.abspath(git.repo())
View
@@ -15,6 +15,7 @@
r = re.compile(r'([\r\n])')
lastlen = 0
all = ''
+width = options._tty_width() or 78
while 1:
l = r.split(all, 1)
if len(l) <= 1:
@@ -32,6 +33,8 @@
else:
assert(len(l) == 3)
(line, splitchar, all) = l
+ if splitchar == '\r':
+ line = line[:width]
sys.stdout.write('%-*s%s' % (lastlen, line, splitchar))
if splitchar == '\r':
lastlen = len(line)
View
@@ -2,6 +2,7 @@
import sys, stat, time, math
from bup import hashsplit, git, options, index, client
from bup.helpers import *
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
optspec = """
@@ -30,7 +31,7 @@
if not extra:
o.fatal("no filenames given")
-opt.progress = (istty and not opt.quiet)
+opt.progress = (istty2 and not opt.quiet)
opt.smaller = parse_num(opt.smaller or 0)
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
@@ -99,8 +100,9 @@ def _pop(force_tree):
shalist = shalists.pop()
tree = force_tree or w.new_tree(shalist)
if shalists:
- shalists[-1].append(('40000',
- git.mangle_name(part, 040000, 40000),
+ shalists[-1].append((GIT_MODE_TREE,
+ git.mangle_name(part,
+ GIT_MODE_TREE, GIT_MODE_TREE),
tree))
else: # this was the toplevel, so put it back for sanity
shalists.append(shalist)
@@ -237,7 +239,7 @@ def wantrecurse_during(ent):
if lastskip_name and lastskip_name.startswith(ent.name):
ent.invalidate()
else:
- ent.validate(040000, newtree)
+ ent.validate(GIT_MODE_TREE, newtree)
ent.repack()
if exists and wasmissing:
count += oldsize
@@ -246,9 +248,8 @@ def wantrecurse_during(ent):
# it's not a directory
id = None
if hashvalid:
- mode = '%o' % ent.gitmode
id = ent.sha
- shalists[-1].append((mode,
+ shalists[-1].append((ent.gitmode,
git.mangle_name(file, ent.mode, ent.gitmode),
id))
else:
@@ -263,7 +264,8 @@ def wantrecurse_during(ent):
lastskip_name = ent.name
else:
try:
- (mode, id) = hashsplit.split_to_blob_or_tree(w, [f],
+ (mode, id) = hashsplit.split_to_blob_or_tree(
+ w.new_blob, w.new_tree, [f],
keep_boundaries=False)
except IOError, e:
add_error('%s: %s' % (ent.name, e))
@@ -286,7 +288,7 @@ def wantrecurse_during(ent):
add_error(Exception('skipping special file "%s"' % ent.name))
lastskip_name = ent.name
if id:
- ent.validate(int(mode, 8), id)
+ ent.validate(mode, id)
ent.repack()
shalists[-1].append((mode,
git.mangle_name(file, ent.mode, ent.gitmode),
View
@@ -5,20 +5,22 @@
optspec = """
-bup split [-tcb] [-n name] [--bench] [filenames...]
+bup split <-t|-c|-b|-n name|--copy|--noop> [--bench] [filenames...]
--
-r,remote= remote repository path
+ Modes:
b,blobs output a series of blob ids
t,tree output a tree id
c,commit output a commit id
-n,name= name of backup set to update (if any)
+n,name= save the result under the given name
+noop split the input, but throw away the result
+copy split the input, copy it to stdout, don't save to repo
+ Options:
+r,remote= remote repository path
d,date= date for the commit (seconds since the epoch)
q,quiet don't print progress messages
v,verbose increase log output (can be used more than once)
git-ids read a list of git object ids from stdin and split their contents
keep-boundaries don't let one chunk span two input files
-noop don't actually save the data anywhere
-copy just copy input to output, hashsplitting along the way
bench print benchmark timings to stderr
max-pack-size= maximum bytes in a single pack
max-pack-objects= maximum number of objects in a single pack
@@ -36,16 +38,18 @@
if (opt.noop or opt.copy) and (opt.blobs or opt.tree or
opt.commit or opt.name):
o.fatal('-N and --copy are incompatible with -b, -t, -c, -n')
+if opt.blobs and (opt.tree or opt.commit or opt.name):
+ o.fatal('-b is incompatible with -t, -c, -n')
if extra and opt.git_ids:
o.fatal("don't provide filenames when using --git-ids")
if opt.verbose >= 2:
git.verbose = opt.verbose - 1
opt.bench = 1
if opt.max_pack_size:
- hashsplit.max_pack_size = parse_num(opt.max_pack_size)
+ git.max_pack_size = parse_num(opt.max_pack_size)
if opt.max_pack_objects:
- hashsplit.max_pack_objects = parse_num(opt.max_pack_objects)
+ git.max_pack_objects = parse_num(opt.max_pack_objects)
if opt.fanout:
hashsplit.fanout = parse_num(opt.fanout)
if opt.blobs:
@@ -123,30 +127,35 @@ def read_ids():
# the input either comes from a series of files or from stdin.
files = extra and (open(fn) for fn in extra) or [sys.stdin]
-if pack_writer:
- shalist = hashsplit.split_to_shalist(pack_writer, files,
+if pack_writer and opt.blobs:
+ shalist = hashsplit.split_to_blobs(pack_writer.new_blob, files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ for (sha, size, level) in shalist:
+ print sha.encode('hex')
+ reprogress()
+elif pack_writer: # tree or commit or name
+ shalist = hashsplit.split_to_shalist(pack_writer.new_blob,
+ pack_writer.new_tree,
+ files,
keep_boundaries=opt.keep_boundaries,
progress=prog)
tree = pack_writer.new_tree(shalist)
else:
last = 0
- for (blob, bits) in hashsplit.hashsplit_iter(files,
- keep_boundaries=opt.keep_boundaries,
- progress=prog):
+ it = hashsplit.hashsplit_iter(files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ for (blob, level) in it:
hashsplit.total_split += len(blob)
if opt.copy:
sys.stdout.write(str(blob))
megs = hashsplit.total_split/1024/1024
if not opt.quiet and last != megs:
- progress('%d Mbytes read\r' % megs)
last = megs
- progress('%d Mbytes read, done.\n' % megs)
if opt.verbose:
log('\n')
-if opt.blobs:
- for (mode,name,bin) in shalist:
- print bin.encode('hex')
if opt.tree:
print tree.encode('hex')
if opt.commit or opt.name:
@@ -171,7 +180,7 @@ def read_ids():
secs = time.time() - start_time
size = hashsplit.total_split
if opt.bench:
- log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
+ log('bup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
% (size/1024., secs, size/1024./secs))
if saved_errors:
View
@@ -9,7 +9,7 @@
#include <stdlib.h>
#include <stdio.h>
-static int istty = 0;
+static int istty2 = 0;
// Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
#if __WIN32__ || __CYGWIN__
@@ -87,6 +87,7 @@ static PyObject *splitbuf(PyObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "t#", &buf, &len))
return NULL;
out = bupsplit_find_ofs(buf, len, &bits);
+ if (out) assert(bits >= BUP_BLOBBITS);
return Py_BuildValue("ii", out, bits);
}
@@ -399,7 +400,7 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
{
struct idx *idx;
uint32_t new_prefix;
- if (count % 102424 == 0 && istty)
+ if (count % 102424 == 0 && istty2)
fprintf(stderr, "midx: writing %.2f%% (%d/%d)\r",
count*100.0/total, count, total);
idx = idxs[last_i];
@@ -654,7 +655,8 @@ static PyMethodDef faster_methods[] = {
PyMODINIT_FUNC init_helpers(void)
{
+ char *e = getenv("BUP_FORCE_TTY");
Py_InitModule("_helpers", faster_methods);
- istty = isatty(2) || getenv("BUP_FORCE_TTY");
+ istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
unpythonize_argv();
}
View
@@ -177,7 +177,6 @@ def sync_indexes(self):
self.sync_index(idx)
git.auto_midx(self.cachedir)
-
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
Oops, something went wrong.

0 comments on commit f3e3307

Please sign in to comment.