Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Introduce BUP_DEBUG, --debug, and tone down the log messages a lot.

There's a new global bup option, --debug (-D) that increments BUP_DEBUG.  If
BUP_DEBUG >=1, debug1() prints; if >= 2, debug2() prints.

We change a bunch of formerly-always-printing log() messages to debug1 or
debug2, so now a typical bup session should be a lot less noisy.

This affects midx in particular, which was *way* too noisy now that 'bup
save' and 'bup server' were running it automatically every now and then.

Signed-off-by: Avery Pennarun <apenwarr@gmail.com>
  • Loading branch information...
commit e3d646243627020f5b99e6d7a0b78e2584b422d1 1 parent 65cc02d
apenwarr apenwarr authored
51 cmd/midx-cmd.py
@@ -12,6 +12,7 @@
12 12 o,output= output midx filename (default: auto-generated)
13 13 a,auto automatically create .midx from any unindexed .idx files
14 14 f,force automatically create .midx from *all* .idx files
  15 +p,print print names of generated midx files
15 16 max-files= maximum number of idx files to open at once [-1]
16 17 dir= directory containing idx/midx files
17 18 """
@@ -32,14 +33,14 @@ def max_files():
32 33
33 34 def merge(idxlist, bits, table):
34 35 count = 0
35   - for e in git.idxmerge(idxlist):
  36 + for e in git.idxmerge(idxlist, final_progress=False):
36 37 count += 1
37 38 prefix = git.extract_bits(e, bits)
38 39 table[prefix] = count
39 40 yield e
40 41
41 42
42   -def _do_midx(outdir, outfilename, infilenames):
  43 +def _do_midx(outdir, outfilename, infilenames, prefixstr):
43 44 if not outfilename:
44 45 assert(outdir)
45 46 sum = Sha1('\0'.join(infilenames)).hexdigest()
@@ -55,17 +56,18 @@ def _do_midx(outdir, outfilename, infilenames):
55 56 inp.append(ix)
56 57 total += len(ix)
57 58
58   - log('Merging %d indexes (%d objects).\n' % (len(infilenames), total))
  59 + log('midx: %smerging %d indexes (%d objects).\n'
  60 + % (prefixstr, len(infilenames), total))
59 61 if (not opt.force and (total < 1024 and len(infilenames) < 3)) \
60 62 or len(infilenames) < 2 \
61 63 or (opt.force and not total):
62   - log('midx: nothing to do.\n')
  64 + debug1('midx: nothing to do.\n')
63 65 return
64 66
65 67 pages = int(total/SHA_PER_PAGE) or 1
66 68 bits = int(math.ceil(math.log(pages, 2)))
67 69 entries = 2**bits
68   - log('Table size: %d (%d bits)\n' % (entries*4, bits))
  70 + debug1('midx: table size: %d (%d bits)\n' % (entries*4, bits))
69 71
70 72 table = [0]*entries
71 73
@@ -103,9 +105,9 @@ def _do_midx(outdir, outfilename, infilenames):
103 105 return total,outfilename
104 106
105 107
106   -def do_midx(outdir, outfilename, infilenames):
107   - rv = _do_midx(outdir, outfilename, infilenames)
108   - if rv:
  108 +def do_midx(outdir, outfilename, infilenames, prefixstr):
  109 + rv = _do_midx(outdir, outfilename, infilenames, prefixstr)
  110 + if rv and opt['print']:
109 111 print rv[1]
110 112
111 113
@@ -133,7 +135,7 @@ def do_midx_dir(path):
133 135 already[iname] = 1
134 136 any = 1
135 137 if not any:
136   - log('%r is redundant\n' % mname)
  138 + debug1('%r is redundant\n' % mname)
137 139 unlink(mname)
138 140 already[mname] = 1
139 141
@@ -150,30 +152,37 @@ def do_midx_dir(path):
150 152 DESIRED_HWM = opt.force and 1 or 5
151 153 DESIRED_LWM = opt.force and 1 or 2
152 154 existed = dict((name,1) for sz,name in all)
153   - log('midx: %d indexes; want no more than %d.\n' % (len(all), DESIRED_HWM))
  155 + debug1('midx: %d indexes; want no more than %d.\n'
  156 + % (len(all), DESIRED_HWM))
154 157 if len(all) <= DESIRED_HWM:
155   - log('midx: nothing to do.\n')
  158 + debug1('midx: nothing to do.\n')
156 159 while len(all) > DESIRED_HWM:
157 160 all.sort()
158 161 part1 = [name for sz,name in all[:len(all)-DESIRED_LWM+1]]
159 162 part2 = all[len(all)-DESIRED_LWM+1:]
160 163 all = list(do_midx_group(path, part1)) + part2
161 164 if len(all) > DESIRED_HWM:
162   - log('\nStill too many indexes (%d > %d). Merging again.\n'
163   - % (len(all), DESIRED_HWM))
  165 + debug1('\nStill too many indexes (%d > %d). Merging again.\n'
  166 + % (len(all), DESIRED_HWM))
164 167
165   - for sz,name in all:
166   - if not existed.get(name):
167   - print name
  168 + if opt['print']:
  169 + for sz,name in all:
  170 + if not existed.get(name):
  171 + print name
168 172
169 173
170 174 def do_midx_group(outdir, infiles):
171   - for sublist in _group(infiles, opt.max_files):
172   - rv = _do_midx(path, None, sublist)
  175 + groups = list(_group(infiles, opt.max_files))
  176 + gprefix = ''
  177 + for n,sublist in enumerate(groups):
  178 + if len(groups) != 1:
  179 + gprefix = 'Group %d: ' % (n+1)
  180 + rv = _do_midx(path, None, sublist, gprefix)
173 181 if rv:
174 182 yield rv
175 183
176 184
  185 +handle_ctrl_c()
177 186
178 187 o = options.Options('bup midx', optspec)
179 188 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -188,7 +197,7 @@ def do_midx_group(outdir, infiles):
188 197 assert(opt.max_files >= 5)
189 198
190 199 if extra:
191   - do_midx(git.repo('objects/pack'), opt.output, extra)
  200 + do_midx(git.repo('objects/pack'), opt.output, extra, '')
192 201 elif opt.auto or opt.force:
193 202 if opt.dir:
194 203 paths = [opt.dir]
@@ -196,8 +205,8 @@ def do_midx_group(outdir, infiles):
196 205 paths = [git.repo('objects/pack')]
197 206 paths += glob.glob(git.repo('index-cache/*/.'))
198 207 for path in paths:
199   - log('midx: scanning %s\n' % path)
  208 + debug1('midx: scanning %s\n' % path)
200 209 do_midx_dir(path)
201   - log('\n')
  210 + debug1('\n')
202 211 else:
203 212 o.fatal("you must use -f or -a or provide input filenames")
22 cmd/server-cmd.py
@@ -8,13 +8,13 @@
8 8
9 9 def init_dir(conn, arg):
10 10 git.init_repo(arg)
11   - log('bup server: bupdir initialized: %r\n' % git.repodir)
  11 + debug1('bup server: bupdir initialized: %r\n' % git.repodir)
12 12 conn.ok()
13 13
14 14
15 15 def set_dir(conn, arg):
16 16 git.check_repo_or_die(arg)
17   - log('bup server: bupdir is %r\n' % git.repodir)
  17 + debug1('bup server: bupdir is %r\n' % git.repodir)
18 18 conn.ok()
19 19
20 20
@@ -51,9 +51,9 @@ def receive_objects(conn, junk):
51 51 w.abort()
52 52 raise Exception('object read: expected length header, got EOF\n')
53 53 n = struct.unpack('!I', ns)[0]
54   - #log('expecting %d bytes\n' % n)
  54 + #debug2('expecting %d bytes\n' % n)
55 55 if not n:
56   - log('bup server: received %d object%s.\n'
  56 + debug1('bup server: received %d object%s.\n'
57 57 % (w.count, w.count!=1 and "s" or ''))
58 58 fullpath = w.close()
59 59 if fullpath:
@@ -62,13 +62,13 @@ def receive_objects(conn, junk):
62 62 conn.ok()
63 63 return
64 64 elif n == 0xffffffff:
65   - log('bup server: receive-objects suspended.\n')
  65 + debug2('bup server: receive-objects suspended.\n')
66 66 suspended_w = w
67 67 conn.ok()
68 68 return
69 69
70 70 buf = conn.read(n) # object sizes in bup are reasonably small
71   - #log('read %d bytes\n' % n)
  71 + #debug2('read %d bytes\n' % n)
72 72 if len(buf) < n:
73 73 w.abort()
74 74 raise Exception('object read: expected %d bytes, got %d\n'
@@ -90,7 +90,7 @@ def receive_objects(conn, junk):
90 90 # efficient.
91 91 w.objcache.refresh(skip_midx = True)
92 92 oldpack = w.objcache.exists(sha)
93   - log('new suggestion: %r\n' % oldpack)
  93 + debug2('new suggestion: %r\n' % oldpack)
94 94 assert(oldpack)
95 95 assert(oldpack != True)
96 96 assert(not oldpack.endswith('.midx'))
@@ -99,7 +99,7 @@ def receive_objects(conn, junk):
99 99 assert(oldpack.endswith('.idx'))
100 100 (dir,name) = os.path.split(oldpack)
101 101 if not (name in suggested):
102   - log("bup server: suggesting index %s\n" % name)
  102 + debug1("bup server: suggesting index %s\n" % name)
103 103 conn.write('index %s\n' % name)
104 104 suggested[name] = 1
105 105 else:
@@ -150,7 +150,7 @@ def cat(conn, id):
150 150 if extra:
151 151 o.fatal('no arguments expected')
152 152
153   -log('bup server: reading from stdin.\n')
  153 +debug2('bup server: reading from stdin.\n')
154 154
155 155 commands = {
156 156 'init-dir': init_dir,
@@ -171,7 +171,7 @@ def cat(conn, id):
171 171 line = _line.strip()
172 172 if not line:
173 173 continue
174   - log('bup server: command: %r\n' % line)
  174 + debug1('bup server: command: %r\n' % line)
175 175 words = line.split(' ', 1)
176 176 cmd = words[0]
177 177 rest = len(words)>1 and words[1] or ''
@@ -184,4 +184,4 @@ def cat(conn, id):
184 184 else:
185 185 raise Exception('unknown server command: %r\n' % line)
186 186
187   -log('bup server: done\n')
  187 +debug1('bup server: done\n')
12 lib/bup/client.py
@@ -134,11 +134,11 @@ def sync_indexes_del(self):
134 134 mkdirp(self.cachedir)
135 135 for f in os.listdir(self.cachedir):
136 136 if f.endswith('.idx') and not f in all:
137   - log('pruning old index: %r\n' % f)
  137 + debug1('client: pruning old index: %r\n' % f)
138 138 os.unlink(os.path.join(self.cachedir, f))
139 139
140 140 def sync_index(self, name):
141   - #log('requesting %r\n' % name)
  141 + #debug1('requesting %r\n' % name)
142 142 self.check_busy()
143 143 mkdirp(self.cachedir)
144 144 self.conn.write('send-index %s\n' % name)
@@ -147,12 +147,12 @@ def sync_index(self, name):
147 147 fn = os.path.join(self.cachedir, name)
148 148 f = open(fn + '.tmp', 'w')
149 149 count = 0
150   - progress('Receiving index: %d/%d\r' % (count, n))
  150 + progress('Receiving index from server: %d/%d\r' % (count, n))
151 151 for b in chunkyreader(self.conn, n):
152 152 f.write(b)
153 153 count += len(b)
154   - progress('Receiving index: %d/%d\r' % (count, n))
155   - progress('Receiving index: %d/%d, done.\n' % (count, n))
  154 + progress('Receiving index from server: %d/%d\r' % (count, n))
  155 + progress('Receiving index from server: %d/%d, done.\n' % (count, n))
156 156 self.check_ok()
157 157 f.close()
158 158 os.rename(fn + '.tmp', fn)
@@ -166,7 +166,7 @@ def _make_objcache(self):
166 166 return git.PackIdxList(self.cachedir)
167 167
168 168 def _suggest_pack(self, indexname):
169   - log('received index suggestion: %s\n' % indexname)
  169 + debug1('client: received index suggestion: %s\n' % indexname)
170 170 ob = self._busy
171 171 if ob:
172 172 assert(ob == 'receive-objects')
11 lib/bup/git.py
@@ -383,8 +383,8 @@ def refresh(self, skip_midx = False):
383 383 any += 1
384 384 break
385 385 if not any and not ix.force_keep:
386   - log('midx: removing redundant: %s\n'
387   - % os.path.basename(ix.name))
  386 + debug1('midx: removing redundant: %s\n'
  387 + % os.path.basename(ix.name))
388 388 unlink(ix.name)
389 389 for f in os.listdir(self.dir):
390 390 full = os.path.join(self.dir, f)
@@ -392,7 +392,7 @@ def refresh(self, skip_midx = False):
392 392 ix = PackIdx(full)
393 393 d[full] = ix
394 394 self.packs = list(set(d.values()))
395   - log('PackIdxList: using %d index%s.\n'
  395 + debug1('PackIdxList: using %d index%s.\n'
396 396 % (len(self.packs), len(self.packs)!=1 and 'es' or ''))
397 397
398 398 def add(self, hash):
@@ -429,7 +429,7 @@ def open_idx(filename):
429 429 raise GitError('idx filenames must end with .idx or .midx')
430 430
431 431
432   -def idxmerge(idxlist):
  432 +def idxmerge(idxlist, final_progress=True):
433 433 """Generate a list of all the objects reachable in a PackIdxList."""
434 434 total = sum(len(i) for i in idxlist)
435 435 iters = (iter(i) for i in idxlist)
@@ -451,7 +451,8 @@ def idxmerge(idxlist):
451 451 heapq.heapreplace(heap, (e, it))
452 452 else:
453 453 heapq.heappop(heap)
454   - log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
  454 + if final_progress:
  455 + log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
455 456
456 457
457 458 class PackWriter:
29 lib/bup/helpers.py
@@ -3,6 +3,17 @@
3 3 from bup import _version
4 4
5 5
  6 +def atoi(s):
  7 + """Convert the string 's' to an integer. Return 0 if s is not a number."""
  8 + try:
  9 + return int(s or '0')
  10 + except ValueError:
  11 + return 0
  12 +
  13 +
  14 +buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
  15 +
  16 +
6 17 # Write (blockingly) to sockets that may or may not be in blocking mode.
7 18 # We need this because our stderr is sometimes eaten by subprocesses
8 19 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
@@ -26,6 +37,16 @@ def log(s):
26 37 _hard_write(sys.stderr.fileno(), s)
27 38
28 39
  40 +def debug1(s):
  41 + if buglvl >= 1:
  42 + log(s)
  43 +
  44 +
  45 +def debug2(s):
  46 + if buglvl >= 2:
  47 + log(s)
  48 +
  49 +
29 50 def mkdirp(d, mode=None):
30 51 """Recursively create directories on path 'd'.
31 52
@@ -305,14 +326,6 @@ def count(l):
305 326 return reduce(lambda x,y: x+1, l)
306 327
307 328
308   -def atoi(s):
309   - """Convert the string 's' to an integer. Return 0 if s is not a number."""
310   - try:
311   - return int(s or '0')
312   - except ValueError:
313   - return 0
314   -
315   -
316 329 saved_errors = []
317 330 def add_error(e):
318 331 """Append an error message to the list of saved errors.
5 lib/bup/ssh.py
@@ -28,10 +28,11 @@ def connect(rhost, subcmd):
28 28 # allow shellquoting. So we end up having to double-shellquote
29 29 # stuff here.
30 30 escapedir = re.sub(r'([^\w/])', r'\\\\\\\1', nicedir)
  31 + buglvl = helpers.atoi(os.environ.get('BUP_DEBUG'))
31 32 force_tty = helpers.atoi(os.environ.get('BUP_FORCE_TTY'))
32 33 cmd = r"""
33   - sh -c PATH=%s:'$PATH BUP_FORCE_TTY=%s bup %s'
34   - """ % (escapedir, force_tty, subcmd)
  34 + sh -c PATH=%s:'$PATH BUP_DEBUG=%s BUP_FORCE_TTY=%s bup %s'
  35 + """ % (escapedir, buglvl, force_tty, subcmd)
35 36 argv = ['ssh', rhost, '--', cmd.strip()]
36 37 #helpers.log('argv is: %r\n' % argv)
37 38 def setup():
10 lib/bup/vfs.py
@@ -120,7 +120,7 @@ def next(self, size):
120 120 self.blob = self.blob[want:]
121 121 if not self.it:
122 122 break
123   - log('next(%d) returned %d\n' % (size, len(out)))
  123 + debug2('next(%d) returned %d\n' % (size, len(out)))
124 124 self.ofs += len(out)
125 125 return out
126 126
@@ -220,7 +220,7 @@ def fs_top(self):
220 220 return self
221 221
222 222 def _lresolve(self, parts):
223   - #log('_lresolve %r in %r\n' % (parts, self.name))
  223 + #debug2('_lresolve %r in %r\n' % (parts, self.name))
224 224 if not parts:
225 225 return self
226 226 (first, rest) = (parts[0], parts[1:])
@@ -253,7 +253,7 @@ def lresolve(self, path, stay_inside_fs=False):
253 253 parts = re.split(r'/+', path or '.')
254 254 if not parts[-1]:
255 255 parts[-1] = '.'
256   - #log('parts: %r %r\n' % (path, parts))
  256 + #debug2('parts: %r %r\n' % (path, parts))
257 257 return start._lresolve(parts)
258 258
259 259 def resolve(self, path = ''):
@@ -310,12 +310,12 @@ def open(self):
310 310 def size(self):
311 311 """Get this file's size."""
312 312 if self._cached_size == None:
313   - log('<<<<File.size() is calculating...\n')
  313 + debug1('<<<<File.size() is calculating...\n')
314 314 if self.bupmode == git.BUP_CHUNKED:
315 315 self._cached_size = _total_size(self.hash)
316 316 else:
317 317 self._cached_size = _chunk_len(self.hash)
318   - log('<<<<File.size() done.\n')
  318 + debug1('<<<<File.size() done.\n')
319 319 return self._cached_size
320 320
321 321
18 main.py
@@ -23,12 +23,13 @@
23 23 os.environ['BUP_MAIN_EXE'] = os.path.abspath(exe)
24 24 os.environ['BUP_RESOURCE_PATH'] = resourcepath
25 25
  26 +from bup import helpers
26 27 from bup.helpers import *
27 28
28 29
29 30 def usage():
30   - log('Usage: bup [-?|--help] [-d=BUP_DIR|--bup-dir=BUP_DIR] COMMAND [ARGS]'
31   - + '\n\n')
  31 + log('Usage: bup [-?|--help] [-d BUP_DIR] [--debug] '
  32 + '<command> [options...]\n\n')
32 33 common = dict(
33 34 ftp = 'Browse backup sets using an ftp-like client',
34 35 fsck = 'Check backup sets for damage and add redundancy information',
@@ -65,8 +66,8 @@ def usage():
65 66
66 67 # Handle global options.
67 68 try:
68   - global_args, subcmd = getopt.getopt(argv[1:], '?Vd:',
69   - ['help', 'version', 'bup-dir='])
  69 + global_args, subcmd = getopt.getopt(argv[1:], '?VDd:',
  70 + ['help', 'version', 'debug', 'bup-dir='])
70 71 except getopt.GetoptError, ex:
71 72 log('error: ' + ex.msg + '\n')
72 73 usage()
@@ -75,11 +76,14 @@ def usage():
75 76 dest_dir = None
76 77
77 78 for opt in global_args:
78   - if opt[0] == '-?' or opt[0] == '--help':
  79 + if opt[0] in ['-?', '--help']:
79 80 help_requested = True
80   - if opt[0] == '-V' or opt[0] == '--version':
  81 + elif opt[0] in ['-V', '--version']:
81 82 subcmd = ['version']
82   - elif opt[0] == '-d' or opt[0] == '--bup-dir':
  83 + elif opt[0] in ['-D', '--debug']:
  84 + helpers.buglvl += 1
  85 + os.environ['BUP_DEBUG'] = str(helpers.buglvl)
  86 + elif opt[0] in ['-d', '--bup-dir']:
83 87 dest_dir = opt[1]
84 88 else:
85 89 log('error: unexpected option "%s"\n' % opt[0])

0 comments on commit e3d6462

Please sign in to comment.
Something went wrong with that request. Please try again.