Skip to content
Browse files

fsck: add a -j# (run multiple threads) option.

Sort of like make -j.  par2 can be pretty slow, so this lets us verify
multiple files in parallel.  Since the files are so big, though, this might
actually make performance *worse* if you don't have a lot of RAM.  I haven't
benchmarked this too much, but on my quad-core with 6 gigs of RAM, -j4 does
definitely make it go "noticeably" faster.
  • Loading branch information...
1 parent a5de7ca commit 4fcb3c9cb94455fd3273dabc67aa4c4393683654 @apenwarr apenwarr committed Jan 30, 2010
Showing with 68 additions and 31 deletions.
  1. +66 −29 cmd-fsck.py
  2. +2 −2 t/test.sh
View
95 cmd-fsck.py
@@ -78,6 +78,39 @@ def git_verify(base):
return 0
else:
return run(['git', 'verify-pack', '--', base])
+
+
+def do_pack(base, last):
+ code = 0
+ if par2_ok and par2_exists and (opt.repair or not opt.generate):
+ vresult = par2_verify(base)
+ if vresult != 0:
+ if opt.repair:
+ rresult = par2_repair(base)
+ if rresult != 0:
+ print '%s par2 repair: failed (%d)' % (last, rresult)
+ code = rresult
+ else:
+ print '%s par2 repair: succeeded (0)' % last
+ code = 100
+ else:
+ print '%s par2 verify: failed (%d)' % (last, vresult)
+ code = vresult
+ else:
+ print '%s ok' % last
+ elif not opt.generate or (par2_ok and not par2_exists):
+ gresult = git_verify(base)
+ if gresult != 0:
+ print '%s git verify: failed (%d)' % (last, gresult)
+ code = gresult
+ else:
+ if par2_ok and opt.generate:
+ par2_generate(base)
+ print '%s ok' % last
+ else:
+ assert(opt.generate and (not par2_ok or par2_exists))
+ debug(' skipped: par2 file already generated.\n')
+ return code
optspec = """
@@ -87,6 +120,7 @@ def git_verify(base):
g,generate generate auto-repair information using par2
v,verbose increase verbosity (can be used more than once)
quick just check pack sha1sum, don't use git verify-pack
+j,jobs= run 'n' jobs in parallel
par2-ok immediately return 0 if par2 is ok, 1 if not
disable-par2 ignore par2 even if it is available
"""
@@ -110,6 +144,7 @@ def git_verify(base):
code = 0
count = 0
+outstanding = {}
for name in extra:
if name.endswith('.pack'):
base = name[:-5]
@@ -130,36 +165,38 @@ def git_verify(base):
% (last, par2_ok and par2_exists and 'par2' or 'git'))
if not opt.verbose and istty:
log('fsck (%d/%d)\r' % (count, len(extra)))
-
- if par2_ok and par2_exists and (opt.repair or not opt.generate):
- vresult = par2_verify(base)
- if vresult != 0:
- if opt.repair:
- rresult = par2_repair(base)
- if rresult != 0:
- print '%s par2 repair: failed (%d)' % (last, rresult)
- code = code or rresult
- else:
- print '%s par2 repair: succeeded (0)' % last
- code = code or 100
- else:
- print '%s par2 verify: failed (%d)' % (last, vresult)
- code = code or vresult
- else:
- print '%s ok' % last
- elif not opt.generate or (par2_ok and not par2_exists):
- gresult = git_verify(base)
- if gresult != 0:
- print '%s git verify: failed (%d)' % (last, gresult)
- code = code or gresult
- else:
- if par2_ok and opt.generate:
- par2_generate(base)
- print '%s ok' % last
+
+ if not opt.jobs:
+ nc = do_pack(base, last)
+ code = code or nc
+ count += 1
else:
- assert(opt.generate and (not par2_ok or par2_exists))
- debug(' skipped: par2 file already generated.\n')
- count += 1
+ while len(outstanding) >= opt.jobs:
+ (pid,nc) = os.wait()
+ nc >>= 8
+ if pid in outstanding:
+ del outstanding[pid]
+ code = code or nc
+ count += 1
+ pid = os.fork()
+ if pid: # parent
+ outstanding[pid] = 1
+ else: # child
+ try:
+ sys.exit(do_pack(base, last))
+ except Exception, e:
+ log('exception: %r\n' % e)
+ sys.exit(99)
+
+while len(outstanding):
+ (pid,nc) = os.wait()
+ nc >>= 8
+ if pid in outstanding:
+ del outstanding[pid]
+ code = code or nc
+ count += 1
+ if not opt.verbose and istty:
+ log('fsck (%d/%d)\r' % (count, len(extra)))
if not opt.verbose and istty:
log('fsck done. \n')
View
4 t/test.sh
@@ -115,10 +115,10 @@ WVFAIL bup fsck
WVFAIL bup fsck --disable-par2
chmod u+w $BUP_DIR/objects/pack/*.idx
WVPASS bup damage $BUP_DIR/objects/pack/*.idx -n10 -s1 -S0
-WVFAIL bup fsck
+WVFAIL bup fsck -j4
WVPASS bup damage $BUP_DIR/objects/pack/*.pack -n10 -s1024 --percent 0.4 -S0
WVFAIL bup fsck
-WVFAIL bup fsck -rvv # fails because repairs were needed
+WVFAIL bup fsck -rvv -j99 # fails because repairs were needed
if bup fsck --par2-ok; then
WVPASS bup fsck -r # ok because of repairs from last time
WVPASS bup damage $BUP_DIR/objects/pack/*.pack -n201 -s1 --equal -S0

0 comments on commit 4fcb3c9

Please sign in to comment.
Something went wrong with that request. Please try again.