client.py,git.py: run 'bup midx -a' automatically sometimes.

Now that 'bup midx -a' is smarter, we should run it automatically after
creating a new index file.  This should remove the need for running it by
hand.

Thus, we also remove 'bup midx' from the lists of commonly-used subcommands.
(While we're here, let's take out 'split' and 'join' too; you should be
using 'index' and 'save' most of the time.)

Signed-off-by: Avery Pennarun <apenwarr@gmail.com>

Documentation/bup-midx.md

@@ -15,13 +15,13 @@ bup midx [-o *outfile*] <-a|-f|*idxnames*...>
 `bup midx` creates a multi-index (.midx) file from one or more
 git pack index (.idx) files.
 
-You should run this command
-occasionally to ensure your backups run quickly and without
-requiring too much RAM.
+Note: you should no longer need to run this command by hand. 
+It gets run automatically by `bup-save`(1) and similar
+commands.
 
 # OPTIONS
 
--o, --output
+-o, --output=*filename.midx*
 :   use the given output filename for the .midx file. 
     Default is auto-generated.
 
@@ -35,6 +35,11 @@ requiring too much RAM.
     already exist.  This will result in the fastest backup
     performance, but may take a long time to run.
 
+--dir=*packdir*
+:   specify the directory containing the .idx/.midx files
+    to work with.  The default is $BUP_DIR/objects/pack and
+    $BUP_DIR/indexcache/*.
+
 --max-files
 :   maximum number of .idx files to open at a time.  You
     can use this if you have an especially small number of file
@@ -86,10 +91,6 @@ objects that *do* exist can be optimized; for example,
 consecutive objects are often stored in the same pack, so
 we can search that one first using an MRU algorithm.)
 
-With large repositories, you should be sure to run
-`bup midx -a` or `bup midx -f` every now and then so that
-creating backups will remain efficient.
-
 
 # SEE ALSO
 

Documentation/bup.md

@@ -49,8 +49,6 @@ pages.
 :   Print detailed help for the given command
 `bup-index`(1)
 :   Create or display the index of files to back up
-`bup-midx`(1)
-:   Index objects to speed up future backups
 `bup-on`(1)
 :   Backup a remote machine to the local one
 `bup-save`(1)
@@ -75,6 +73,8 @@ pages.
 :   Determine how close your bup repository is to armageddon
 `bup-memtest`(1)
 :   Test bup memory usage statistics
+`bup-midx`(1)
+:   Index objects to speed up future backups
 `bup-newliner`(1)
 :   Make sure progress messages don't overlap with output
 `bup-random`(1)

cmd/midx-cmd.py

@@ -13,6 +13,7 @@
 a,auto     automatically create .midx from any unindexed .idx files
 f,force    automatically create .midx from *all* .idx files
 max-files= maximum number of idx files to open at once [-1]
+dir=       directory containing idx/midx files
 """
 
 def _group(l, count):
@@ -149,9 +150,9 @@ def do_midx_dir(path):
     DESIRED_HWM = opt.force and 1 or 5
     DESIRED_LWM = opt.force and 1 or 2
     existed = dict((name,1) for sz,name in all)
-    log('%d indexes; want no more than %d.\n' % (len(all), DESIRED_HWM))
+    log('midx: %d indexes; want no more than %d.\n' % (len(all), DESIRED_HWM))
     if len(all) <= DESIRED_HWM:
-        log('Nothing to do.\n')
+        log('midx: nothing to do.\n')
     while len(all) > DESIRED_HWM:
         all.sort()
         part1 = [name for sz,name in all[:len(all)-DESIRED_LWM+1]]
@@ -189,8 +190,11 @@ def do_midx_group(outdir, infiles):
 if extra:
     do_midx(git.repo('objects/pack'), opt.output, extra)
 elif opt.auto or opt.force:
-    paths = [git.repo('objects/pack')]
-    paths += glob.glob(git.repo('index-cache/*/.'))
+    if opt.dir:
+        paths = [opt.dir]
+    else:
+        paths = [git.repo('objects/pack')]
+        paths += glob.glob(git.repo('index-cache/*/.'))
     for path in paths:
         log('midx: scanning %s\n' % path)
         do_midx_dir(path)

lib/bup/client.py

@@ -156,6 +156,7 @@ def sync_index(self, name):
         self.check_ok()
         f.close()
         os.rename(fn + '.tmp', fn)
+        git.auto_midx(self.cachedir)
 
     def _make_objcache(self):
         ob = self._busy

lib/bup/git.py

@@ -39,6 +39,14 @@ def repo(sub = ''):
     return os.path.join(repodir, sub)
 
 
+def auto_midx(objdir):
+    main_exe = os.environ.get('BUP_MAIN_EXE') or sys.argv[0]
+    args = [main_exe, 'midx', '--auto', '--dir', objdir]
+    rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
+    if rv:
+        add_error('%r: returned %d' % (args, rv))
+
+
 def mangle_name(name, mode, gitmode):
     """Mangle a file name to present an abstract name for segmented files.
     Mangled file names will have the ".bup" extension added to them. If a
@@ -599,6 +607,8 @@ def _end(self):
             os.unlink(self.filename + '.map')
         os.rename(self.filename + '.pack', nameprefix + '.pack')
         os.rename(self.filename + '.idx', nameprefix + '.idx')
+
+        auto_midx(repo('objects/pack'))
         return nameprefix
 
     def close(self):

main.py

@@ -35,12 +35,8 @@ def usage():
         fuse = 'Mount your backup sets as a filesystem',
         help = 'Print detailed help for the given command',
         index = 'Create or display the index of files to back up',
-        join = 'Retrieve a file backed up using "bup split"',
-        ls = 'Browse the files in your backup sets',
-        midx = 'Index objects to speed up future backups',
         on = 'Backup a remote machine to the local one',
         save = 'Save files into a backup set (note: run "bup index" first)',
-        split = 'Split a single file into its own backup set',
         web = 'Launch a web server to examine backup sets',
     )