Merge branch 'meta'

* meta:
  Add utimes/lutimes implementations of _helpers utime() and lutime().
  Replace _helpers.utimensat() with utime() and lutime().
  Test for available nanosecond stat timestamp members.
  Add config.h dependency to _helpers in csetup.py.
  Add -*-shell-script-*- to configure.inc.
  Use FS_IOC_GETFLAGS/FS_IOC_SETFLAGS directly as the preprocessor guards.
  Verify the expected length of saved_errors in tmetadata.py.
  Don't use xstat.lutime() in test-meta.sh when xstat.utime() will do.
  Add meta support for restoring filesystem sockets.
  Add _recognized_file_types(); defer error for unrecognized restore.
  index.py: new format (V3), with inodes, link counts, and 64-bit times.
  Cap timestamps in index to avoid needing to worry about fractional parts.
  index.py: factor out an Entry._fixup_time method.
  Rely on options.parse() for more of the meta and xstat argument processing.
  Remove vestigal clean target comment regarding pybuptest.tmp permissions.
  Add initial timespec behavior tests.
  Return None from bup_set_linux_file_attr() and bup_utimensat().
  Replace os.*stat() with xstat.*stat(); use integer ns for all fs times.
  Drop xstat floating point timestamp support -- use integer ns.
  xstst-cmd.py: test for _have_utimensat rather than _have_ns_fs_timestamps.

Makefile

@@ -146,8 +146,6 @@ import-docs: Documentation/clean
 	git archive origin/html | (cd Documentation; tar -xvf -)
 	git archive origin/man | (cd Documentation; tar -xvf -)
 
-# tgit.py plays with permissions on lib/bup/t/pybuptest.tmp, so we should
-# ensure that we can delete the directory before doing it.
 clean: Documentation/clean
 	rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll lib/*/*.dll *.exe \
 		.*~ *~ */*~ lib/*/*~ lib/*/*/*~ \

cmd/index-cmd.py

@@ -49,8 +49,9 @@ def check_index(reader):
 
 
 def update_index(top, excluded_paths):
+    tmax = time.time() - 1
     ri = index.Reader(indexfile)
-    wi = index.Writer(indexfile)
+    wi = index.Writer(indexfile, tmax)
     rig = IterHelper(ri.iter(name=top))
     tstart = int(time.time())
 
@@ -101,7 +102,7 @@ def hashgen(name):
                 check_index(ri)
                 log('check: before merging: newfile\n')
                 check_index(wr)
-            mi = index.Writer(indexfile)
+            mi = index.Writer(indexfile, tmax)
 
             for e in index.merge(ri, wr):
                 # FIXME: shouldn't we remove deleted entries eventually?  When?

cmd/meta-cmd.py

@@ -13,6 +13,14 @@
 from bup import options
 from bup.helpers import handle_ctrl_c, log, saved_errors
 
+
+def open_input(name):
+    if name != '-':
+        return open(name, 'r')
+    else:
+        return sys.stdin
+
+
 optspec = """
 bup meta --create [OPTION ...] <PATH ...>
 bup meta --extract [OPTION ...]
@@ -34,112 +42,57 @@
 q,quiet        don't show progress meter
 """
 
-action = None
-target_filename = ''
-should_recurse = False
-restore_numeric_ids = False
-include_paths = True
-handle_symlinks = True
-xdev = False
-
 handle_ctrl_c()
 
 o = options.Options(optspec)
-(opt, flags, remainder) = o.parse(sys.argv[1:])
+(opt, flags, remainder) = o.parse(['--paths', '--symlinks'] + sys.argv[1:])
 
-for flag, value in flags:
-    if flag == '--create' or flag == '-c':
-        action = 'create'
-    elif flag == '--list' or flag == '-t':
-        action = 'list'
-    elif flag == '--extract' or flag == '-x':
-        action = 'extract'
-    elif flag == '--start-extract':
-        action = 'start-extract'
-    elif flag == '--finish-extract':
-        action = 'finish-extract'
-    elif flag == '--file' or flag == '-f':
-        target_filename = value
-    elif flag == '--recurse' or flag == '-R':
-        should_recurse = True
-    elif flag == '--no-recurse':
-        should_recurse = False
-    elif flag in frozenset(['--xdev', '--one-file-system']):
-        xdev = True
-    elif flag in frozenset(['--no-xdev', '--no-one-file-system']):
-        xdev = False
-    elif flag == '--numeric-ids':
-        restore_numeric_ids = True
-    elif flag == '--no-numeric-ids':
-        restore_numeric_ids = False
-    elif flag == '--paths':
-        include_paths = True
-    elif flag == '--no-paths':
-        include_paths = False
-    elif flag == '--symlinks':
-        handle_symlinks = True
-    elif flag == '--no-symlinks':
-        handle_symlinks = False
-    elif flag == '--verbose' or flag == '-v':
-        metadata.verbose += 1
-    elif flag == '--quiet' or flag == '-q':
-        metadata.verbose = 0
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
 
-if not action:
-    o.fatal("no action specified")
+action_count = sum([bool(x) for x in [opt.create, opt.list, opt.extract,
+                                      opt.start_extract, opt.finish_extract]])
+if action_count > 1:
+    o.fatal("bup: only one action permitted: --create --list --extract")
+if action_count == 0:
+    o.fatal("bup: no action specified")
 
-if action == 'create':
+if opt.create:
     if len(remainder) < 1:
         o.fatal("no paths specified for create")
-    if target_filename != '-':
-        output_file = open(target_filename, 'w')
+    if opt.file != '-':
+        output_file = open(opt.file, 'w')
     else:
         output_file = sys.stdout
     metadata.save_tree(output_file,
                        remainder,
-                       recurse=should_recurse,
-                       write_paths=include_paths,
-                       save_symlinks=handle_symlinks,
-                       xdev=xdev)
-
-elif action == 'list':
+                       recurse=opt.recurse,
+                       write_paths=opt.paths,
+                       save_symlinks=opt.symlinks,
+                       xdev=opt.xdev)
+elif opt.list:
     if len(remainder) > 0:
         o.fatal("cannot specify paths for --list")
-    if target_filename != '-':
-        src = open(target_filename, 'r')
-    else:
-        src = sys.stdin
+    src = open_input(opt.file)
     metadata.display_archive(src)
-
-elif action == 'start-extract':
+elif opt.start_extract:
     if len(remainder) > 0:
         o.fatal("cannot specify paths for --start-extract")
-    if target_filename != '-':
-        src = open(target_filename, 'r')
-    else:
-        src = sys.stdin
-    metadata.start_extract(src, create_symlinks=handle_symlinks)
-
-elif action == 'finish-extract':
+    src = open_input(opt.file)
+    metadata.start_extract(src, create_symlinks=opt.symlinks)
+elif opt.finish_extract:
     if len(remainder) > 0:
         o.fatal("cannot specify paths for --finish-extract")
-    if target_filename != '-':
-        src = open(target_filename, 'r')
-    else:
-        src = sys.stdin
-    num_ids = restore_numeric_ids
-    metadata.finish_extract(src, restore_numeric_ids=num_ids)
-
-elif action == 'extract':
+    src = open_input(opt.file)
+    metadata.finish_extract(src, restore_numeric_ids=opt.numeric_ids)
+elif opt.extract:
     if len(remainder) > 0:
         o.fatal("cannot specify paths for --extract")
-    if target_filename != '-':
-        src = open(target_filename, 'r')
-    else:
-        src = sys.stdin
+    src = open_input(opt.file)
     metadata.extract(src,
-                     restore_numeric_ids=restore_numeric_ids,
-                     create_symlinks=handle_symlinks)
+                     restore_numeric_ids=opt.numeric_ids,
+                     create_symlinks=opt.symlinks)
 
 if saved_errors:
     log('WARNING: %d errors encountered.\n' % len(saved_errors))

cmd/midx-cmd.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import sys, math, struct, glob, resource
 import tempfile
-from bup import options, git, midx, _helpers
+from bup import options, git, midx, _helpers, xstat
 from bup.helpers import *
 
 PAGE_SIZE=4096
@@ -162,7 +162,7 @@ def do_midx_dir(path):
 
         # sort the biggest+newest midxes first, so that we can eliminate
         # smaller (or older) redundant ones that come later in the list
-        midxs.sort(key=lambda ix: (-sizes[ix], -os.stat(ix).st_mtime))
+        midxs.sort(key=lambda ix: (-sizes[ix], -xstat.stat(ix).st_mtime))
 
         for mname in midxs:
             any = 0

cmd/xstat-cmd.py

@@ -6,11 +6,12 @@
 import sys, stat, errno
 from bup import metadata, options, xstat
 from bup.helpers import handle_ctrl_c, saved_errors, add_error, log
-from bup import _helpers
 
 
 def fstimestr(fstime):
-    (s, ns) = fstime.secs_nsecs()
+    (s, ns) = xstat.fstime_to_timespec(fstime)
+    if(s < 0):
+        s += 1
     if ns == 0:
         return '%d' % s
     else:
@@ -50,11 +51,7 @@ def fstimestr(fstime):
 
 treat_include_fields_as_definitive = True
 for flag, value in flags:
-    if flag == '--verbose' or flag == '-v':
-        metadata.verbose += 1
-    elif flag == '--quiet' or flag == '-q':
-        metadata.verbose = 0
-    elif flag == '--exclude-fields':
+    if flag == '--exclude-fields':
         exclude_fields = frozenset(value.split(','))
         for f in exclude_fields:
             if not f in all_fields:
@@ -72,6 +69,10 @@ def fstimestr(fstime):
         else:
             active_fields = active_fields | include_fields
 
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
+
 for path in remainder:
     try:
         m = metadata.from_path(path, archive_path = path)
@@ -98,17 +99,17 @@ def fstimestr(fstime):
     if 'group' in active_fields:
         print 'group:', m.group
     if 'atime' in active_fields:
-        # if we don't have_ns_fs_timestamps, that means we have to use
+        # If we don't have utimensat, that means we have to use
         # utime(), and utime() has no way to set the mtime/atime of a
-        # symlink.  Thus, the mtime/atime of a symlink is meaningless, so
-        # let's not report it.  (That way scripts comparing before/after
-        # won't trigger.)
-        if _helpers._have_ns_fs_timestamps or not stat.S_ISLNK(m.mode):
+        # symlink.  Thus, the mtime/atime of a symlink is meaningless,
+        # so let's not report it.  (That way scripts comparing
+        # before/after won't trigger.)
+        if xstat.lutime or not stat.S_ISLNK(m.mode):
             print 'atime: ' + fstimestr(m.atime)
         else:
             print 'atime: 0'
     if 'mtime' in active_fields:
-        if _helpers._have_ns_fs_timestamps or not stat.S_ISLNK(m.mode):
+        if xstat.lutime or not stat.S_ISLNK(m.mode):
             print 'mtime: ' + fstimestr(m.mtime)
         else:
             print 'mtime: 0'

config/configure

@@ -50,8 +50,25 @@ MF_PATH_INCLUDE GIT git
 MF_PATH_INCLUDE TAR tar gtar
 MF_PATH_INCLUDE PYTHON python
 
-AC_CHECK_HEADERS sys/stat.h 
+# For stat.
+AC_CHECK_HEADERS sys/stat.h
+AC_CHECK_HEADERS sys/types.h
+AC_CHECK_HEADERS unistd.h
+
+# For FS_IOC_GETFLAGS and FS_IOC_SETFLAGS.
+AC_CHECK_HEADERS linux/fs.h
+AC_CHECK_HEADERS sys/ioctl.h
+
 AC_CHECK_FUNCS utimensat 
-AC_CHECK_FUNCS utime
+AC_CHECK_FUNCS utimes
+AC_CHECK_FUNCS lutimes
+
+AC_CHECK_FIELD stat st_atim sys/types.h sys/stat.h unistd.h
+AC_CHECK_FIELD stat st_mtim sys/types.h sys/stat.h unistd.h
+AC_CHECK_FIELD stat st_ctim sys/types.h sys/stat.h unistd.h
+
+AC_CHECK_FIELD stat st_atimensec sys/types.h sys/stat.h unistd.h
+AC_CHECK_FIELD stat st_mtimensec sys/types.h sys/stat.h unistd.h
+AC_CHECK_FIELD stat st_ctimensec sys/types.h sys/stat.h unistd.h
 
 AC_OUTPUT config.vars

config/configure.inc

@@ -1,3 +1,5 @@
+# -*-shell-script-*-
+
 #   @(#) configure.inc 1.40@(#)
 #   Copyright (c) 1999-2007 David Parsons. All rights reserved.
 #