fixed conflicts with master

src/allmydata/scripts/tahoe_add_alias.py

@@ -132,9 +132,9 @@ def list_aliases(options):
         for name, details in data.items():
             dircap = details['readonly'] if options['readonly-uri'] else details['readwrite']
             try:
-                print >>stdout, fmt % (unicode_to_output(name), unicode_to_output(dircap.decode('utf-8')))
+                print(fmt % (unicode_to_output(name), unicode_to_output(dircap.decode('utf-8'))), file=stdout)
             except (UnicodeEncodeError, UnicodeDecodeError):
-                print >>stderr, fmt % (quote_output(name), quote_output(dircap))
+                print(fmt % (quote_output(name), quote_output(dircap)), file=stderr)
                 rc = 1
 
     if rc == 1:

src/allmydata/scripts/tahoe_backup.py

@@ -1,4 +1,4 @@
-
+from __future__ import print_function
 import os.path
 import time
 import urllib
@@ -134,11 +134,11 @@ def run(self):
 
         put_child(archives_url, now, new_backup_dircap)
         put_child(to_url, "Latest", new_backup_dircap)
-        print >>stdout, completed.report(
+        print(completed.report(
             self.verbosity,
             self._files_checked,
             self._directories_checked,
-        )
+        ), file=stdout)
 
         # The command exits with code 2 if files or directories were skipped
         if completed.any_skips():
@@ -222,7 +222,7 @@ def check_backupdb_directory(self, compare_contents):
         self.verboseprint("checking %s" % quote_output(dircap))
         nodeurl = self.options['node-url']
         checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(dircap)
-        self.directories_checked += 1
+        self._directories_checked += 1
         resp = do_http("POST", checkurl)
         if resp.status != 200:
             # can't check, so we must assume it's bad
@@ -263,14 +263,329 @@ def upload(self, childpath):
             if bdb_results:
                 bdb_results.did_upload(filecap)
 
-            self.files_uploaded += 1
-            return filecap, metadata
+            return True, filecap, metadata
 
         else:
             self.verboseprint("skipping %s.." % quote_local_unicode_path(childpath))
-            self.files_reused += 1
-            return bdb_results.was_uploaded(), metadata
+            return False, bdb_results.was_uploaded(), metadata
+
 
 def backup(options):
     bu = BackerUpper(options)
     return bu.run()
+
+
+def collect_backup_targets(root, listdir, filter_children):
+    """
+    Yield BackupTargets in a suitable order for processing (deepest targets
+    before their parents).
+    """
+    try:
+        children = listdir(root)
+    except EnvironmentError:
+        yield PermissionDeniedTarget(root, isdir=True)
+    except FilenameEncodingError:
+        yield FilenameUndecodableTarget(root, isdir=True)
+    else:
+        for child in filter_children(children):
+            assert isinstance(child, unicode), child
+            childpath = os.path.join(root, child)
+            if os.path.islink(childpath):
+                yield LinkTarget(childpath, isdir=False)
+            elif os.path.isdir(childpath):
+                child_targets = collect_backup_targets(
+                    childpath,
+                    listdir,
+                    filter_children,
+                )
+                for child_target in child_targets:
+                    yield child_target
+            elif os.path.isfile(childpath):
+                yield FileTarget(childpath)
+            else:
+                yield SpecialTarget(childpath)
+        yield DirectoryTarget(root)
+
+
+def run_backup(
+        warn,
+        upload_file,
+        upload_directory,
+        targets,
+        start_timestamp,
+        stdout,
+):
+    progress = BackupProgress(warn, start_timestamp, len(targets))
+    for target in targets:
+        # Pass in the progress and get back a progress.  It would be great if
+        # progress objects were immutable.  Then the target's backup would
+        # make a new progress with the desired changes and return it to us.
+        # Currently, BackupProgress is mutable, though, and everything just
+        # mutates it.
+        progress = target.backup(progress, upload_file, upload_directory)
+        print(progress.report(datetime.datetime.now()), file=stdout)
+    return progress.backup_finished()
+
+
+class FileTarget(object):
+    def __init__(self, path):
+        self._path = path
+
+    def __repr__(self):
+        return "<File {}>".format(self._path)
+
+    def backup(self, progress, upload_file, upload_directory):
+        try:
+            created, childcap, metadata = upload_file(self._path)
+        except EnvironmentError:
+            target = PermissionDeniedTarget(self._path, isdir=False)
+            return target.backup(progress, upload_file, upload_directory)
+        else:
+            assert isinstance(childcap, str)
+            if created:
+                return progress.created_file(self._path, childcap, metadata)
+            return progress.reused_file(self._path, childcap, metadata)
+
+
+class DirectoryTarget(object):
+    def __init__(self, path):
+        self._path = path
+
+    def __repr__(self):
+        return "<Directory {}>".format(self._path)
+
+    def backup(self, progress, upload_file, upload_directory):
+        metadata = get_local_metadata(self._path)
+        progress, create, compare = progress.consume_directory(self._path)
+        did_create, dircap = upload_directory(self._path, compare, create)
+        if did_create:
+            return progress.created_directory(self._path, dircap, metadata)
+        return progress.reused_directory(self._path, dircap, metadata)
+
+
+class _ErrorTarget(object):
+    def __init__(self, path, isdir):
+        self._path = path
+        self._quoted_path = quote_local_unicode_path(path)
+        self._isdir = isdir
+
+
+class PermissionDeniedTarget(_ErrorTarget):
+    def backup(self, progress, upload_file, upload_directory):
+        return progress.permission_denied(self._isdir, self._quoted_path)
+
+
+class FilenameUndecodableTarget(_ErrorTarget):
+    def backup(self, progress, upload_file, upload_directory):
+        return progress.decoding_failed(self._isdir, self._quoted_path)
+
+
+class LinkTarget(_ErrorTarget):
+    def backup(self, progress, upload_file, upload_directory):
+        return progress.unsupported_filetype(
+            self._isdir,
+            self._quoted_path,
+            "symlink",
+        )
+
+
+class SpecialTarget(_ErrorTarget):
+    def backup(self, progress, upload_file, upload_directory):
+        return progress.unsupported_filetype(
+            self._isdir,
+            self._quoted_path,
+            "special",
+        )
+
+
+class BackupComplete(object):
+    def __init__(self,
+                 start_timestamp,
+                 end_timestamp,
+                 files_created,
+                 files_reused,
+                 files_skipped,
+                 directories_created,
+                 directories_reused,
+                 directories_skipped,
+                 dircap,
+    ):
+        self._start_timestamp = start_timestamp
+        self._end_timestamp = end_timestamp
+        self._files_created = files_created
+        self._files_reused = files_reused
+        self._files_skipped = files_skipped
+        self._directories_created = directories_created
+        self._directories_reused = directories_reused
+        self._directories_skipped = directories_skipped
+        self.dircap = dircap
+
+    def any_skips(self):
+        return self._files_skipped or self._directories_skipped
+
+    def report(self, verbosity, files_checked, directories_checked):
+        result = []
+
+        if verbosity >= 1:
+            result.append(
+                " %d files uploaded (%d reused),"
+                " %d files skipped,"
+                " %d directories created (%d reused),"
+                " %d directories skipped" % (
+                    self._files_created,
+                    self._files_reused,
+                    self._files_skipped,
+                    self._directories_created,
+                    self._directories_reused,
+                    self._directories_skipped,
+                ),
+            )
+
+        if verbosity >= 2:
+            result.append(
+                " %d files checked, %d directories checked" % (
+                    files_checked,
+                    directories_checked,
+                ),
+            )
+        # calc elapsed time, omitting microseconds
+        elapsed_time = str(
+            self._end_timestamp - self._start_timestamp
+        ).split('.')[0]
+        result.append(" backup done, elapsed time: %s" % (elapsed_time,))
+
+        return "\n".join(result)
+
+
+class BackupProgress(object):
+    # Would be nice if this data structure were immutable and its methods were
+    # transformations that created a new slightly different object.  Not there
+    # yet, though.
+    def __init__(self, warn, start_timestamp, target_count):
+        self._warn = warn
+        self._start_timestamp = start_timestamp
+        self._target_count = target_count
+        self._files_created = 0
+        self._files_reused = 0
+        self._files_skipped = 0
+        self._directories_created = 0
+        self._directories_reused = 0
+        self._directories_skipped = 0
+        self.last_dircap = None
+        self._create_contents = {}
+        self._compare_contents = {}
+
+    def report(self, now):
+        report_format = (
+            "Backing up {target_progress}/{target_total}... {elapsed} elapsed..."
+        )
+        return report_format.format(
+            target_progress=(
+                self._files_created
+                + self._files_reused
+                + self._files_skipped
+                + self._directories_created
+                + self._directories_reused
+                + self._directories_skipped
+            ),
+            target_total=self._target_count,
+            elapsed=self._format_elapsed(now - self._start_timestamp),
+        )
+
+    def _format_elapsed(self, elapsed):
+        seconds = elapsed.total_seconds()
+        hours = int(seconds / 3600)
+        minutes = int(seconds / 60 % 60)
+        seconds = int(seconds % 60)
+        return "{}h {}m {}s".format(
+            hours,
+            minutes,
+            seconds,
+        )
+
+    def backup_finished(self):
+        end_timestamp = datetime.datetime.now()
+        return BackupComplete(
+            self._start_timestamp,
+            end_timestamp,
+            self._files_created,
+            self._files_reused,
+            self._files_skipped,
+            self._directories_created,
+            self._directories_reused,
+            self._directories_skipped,
+            self.last_dircap,
+        )
+
+    def consume_directory(self, dirpath):
+        return self, {
+            os.path.basename(create_path): create_value
+            for (create_path, create_value)
+            in self._create_contents.iteritems()
+            if os.path.dirname(create_path) == dirpath
+        }, {
+            os.path.basename(compare_path): compare_value
+            for (compare_path, compare_value)
+            in self._compare_contents.iteritems()
+            if os.path.dirname(compare_path) == dirpath
+        }
+
+    def created_directory(self, path, dircap, metadata):
+        self._create_contents[path] = ("dirnode", dircap, metadata)
+        self._compare_contents[path] = dircap
+        self._directories_created += 1
+        self.last_dircap = dircap
+        return self
+
+    def reused_directory(self, path, dircap, metadata):
+        self._create_contents[path] = ("dirnode", dircap, metadata)
+        self._compare_contents[path] = dircap
+        self._directories_reused += 1
+        self.last_dircap = dircap
+        return self
+
+    def created_file(self, path, cap, metadata):
+        self._create_contents[path] = ("filenode", cap, metadata)
+        self._compare_contents[path] = cap
+        self._files_created += 1
+        return self
+
+    def reused_file(self, path, cap, metadata):
+        self._create_contents[path] = ("filenode", cap, metadata)
+        self._compare_contents[path] = cap
+        self._files_reused += 1
+        return self
+
+    def permission_denied(self, isdir, quoted_path):
+        return self._skip(
+            "WARNING: permission denied on {kind} {path}",
+            isdir,
+            path=quoted_path,
+        )
+
+    def decoding_failed(self, isdir, quoted_path):
+        return self._skip(
+            "WARNING: could not list {kind} {path} due to a filename encoding error",
+            isdir,
+            path=quoted_path,
+        )
+
+    def unsupported_filetype(self, isdir, quoted_path, filetype):
+        return self._skip(
+            "WARNING: cannot backup {filetype} {path}",
+            isdir,
+            path=quoted_path,
+            filetype=filetype,
+        )
+
+    def _skip(self, message, isdir, **kw):
+        if isdir:
+            self._directories_skipped += 1
+            kind = "directory"
+        else:
+            self._files_skipped += 1
+            kind = "file"
+        self._warn(message.format(kind=kind, **kw))
+        # Pretend we're a persistent data structure being transformed.
+        return self
+

src/allmydata/scripts/tahoe_cp.py

@@ -857,7 +857,7 @@ def copy_file_into_dir(self, source, name, target):
 
 
     def progress(self, message):
-        #print message
+        #print(message)
         if self.progressfunc:
             self.progressfunc(message)
 

src/allmydata/scripts/tahoe_mv.py

@@ -58,7 +58,7 @@ def mv(options, mode="move"):
         if status == 409:
             print("Error: You can't overwrite a directory with a file", file=stderr)
         else:
-            print >>stderr, format_http_error("Error", resp)
+            print(format_http_error("Error", resp), file=stderr)
             if mode == "move":
                 print("NOT removing the original", file=stderr)
         return 1