refactor duplicity restore logic out of restore module and into dupli…

…city module

rational:

- The duplicity control logic doesn't really belong in Restore. That's why it
  was a staticmethod.

- Simplifies Restore initialization

- The separation makes it possible to cleanly implement restoring from a
  backup extract path rather than a duplicity address

cmd_restore.py

@@ -94,9 +94,10 @@
 
 from os.path import *
 from restore import Restore
+import duplicity
 
 from stdtrap import UnitedStdTrap
-from temp import TempFile
+from temp import TempDir
 import executil
 
 import hub
@@ -109,7 +110,6 @@
 from version import get_turnkey_version, codename
 from utils import is_writeable
 
-import backup
 from conf import Conf
 
 import traceback
@@ -132,8 +132,8 @@ def do_compatibility_check(backup_turnkey_version, interactive=True):
     if local_codename == backup_codename:
         return
 
-    def fmt(codename):
-        return codename.upper().replace("-", " ")
+    def fmt(s):
+        return s.upper().replace("-", " ")
 
     backup_codename = fmt(backup_codename)
     local_codename = fmt(local_codename)
@@ -265,6 +265,10 @@ def main():
             if exists(download_path):
                 if not isdir(download_path):
                     fatal("--download-only=%s is not a directory" % download_path)
+
+                if os.listdir(download_path) != []:
+                    fatal("--download-only=%s is not an empty directory" % download_path)
+
             else:
                 os.mkdir(download_path)
 
@@ -352,26 +356,32 @@ def main():
     key = opt_key if opt_key else hbr.key
     secret = decrypt_key(key, interactive)
 
-    if download_path:
-        restore = Restore(address, secret, restore_cache_size, restore_cache_dir,
-                          opt_limits, opt_time, credentials=credentials, download_path=download_path)
+    def get_backup_extract():
+        print "Restoring backup extract from duplicity archive at %s" % (address)
+        duplicity.restore(download_path, address, restore_cache_size, restore_cache_dir, credentials, secret, opt_time)
+        return download_path
 
+    if download_path:
+        get_backup_extract()
         return
+    else:
+        download_path = TempDir(prefix="tklbam-")
+        os.chmod(download_path, 0700)
 
     trap = UnitedStdTrap(usepty=True, transparent=(False if silent else True))
     log_fh = None
     try:
         try:
             hooks.restore.pre()
-            restore = Restore(address, secret, restore_cache_size, restore_cache_dir,
-                              opt_limits, opt_time, credentials=credentials, rollback=not no_rollback)
 
+            extract_path = get_backup_extract()
+            restore = Restore(extract_path, limits=opt_limits, rollback=not no_rollback)
             hooks.restore.inspect(restore.extras.path)
             if opt_debug:
                 trap.close()
                 trap = None
 
-                os.chdir(restore.backup_archive)
+                os.chdir(extract_path)
                 executil.system(os.environ.get("SHELL", "/bin/bash"))
                 os.chdir('/')
 

duplicity.py

@@ -14,6 +14,10 @@
 import sys
 
 from subprocess import *
+from squid import Squid
+
+import resource
+RLIMIT_NOFILE_MAX = 8192
 
 def _find_duplicity_pylib(path):
     if not isdir(path):
@@ -79,3 +83,43 @@ def run(self, passphrase, creds=None, debug=False):
 
     def __str__(self):
         return " ".join(self.command)
+
+
+def _raise_rlimit(type, newlimit):
+
+
+    soft, hard = resource.getrlimit(type)
+    if soft > newlimit:
+        return
+
+    if hard > newlimit:
+        return resource.setrlimit(type, (newlimit, hard))
+
+    try:
+        resource.setrlimit(type, (newlimit, newlimit))
+    except ValueError:
+        return
+
+def restore(download_path, address, cache_size, cache_dir, credentials, secret, time=None):
+    if time:
+        opts = [("restore-time", time)]
+    else:
+        opts = []
+
+    squid = Squid(cache_size, cache_dir)
+    squid.start()
+
+    orig_env = os.environ.get('http_proxy')
+    os.environ['http_proxy'] = squid.address
+
+    _raise_rlimit(resource.RLIMIT_NOFILE, RLIMIT_NOFILE_MAX)
+    Command(opts, '--s3-unencrypted-connection', address, download_path).run(secret, credentials)
+
+    if orig_env:
+        os.environ['http_proxy'] = orig_env
+    else:
+        del os.environ['http_proxy']
+
+    sys.stdout.flush()
+
+    squid.stop()

restore.py

@@ -13,47 +13,21 @@
 import os
 from os.path import *
 
-import resource
-
-import shutil
-import commands
-
 import userdb
-from paths import Paths
 from changes import Changes
 from pathmap import PathMap
-from dirindex import DirIndex
 from pkgman import Installer
 from rollback import Rollback
-from temp import TempDir
 
 import utils
 
 import backup
 import conf
 import mysql
-import duplicity
-
-from squid import Squid
 
 class Error(Exception):
     pass
 
-RLIMIT_NOFILE_MAX = 8192
-
-def raise_rlimit(type, newlimit):
-    soft, hard = resource.getrlimit(type)
-    if soft > newlimit:
-        return
-
-    if hard > newlimit:
-        return resource.setrlimit(type, (newlimit, hard))
-
-    try:
-        resource.setrlimit(type, (newlimit, newlimit))
-    except ValueError:
-        return
-
 def system(command):
     sys.stdout.flush()
     sys.stderr.flush()
@@ -68,48 +42,15 @@ class Restore:
     def _title(title, c='='):
         return title + "\n" + c * len(title) + "\n"
 
-    @staticmethod
-    def _duplicity_restore(address, cache_size, cache_dir, credentials, secret, time=None, download_path=None):
-        if not download_path:
-            download_path = TempDir(prefix="tklbam-")
-            os.chmod(download_path, 0700)
-
-        if time:
-            opts = [("restore-time", time)]
-        else:
-            opts = []
-
-        squid = Squid(cache_size, cache_dir)
-        squid.start()
-
-        orig_env = os.environ.get('http_proxy')
-        os.environ['http_proxy'] = squid.address
-
-        raise_rlimit(resource.RLIMIT_NOFILE, RLIMIT_NOFILE_MAX)
-        duplicity.Command(opts, '--s3-unencrypted-connection', address, download_path).run(secret, credentials)
-
-        if orig_env:
-            os.environ['http_proxy'] = orig_env
-        else:
-            del os.environ['http_proxy']
-
-        sys.stdout.flush()
-
-        squid.stop()
-
-        return download_path
-
-    def __init__(self, address, secret, cache_size, cache_dir,
-                 limits=[], time=None, credentials=None, rollback=True, download_path=None):
-        print "Restoring duplicity archive from " + address
-        backup_archive = self._duplicity_restore(address, cache_size, cache_dir, credentials, secret, time, download_path)
+    def __init__(self, backup_extract_path, limits=[], rollback=True):
+        extras_path = backup_extract_path + backup.ExtrasPaths.PATH
+        if not isdir(extras_path):
+            raise self.Error("illegal backup_extract_path: can't find '%s'" % extras_path)
 
-        extras_path = backup_archive + backup.ExtrasPaths.PATH
         self.extras = backup.ExtrasPaths(extras_path)
         self.rollback = Rollback.create() if rollback else None
         self.limits = conf.Limits(limits)
-        self.credentials = credentials
-        self.backup_archive = backup_archive
+        self.backup_extract_path = backup_extract_path
 
     def database(self):
         if not exists(self.extras.myfs):
@@ -173,7 +114,7 @@ def r(path):
 
     @staticmethod
     def _iter_apply_overlay(overlay, root, limits=[]):
-        def walk(dir):
+        def _walk(dir):
             fnames = []
             subdirs = []
 
@@ -188,7 +129,7 @@ def walk(dir):
             yield dir, fnames
 
             for subdir in subdirs:
-                for val in walk(subdir):
+                for val in _walk(subdir):
                     yield val
 
         class OverlayError:
@@ -201,7 +142,7 @@ def __str__(self):
 
         pathmap = PathMap(limits)
         overlay = overlay.rstrip('/')
-        for overlay_dpath, fnames in walk(overlay):
+        for overlay_dpath, fnames in _walk(overlay):
             root_dpath = root + overlay_dpath[len(overlay) + 1:]
 
             for fname in fnames:
@@ -230,7 +171,7 @@ def files(self):
         if not exists(extras.fsdelta):
             return
 
-        overlay = self.backup_archive
+        overlay = self.backup_extract_path
         rollback = self.rollback
         limits = self.limits.fs