Add check_index to make clients start faster, and use attrs dict for …

…comparison

backtrac/api/client.py

@@ -40,17 +40,22 @@ def get_exclusions(self):
         return [ excl.glob for excl in self.client_obj.exclusions.all() ]
 
     def get_present_state(self, path):
-        def get_children(item, items):
-            items.append(item.path)
-            for i in item.children.present():
-                get_children(i, items)
-        items = []
+        def get_children(item, index):
+            attrs = {}
+            if item.latest_version:
+                attrs['mtime'] = item.latest_version.mtime
+                attrs['size'] = item.latest_version.size
+            index[item.path] = attrs
+            for i in item.children.present().select_related('latest_version'):
+                get_children(i, index)
+        index = {}
+        qs = []
         try:
             item = Item.objects.get(client=self.client_obj, path=path)
-            get_children(item, items)
+            get_children(item, index)
         except Item.DoesNotExist:
             pass
-        return items
+        return index
 
     def is_excluded(self, path):
         _, basename = os.path.split(path)
@@ -72,15 +77,33 @@ def delete_item(self, path):
         item_deleted.send(sender=self.client_obj, path=path,
                           client=self.client_obj)
 
-    def backup_required(self, path, mtime, size):
+    def compare_attrs(self, old_attrs, new_attrs):
+        #TODO add a configurable tolerance for these values
+        diff = 0
+        if 'mtime' in old_attrs and 'mtime' in new_attrs:
+            if abs(old_attrs['mtime'] - new_attrs['mtime']) > 1:
+                diff += 1
+        if 'size' in old_attrs and 'size' in new_attrs:
+            if abs(old_attrs['size'] - new_attrs['size']) > 1:
+                diff += 2
+        return diff
+
+    def backup_required(self, path, attrs):
         try:
             item = Item.objects.get(client=self.client_obj, path=path)
             if not item.latest_version or item.deleted:
                 return True
-            if abs(size - item.latest_version.size) < 1:
-                if item.latest_version.is_restored() or \
-                   abs(mtime - item.latest_version.mtime) < 1:
-                    return False
+            new_attrs = {
+                'mtime': item.latest_version.mtime,
+                'size': item.latest_version.size
+            }
+            diff = self.compare_attrs(attrs, new_attrs)
+            if diff == 0:
+                return False
+            elif diff < 2 and item.latest_version.is_restored():
+                return False
+            else:
+                return True
         except Item.DoesNotExist:
             pass
         return True

backtrac/client/broker.py

@@ -26,8 +26,11 @@ def get_paths(self):
     def get_present_state(self, path):
         return self.perspective.callRemote('get_present_state', path)
 
-    def check_file(self, path, mtime, size):
-        return self.perspective.callRemote('check_file', path, mtime, size)
+    def check_index(self, path, index):
+        return self.perspective.callRemote('check_index', path, index)
+
+    def check_file(self, path, attrs):
+        return self.perspective.callRemote('check_file', path, attrs)
 
     def delete_item(self, path):
         return self.perspective.callRemote('delete_item', path)

backtrac/client/client.py

@@ -51,6 +51,25 @@ def walk_path(self, path):
                 path = os.path.join(root, f)
                 self.backup_queue.add(BackupJob(path))
 
+    def get_index(self, path):
+        index = {}
+        for root, dirs, files in os.walk(path):
+            index[root] = {}
+            for f in files:
+                path = os.path.join(root, f)
+                filepath = FilePath(path)
+                index[path] = {
+                    'mtime': filepath.getModificationTime(),
+                    'size': filepath.getsize(),
+                }
+        return index
+
+    @defer.inlineCallbacks
+    def check_index(self, path, index):
+        backup = yield self.broker.check_index(path, index)
+        for path in backup:
+            self.backup_queue.add(BackupJob(path))
+
     def remote_put_file(self, path):
         self.monitor.add_exclusion(path)
         makedirs(os.path.split(path)[0])
@@ -69,9 +88,11 @@ def start(self):
         paths = yield self.broker.get_paths()
         for path in paths:
             path = normpath(path)
-            self.check_present_state(path)
-            self.walk_path(path)
             self.monitor.add_watch(path)
+            index = self.get_index(path)
+            self.check_present_state(path)
+            self.check_index(path, index)
+            #self.walk_path(path)
 
 def get_server_status():
     broker = BackupBroker(server='localhost', secret_key=settings.SECRET_KEY,

backtrac/client/queue.py

@@ -54,9 +54,11 @@ def consume_create(self, filepath):
 
     def consume_update(self, filepath):
         try:
-            mtime = filepath.getModificationTime()
-            size = filepath.getsize()
-            d = self.client.broker.check_file(filepath.path, mtime, size)
+            attrs = {
+                'mtime': filepath.getModificationTime(),
+                'size': filepath.getsize(),
+            }
+            d = self.client.broker.check_file(filepath.path, attrs)
             d.addCallback(self._check_result, filepath.path)
             return d
         except (OSError, IOError):

backtrac/server/server.py

@@ -114,11 +114,23 @@ def perspective_get_paths(self):
         return self.api.get_paths()
 
     def perspective_get_present_state(self, path):
-        return self.api.get_present_state(path)
-
-    def perspective_check_file(self, path, mtime, size):
+        index = self.api.get_present_state(path)
+        return index.keys()
+
+    def perspective_check_index(self, path, cur_index):
+        old_index = self.api.get_present_state(path)
+        backup = []
+        for path, attrs in cur_index.items():
+            if not path in old_index:
+                backup.append(path)
+            else:
+                if self.api.compare_attrs(old_index[path], attrs) > 0:
+                    backup.append(path)
+        return backup
+
+    def perspective_check_file(self, path, attrs):
         if not self.api.is_excluded(path):
-            return self.api.backup_required(path, mtime, size)
+            return self.api.backup_required(path, attrs)
         return False
 
     def perspective_create_item(self, path, type):