From 45a37ee4ccf709875950e8a7b870f17a78b5e24f Mon Sep 17 00:00:00 2001 From: Rob Golding Date: Mon, 28 Mar 2011 21:06:23 +0100 Subject: [PATCH] Add check_index to make clients start faster, and use attrs dict for comparison --- backtrac/api/client.py | 47 +++++++++++++++++++++++++++++---------- backtrac/client/broker.py | 7 ++++-- backtrac/client/client.py | 25 +++++++++++++++++++-- backtrac/client/queue.py | 8 ++++--- backtrac/server/server.py | 20 +++++++++++++---- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/backtrac/api/client.py b/backtrac/api/client.py index c473023..b23fb8d 100644 --- a/backtrac/api/client.py +++ b/backtrac/api/client.py @@ -40,17 +40,22 @@ def get_exclusions(self): return [ excl.glob for excl in self.client_obj.exclusions.all() ] def get_present_state(self, path): - def get_children(item, items): - items.append(item.path) - for i in item.children.present(): - get_children(i, items) - items = [] + def get_children(item, index): + attrs = {} + if item.latest_version: + attrs['mtime'] = item.latest_version.mtime + attrs['size'] = item.latest_version.size + index[item.path] = attrs + for i in item.children.present().select_related('latest_version'): + get_children(i, index) + index = {} + qs = [] try: item = Item.objects.get(client=self.client_obj, path=path) - get_children(item, items) + get_children(item, index) except Item.DoesNotExist: pass - return items + return index def is_excluded(self, path): _, basename = os.path.split(path) @@ -72,15 +77,33 @@ def delete_item(self, path): item_deleted.send(sender=self.client_obj, path=path, client=self.client_obj) - def backup_required(self, path, mtime, size): + def compare_attrs(self, old_attrs, new_attrs): + #TODO add a configurable tolerance for these values + diff = 0 + if 'mtime' in old_attrs and 'mtime' in new_attrs: + if abs(old_attrs['mtime'] - new_attrs['mtime']) > 1: + diff += 1 + if 'size' in old_attrs and 'size' in new_attrs: + if abs(old_attrs['size'] - new_attrs['size']) > 1: + diff += 2 + return diff + + def backup_required(self, path, attrs): try: item = Item.objects.get(client=self.client_obj, path=path) if not item.latest_version or item.deleted: return True - if abs(size - item.latest_version.size) < 1: - if item.latest_version.is_restored() or \ - abs(mtime - item.latest_version.mtime) < 1: - return False + new_attrs = { + 'mtime': item.latest_version.mtime, + 'size': item.latest_version.size + } + diff = self.compare_attrs(attrs, new_attrs) + if diff == 0: + return False + elif diff < 2 and item.latest_version.is_restored(): + return False + else: + return True except Item.DoesNotExist: pass return True diff --git a/backtrac/client/broker.py b/backtrac/client/broker.py index c6032fa..46b2fa5 100644 --- a/backtrac/client/broker.py +++ b/backtrac/client/broker.py @@ -26,8 +26,11 @@ def get_paths(self): def get_present_state(self, path): return self.perspective.callRemote('get_present_state', path) - def check_file(self, path, mtime, size): - return self.perspective.callRemote('check_file', path, mtime, size) + def check_index(self, path, index): + return self.perspective.callRemote('check_index', path, index) + + def check_file(self, path, attrs): + return self.perspective.callRemote('check_file', path, attrs) def delete_item(self, path): return self.perspective.callRemote('delete_item', path) diff --git a/backtrac/client/client.py b/backtrac/client/client.py index 31057f5..407791a 100644 --- a/backtrac/client/client.py +++ b/backtrac/client/client.py @@ -51,6 +51,25 @@ def walk_path(self, path): path = os.path.join(root, f) self.backup_queue.add(BackupJob(path)) + def get_index(self, path): + index = {} + for root, dirs, files in os.walk(path): + index[root] = {} + for f in files: + path = os.path.join(root, f) + filepath = FilePath(path) + index[path] = { + 'mtime': filepath.getModificationTime(), + 'size': filepath.getsize(), + } + return index + + @defer.inlineCallbacks + def check_index(self, path, index): + backup = yield self.broker.check_index(path, index) + for path in backup: + self.backup_queue.add(BackupJob(path)) + def remote_put_file(self, path): self.monitor.add_exclusion(path) makedirs(os.path.split(path)[0]) @@ -69,9 +88,11 @@ def start(self): paths = yield self.broker.get_paths() for path in paths: path = normpath(path) - self.check_present_state(path) - self.walk_path(path) self.monitor.add_watch(path) + index = self.get_index(path) + self.check_present_state(path) + self.check_index(path, index) + #self.walk_path(path) def get_server_status(): broker = BackupBroker(server='localhost', secret_key=settings.SECRET_KEY, diff --git a/backtrac/client/queue.py b/backtrac/client/queue.py index 836a102..e5f341c 100644 --- a/backtrac/client/queue.py +++ b/backtrac/client/queue.py @@ -54,9 +54,11 @@ def consume_create(self, filepath): def consume_update(self, filepath): try: - mtime = filepath.getModificationTime() - size = filepath.getsize() - d = self.client.broker.check_file(filepath.path, mtime, size) + attrs = { + 'mtime': filepath.getModificationTime(), + 'size': filepath.getsize(), + } + d = self.client.broker.check_file(filepath.path, attrs) d.addCallback(self._check_result, filepath.path) return d except (OSError, IOError): diff --git a/backtrac/server/server.py b/backtrac/server/server.py index bb419f1..25ff402 100644 --- a/backtrac/server/server.py +++ b/backtrac/server/server.py @@ -114,11 +114,23 @@ def perspective_get_paths(self): return self.api.get_paths() def perspective_get_present_state(self, path): - return self.api.get_present_state(path) - - def perspective_check_file(self, path, mtime, size): + index = self.api.get_present_state(path) + return index.keys() + + def perspective_check_index(self, path, cur_index): + old_index = self.api.get_present_state(path) + backup = [] + for path, attrs in cur_index.items(): + if not path in old_index: + backup.append(path) + else: + if self.api.compare_attrs(old_index[path], attrs) > 0: + backup.append(path) + return backup + + def perspective_check_file(self, path, attrs): if not self.api.is_excluded(path): - return self.api.backup_required(path, mtime, size) + return self.api.backup_required(path, attrs) return False def perspective_create_item(self, path, type):