Made command cache debuggable, improved search cache reqs

mailpile/command_cache.py

@@ -31,7 +31,8 @@ class CommandCache(object):
     #    - The app configuration: '!config'
     #
 
-    def __init__(self):
+    def __init__(self, debug=None):
+        self.debug = debug or (lambda s: None)
         self.lock = UiRLock()
         self.cache = {}     # The cache itself
         self.dirty = set()  # Requirements that have changed recently
@@ -47,6 +48,7 @@ def cache_result(self, fprint, expires, req, cmd_obj, result_obj):
             #       as mere presence in the cache makes this a candidate
             #       for refreshing.
             self.cache[str(fprint)] = [expires, req, cmd_obj, result_obj]
+            self.debug('Cached %s, req=%s' % (fprint, req))
 
     def get_result(self, fprint, dirty_check=True):
         with self.lock:
@@ -59,6 +61,7 @@ def get_result(self, fprint, dirty_check=True):
 
     def mark_dirty(self, requirements):
         self.dirty |= set(requirements)
+        self.debug('Marked dirty: %s' % requirements)
 
     def refresh(self, extend=60, event_log=None):
         now = time.time()
@@ -91,3 +94,4 @@ def refresh(self, extend=60, event_log=None):
                           source=self,
                           data={'cache_ids': refreshed},
                           flags=Event.COMPLETE)
+            self.debug('Refreshed: %s' % refreshed)

mailpile/config.py

@@ -1136,13 +1136,17 @@ def __init__(self, workdir=None, rules={}):
         self.event_log = None
         self.index = None
         self.vcards = {}
-        self.command_cache = CommandCache()
         self.search_history = SearchHistory()
         self._mbox_cache = []
         self._running = {}
         self._lock = ConfigRLock()
         self.loaded_config = False
 
+        def cache_debug(msg):
+            if self.background and 'cache' in self.sys.debug:
+                self.background.ui.debug(msg)
+        self.command_cache = CommandCache(debug=cache_debug)
+
         self.gnupg_passphrase = SecurePassphraseStorage()
 
         self.jinja_env = Environment(

mailpile/plugins/search.py

@@ -181,11 +181,15 @@ def fix_term(term):
                 term = term[1:]
             term = ':'.join(reversed(term.split(':', 1)))
             return unicode(term)
-        return set(
-            ['!config'] +
-            [fix_term(t) for t in self.session.searched] +
-            [u'%s:msg' % i for i in msgs]
-        )
+        reqs = set(['!config'] +
+                   [fix_term(t) for t in self.session.searched] +
+                   [u'%s:msg' % i for i in msgs])
+        if self.session.displayed:
+            reqs |= set(u'%s:thread' % int(tmid, 36) for tmid in
+                        self.session.displayed.get('thread_ids', []))
+            reqs |= set(u'%s:msg' % int(tmid, 36) for tmid in
+                        self.session.displayed.get('message_ids', []))
+        return reqs
 
     def command(self):
         session, idx, start, num = self._do_search()

mailpile/search.py

@@ -285,7 +285,7 @@ def process_lines(lines):
         self.EMAILS_SAVED = len(self.EMAILS)
 
     def update_msg_tags(self, msg_idx_pos, msg_info):
-        tags = set([t for t in msg_info[self.MSG_TAGS].split(',') if t])
+        tags = set(self.get_tags(msg_info=msg_info))
         with self._lock:
             for tid in (set(self.TAGS.keys()) - tags):
                 self.TAGS[tid] -= set([msg_idx_pos])
@@ -866,7 +866,7 @@ def index_email(self, session, email):
         self.set_msg_at_idx_pos(email.msg_idx_pos, msg_info)
 
         # Reset the internal tags on this message
-        for tag_id in [t for t in msg_info[self.MSG_TAGS].split(',') if t]:
+        for tag_id in self.get_tags(msg_info=msg_info):
             tag = session.config.get_tag(tag_id)
             if tag and tag.slug.startswith('mp_'):
                 self.remove_tag(session, tag_id, msg_idxs=[email.msg_idx_pos])
@@ -1300,9 +1300,11 @@ def set_msg_at_idx_pos(self, msg_idx, msg_info, original_line=None):
         self.update_msg_tags(msg_idx, msg_info)
 
         if not original_line:
-            self.config.command_cache.mark_dirty([u'mail:all',
-                                                  u'%s:msg' % msg_idx,
-                                                  u'%s:thread' % msg_thr_mid])
+            dirty_tags = [u'%s:in' % self.config.tags[t].slug for t in
+                          self.get_tags(msg_info=msg_info)]
+            self.config.command_cache.mark_dirty(
+                [u'mail:all', u'%s:msg' % msg_idx,
+                 u'%s:thread' % int(msg_thr_mid, 36)] + dirty_tags)
             CachedSearchResultSet.DropCaches(msg_idxs=[msg_idx])
             self.MODIFIED.add(msg_idx)
             try:
@@ -1329,7 +1331,10 @@ def get_replies(self, msg_info=None, msg_idx=None):
     def get_tags(self, msg_info=None, msg_idx=None):
         if not msg_info:
             msg_info = self.get_msg_at_idx_pos(msg_idx)
-        return [r for r in msg_info[self.MSG_TAGS].split(',') if r]
+        taglist = [r for r in msg_info[self.MSG_TAGS].split(',') if r]
+        if not 'tags' in self.config:
+            return taglist
+        return [r for r in taglist if r in self.config.tags]
 
     def add_tag(self, session, tag_id,
                 msg_info=None, msg_idxs=None, conversation=False):
@@ -1351,6 +1356,7 @@ def add_tag(self, session, tag_id,
                         msg_idxs.add(int(reply[self.MSG_MID], 36))
         eids = set()
         added = set()
+        threads = set()
         for msg_idx in msg_idxs:
             if msg_idx >= 0 and msg_idx < len(self.INDEX):
                 msg_info = self.get_msg_at_idx_pos(msg_idx)
@@ -1363,6 +1369,7 @@ def add_tag(self, session, tag_id,
                     self.MODIFIED.add(msg_idx)
                     self.update_msg_sorting(msg_idx, msg_info)
                     added.add(msg_idx)
+                    threads.add(msg_info[self.MSG_THREAD_MID])
                 eids.add(msg_idx)
         with self._lock:
             if tag_id in self.TAGS:
@@ -1372,7 +1379,8 @@ def add_tag(self, session, tag_id,
         try:
             self.config.command_cache.mark_dirty(
                 [u'mail:all', u'%s:in' % self.config.tags[tag_id].slug] +
-                [u'%s:msg' % eid for eid in added])
+                [u'%s:msg' % e_idx for e_idx in added] +
+                [u'%s:thread' % int(mid, 36) for mid in threads])
         except:
             pass
         return added
@@ -1401,6 +1409,7 @@ def remove_tag(self, session, tag_id,
                            ) % (len(msg_idxs), tag_id))
         eids = set()
         removed = set()
+        threads = set()
         for msg_idx in msg_idxs:
             if msg_idx >= 0 and msg_idx < len(self.INDEX):
                 msg_info = self.get_msg_at_idx_pos(msg_idx)
@@ -1413,14 +1422,16 @@ def remove_tag(self, session, tag_id,
                     self.MODIFIED.add(msg_idx)
                     self.update_msg_sorting(msg_idx, msg_info)
                     removed.add(msg_idx)
+                    threads.add(msg_info[self.MSG_THREAD_MID])
                 eids.add(msg_idx)
         with self._lock:
             if tag_id in self.TAGS:
                 self.TAGS[tag_id] -= eids
         try:
             self.config.command_cache.mark_dirty(
                 [u'%s:in' % self.config.tags[tag_id].slug] +
-                [u'%s:msg' % eid for eid in removed])
+                [u'%s:msg' % e_idx for e_idx in removed] +
+                [u'%s:thread' % int(mid, 36) for mid in threads])
         except:
             pass
         return removed
@@ -1569,7 +1580,7 @@ def hits(term):
 
     def _freshness_sorter(self, msg_info):
         ts = long(msg_info[self.MSG_DATE], 36)
-        for tid in msg_info[self.MSG_TAGS].split(','):
+        for tid in self.get_tags(msg_info=msg_info):
             if tid in self._sort_freshness_tags:
                 return ts + self.FRESHNESS_SORT_BOOST
         return ts