WIP: caching

mailpile/command_cache.py

@@ -21,40 +21,65 @@ class CommandCache(object):
     #     - Periodically, the cache is refreshed, which re-runs any dirtied
     #       commands and fires events notifying the UI about changes.
     #
+    # Examples of requirements:
+    #
+    #    - Search terms, eg. 'in:inbox' or 'potato' or 'salad'
+    #    - Messages: 'msg:INDEX' where INDEX is a number (not a MID)
+    #    - Threads: 'thread:MID' were MID is the thread ID.
+    #    - The app configuration: '!config'
+    #
 
     def __init__(self):
-        self.cache = {}
         self.lock = UiRLock()
-        self.dirty = set()
+        self.cache = {}     # The cache itself
+        self.dirty = set()  # Requirements that have changed recently
 
-    def cache_result(self, fprint, expires, req, command_obj, result_obj):
+    def cache_result(self, fprint, expires, req, cmd_obj, result_obj):
         with self.lock:
-            self.cache[str(fprint)] = (expires, req, command_obj, result_obj)
+            # Note: We cache this even if the requirements are "dirty",
+            #       as mere presence in the cache makes this a candidate
+            #       for refreshing.
+            self.cache[str(fprint)] = (expires, req, cmd_obj, result_obj)
 
     def get_result(self, fprint):
-        fprint = str(fprint)
-        if fprint in self.dirty:
+        exp, req, co, ro = self.cache[fprint]
+        if req & self.dirty:
+            # If requirements are dirty, pretend this item does not exist.
             raise KeyError()
         return self.cache[fprint][3]
 
     def mark_dirty(self, requirements):
-        with self.lock:
-            for fprint, (e, r, co, ro) in self.cache.iteritems():
-                for req in requirements:
-                    if req in r:
-                        self.dirty.add(fprint)
-                        break
+        self.dirty |= set(requirements)
+        print 'DIRTY: %s' % requirements
 
-    def refresh(self, extend=60):
+    def refresh(self, extend=60, event_log=None):
         now = time.time()
         with self.lock:
             expired = set([f for f in self.cache if self.cache[f][0] < now])
             for fp in expired:
                 del self.cache[fp]
+
             dirty, self.dirty = self.dirty, set()
+            fingerprints = list(self.cache.keys())
 
-        #for fprint in (dirty - expired):
-        #    exp, req, co, ro = self.cache[fprint]
-        #    ro = co.refresh()
-        #    self.cache[fprint] = (exp + extend, req, co, ro)
+        refreshed = []
+        for fprint in fingerprints:
+            try:
+                exp, req, co, ro = self.cache[fprint]
+                if req & dirty:
+                    ro = co.refresh()
+                    with self.lock:
+                        self.cache[fprint] = (exp + extend, req, co, ro)
+                    refreshed.append(fprint)
+                    play_nice_with_threads()
+            except (ValueError, IndexError, TypeError):
+                # Broken stuff just gets evicted
+                with self.lock:
+                    if fprint in self.cache:
+                        del self.cache[fprint]
 
+        if refreshed and event_log:
+            event_log.log(message=_('New results are available'),
+                          source=self,
+                          data={'cache_ids': refreshed})
+            print 'REFRESHED: %s' % refreshed

mailpile/commands.py

@@ -239,19 +239,23 @@ def cache_id(self, sqa=None):
             return ''
         from mailpile.urlmap import UrlMap
         args = sorted(list((sqa or self.state_as_query_args()).iteritems()))
-        return '%s@%s' % (UrlMap.ui_url(self), md5_hex(str(args)))
+        # The replace() stuff makes these usable as CSS class IDs
+        return ('%s-%s' % (UrlMap.ui_url(self), md5_hex(str(args)))
+                          ).replace('/', '-').replace('.', '-')
 
-    def cache_requirements(self):
+    def cache_requirements(self, result):
         raise NotImplementedError('Cachable commands should override this, '
                                   'returning a set() of requirements.')
 
     def cache_result(self, result):
         if self.COMMAND_CACHE_TTL > 0:
-            print 'CACHING RESULT: %s' % self.cache_id()
+            cache_id = self.cache_id()
+            requirements = self.cache_requirements(result)
+            print 'CACHING RESULT: %s (%s)' % (cache_id, requirements)
             self.session.config.command_cache.cache_result(
-                self.cache_id(),
+                cache_id,
                 time.time() + self.COMMAND_CACHE_TTL,
-                self.cache_requirements(),
+                requirements,
                 self,
                 result
             )
@@ -514,18 +518,20 @@ def _update_finished_event(self):
         if self.name:
             self.session.ui.finish_command(self.name)
 
-    def _run_sync(self, *args, **kwargs):
+    def _run_sync(self, enable_cache, *args, **kwargs):
+        self._run_args = args
+        self._run_kwargs = kwargs
         self._starting()
 
-        if self.COMMAND_CACHE_TTL > 0:
+        if (self.COMMAND_CACHE_TTL > 0 and
+               'http' not in self.session.config.sys.debug and
+               enable_cache):
             cid = self.cache_id()
             try:
                 rv = self.session.config.command_cache.get_result(cid)
                 self._finishing(self, True, just_cleanup=True)
-                print 'CACHE HIT: %s' % cid
                 return rv
             except:
-                print 'CACHE MISS: %s' % cid
                 pass
 
         def command(self, *args, **kwargs):
@@ -552,7 +558,7 @@ def _run(self, *args, **kwargs):
             def streetcar():
                 try:
                     with MultiContext(self.WITH_CONTEXT):
-                        rv = self._run_sync(*args, **kwargs).as_dict()
+                        rv = self._run_sync(True, *args, **kwargs).as_dict()
                         self.event.private_data.update(rv)
                         self._update_finished_event()
                 except:
@@ -571,7 +577,7 @@ def streetcar():
             return result
 
         else:
-            return self._run_sync(*args, **kwargs)
+            return self._run_sync(True, *args, **kwargs)
 
     def run(self, *args, **kwargs):
         with MultiContext(self.WITH_CONTEXT):
@@ -585,6 +591,10 @@ def run(self, *args, **kwargs):
             else:
                 return self._run(*args, **kwargs)
 
+    def refresh(self):
+        self._create_event()
+        return self._run_sync(False, *self._run_args, **self._run_kwargs)
+
     def command(self):
         return None
 

mailpile/config.py

@@ -1416,6 +1416,7 @@ def _unlocked_save(self):
             for mail_source in self.mail_sources.values():
                 mail_source.wake_up(after=delay)
                 delay += 2
+        self.command_cache.mark_dirty([u'!config'])
 
     def _find_mail_source(self, mbx_id):
         for src in self.sources.values():
@@ -1867,7 +1868,7 @@ def start_httpd(sspec=None):
                     config.other_workers.append(w)
 
         # Update the cron jobs, if necessary
-        if config.cron_worker:
+        if config.cron_worker and config.event_log:
             # Schedule periodic rescanning, if requested.
             rescan_interval = config.prefs.rescan_interval
             if rescan_interval:
@@ -1883,7 +1884,8 @@ def rescan():
             def refresh_command_cache():
                 config.async_worker.add_unique_task(
                     config.background, 'refresh_command_cache',
-                    config.command_cache.refresh)
+                    lambda: config.command_cache.refresh(
+                        extend=60, event_log=config.event_log))
             config.cron_worker.add_task('refresh_command_cache',
                                         5, refresh_command_cache)
 

mailpile/plugins/crypto_utils.py

@@ -121,6 +121,7 @@ class GPGKeyImportFromMail(Search):
                 'crypto/gpg/importkeyfrommail', '<mid>')
     HTTP_CALLABLE = ('POST', )
     HTTP_QUERY_VARS = {'mid': 'Message ID', 'att': 'Attachment ID'}
+    COMMAND_CACHE_TTL = 0
 
     class CommandResult(Command.CommandResult):
         def __init__(self, *args, **kwargs):
@@ -203,6 +204,7 @@ class GPGUsageStatistics(Search):
                 'crypto/gpg/statistics', '<address>')
     HTTP_CALLABLE = ('GET', )
     HTTP_QUERY_VARS = {'address': 'E-mail address'}
+    COMMAND_CACHE_TTL = 0
 
     class CommandResult(Command.CommandResult):
         def __init__(self, *args, **kwargs):
@@ -251,8 +253,6 @@ def command(self):
         return self._success("Got statistics for address", res)
 
 
-
-
 _plugins.register_commands(GPGKeySearch)
 _plugins.register_commands(GPGKeyReceive)
 _plugins.register_commands(GPGKeyImport)

mailpile/plugins/search.py

@@ -157,9 +157,18 @@ def _do_search(self, search=None, process_args=False):
 
         return session, idx, self._start, self._num
 
-    def cache_requirements(self):
+    def cache_requirements(self, result):
         msgs = self.session.results[self._start:self._start + self._num]
-        return set(self.session.searched + ['msg:%s' % i for i in msgs])
+        def fix_term(term):
+            # Terms are reversed in the search engine...
+            if term[:1] in ['-', '+']:
+                term = term[1:]
+            term = ':'.join(reversed(term.split(':', 1)))
+            return unicode(term)
+        return set(
+            [fix_term(t) for t in self.session.searched] +
+            [u'%s:msg' % i for i in msgs]
+        )
 
     def command(self):
         session, idx, start, num = self._do_search()
@@ -236,6 +245,7 @@ class View(Search):
     HTTP_QUERY_VARS = {
         'mid': 'metadata-ID'
     }
+    COMMAND_CACHE_TTL = 0
 
     class RawResult(dict):
         def _decode(self):

mailpile/plugins/tags.py

@@ -448,8 +448,9 @@ class ListTags(TagCommand):
     HTTP_STRICT_VARS = False
     COMMAND_CACHE_TTL = 3600
 
-    def cache_requirements(self):
-        return set(['config'])
+    def cache_requirements(self, result):
+        return set([u'!config'] +
+                   [u'%s:in' % ti['slug'] for ti in result.result['tags']])
 
     class CommandResult(TagCommand.CommandResult):
         def as_text(self):

mailpile/search.py

@@ -1262,6 +1262,7 @@ def index_message(self, session, msg_mid, msg_id, msg, msg_size, msg_ts,
                 # FIXME: we just ignore garbage
                 pass
 
+        self.config.command_cache.mark_dirty(set([u'mail:all']) | keywords)
         return keywords, snippet
 
     def get_msg_at_idx_pos(self, msg_idx):
@@ -1289,16 +1290,19 @@ def set_msg_at_idx_pos(self, msg_idx, msg_info, original_line=None):
                 for order in self.INDEX_SORT:
                     self.INDEX_SORT[order].append(0)
 
+        msg_thr_mid = msg_info[self.MSG_THREAD_MID]
         self.INDEX[msg_idx] = original_line or self.m2l(msg_info)
-        self.INDEX_THR[msg_idx] = int(msg_info[self.MSG_THREAD_MID], 36)
+        self.INDEX_THR[msg_idx] = int(msg_thr_mid, 36)
         self.MSGIDS[msg_info[self.MSG_ID]] = msg_idx
         for msg_ptr in msg_info[self.MSG_PTRS].split(','):
             self.PTRS[msg_ptr] = msg_idx
         self.update_msg_sorting(msg_idx, msg_info)
         self.update_msg_tags(msg_idx, msg_info)
 
         if not original_line:
-            self.config.command_cache.mark_dirty(['msg:%s' % msg_idx])
+            self.config.command_cache.mark_dirty([u'mail:all',
+                                                  u'%s:msg' % msg_idx,
+                                                  u'%s:thread' % msg_thr_mid])
             CachedSearchResultSet.DropCaches(msg_idxs=[msg_idx])
             self.MODIFIED.add(msg_idx)
             try:
@@ -1367,8 +1371,8 @@ def add_tag(self, session, tag_id,
                 self.TAGS[tag_id] = eids
         try:
             self.config.command_cache.mark_dirty(
-                ['in:%s' % self.config.tags[tag_id].slug] +
-                ['msg:%s' % eid for eid in added])
+                [u'mail:all', u'%s:in' % self.config.tags[tag_id].slug] +
+                [u'%s:msg' % eid for eid in added])
         except:
             pass
         return added
@@ -1415,8 +1419,8 @@ def remove_tag(self, session, tag_id,
                 self.TAGS[tag_id] -= eids
         try:
             self.config.command_cache.mark_dirty(
-                ['in:%s' % self.config.tags[tag_id].slug] +
-                ['msg:%s' % eid for eid in removed])
+                [u'%s:in' % self.config.tags[tag_id].slug] +
+                [u'%s:msg' % eid for eid in removed])
         except:
             pass
         return removed

static/default/html/layouts/content-wide.html

@@ -1 +1 @@
-{% block content %}{% endblock %}
+<div{% if state and state.cache_id %} class="content-{{ state.cache_id }}"{% endif %}>{% block content %}{% endblock %}</div>

static/default/html/layouts/content.html

@@ -1 +1 @@
-{% block content %}{% endblock %}
+<div{% if state and state.cache_id %} class="content-{{ state.cache_id }}"{% endif %}>{% block content %}{% endblock %}</div>

static/default/html/layouts/full-wide.html

@@ -10,12 +10,12 @@
 </head>
 <body>
 {% include("partials/topbar.html") %}
-<div id="content-wide">
+<div id="content-wide"><div{% if state and state.cache_id %} class="content-{{ state.cache_id }}"{% endif %}>
   {% block content %}{{results}}{% endblock %}
-</div>
+</div></div>
 {% include("partials/hidden.html") %}
 {% include("partials/tooltips.html") %}
 {% include("partials/modals.html") %}
 {% include("partials/javascript.html") %}
 </body>
-</html>
+</html>

static/default/html/layouts/full.html

@@ -29,9 +29,9 @@
   {% endif %}
   {% endif %}
   </div>
-  <div id="content-view">
+  <div id="content-view"><div{% if state and state.cache_id %} class="content-{{ state.cache_id }}"{% endif %}>
     {% block content %}{{results}}{% endblock %}
-  </div>
+  </div></div>
 </div>
 {% include("partials/hidden.html") %}
 {% include("partials/tooltips.html") %}