Merge pull request #4077 from rtibbles/cache_invalidate_at_boot

Don't invalidate caches except at boot when using disk based caches

kalite/caching/__init__.py

@@ -24,6 +24,9 @@
 from fle_utils.internet.webcache import *
 from kalite import i18n, topic_tools
 
+import os
+import glob
+
 def create_cache_entry(path=None, url_name=None, cache=None, force=False):
     """Create a cache entry"""
 
@@ -65,7 +68,18 @@ def invalidate_all_caches():
     call in here.
     """
     invalidate_inmemory_caches()
-    initialize_content_caches(force=True)
+    if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
+
+        # The underlying assumption here is that if generating in memory caches is too onerous a task to conduct
+        # at every system start up, then it is too onerous a task to conduct during server operation.
+        # We defer the regeneration of these caches to next system startup, by deleting the existing disk based
+        # copies of these caches.
+        # This will prompt the caches to be recreated at next system start up, and the disk based copies to be rewritten.
+
+        for filename in glob.glob(os.path.join(settings.CHANNEL_DATA_PATH, "*.cache")):
+            os.remove(filename)
+    else:
+        initialize_content_caches(force=True)
     if caching_is_enabled():
         invalidate_web_cache()
     logging.debug("Great success emptying all caches.")
@@ -81,4 +95,4 @@ def initialize_content_caches(force=False):
         logging.info("Preloading content data for language {lang}.".format(lang=lang))
         topic_tools.get_content_cache(force=force, annotate=True, language=lang)
         logging.info("Preloading topic tree data for language {lang}.".format(lang=lang))
-        topic_tools.get_topic_tree(force=force, annotate=True, language=lang)
+        topic_tools.get_topic_tree(force=force, annotate=True, language=lang)

kalite/updates/management/commands/videodownload.py

@@ -223,7 +223,7 @@ def youtube_dl_cb(stats, progress_callback, *args, **kwargs):
                 "num_handled_videos": len(handled_youtube_ids),
                 "num_total_videos": len(handled_youtube_ids) + len(failed_youtube_ids),
             })
-            caching.initialize_content_caches()
+            caching.invalidate_all_caches()
 
         except Exception as e:
             self.cancel(stage_status="error", notes=_("Error: %(error_msg)s") % {"error_msg": e})

kalite/updates/views.py

@@ -27,7 +27,9 @@ def update_context(request):
 @render_to("updates/update_videos.html")
 def update_videos(request, max_to_show=4):
     context = update_context(request)
-    messages.warning(request, _('For low-powered devices like the Raspberry Pi, please download videos one at a time.'))
+    messages.warning(request, _('For low-powered devices like the Raspberry Pi, please download less than 25 videos at a time.'))
+    if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
+        messages.warning(request, _('After video download, the server must be restarted for them to be available to users.'))
     context.update({
         "video_count": VideoFile.objects.filter(percent_complete=100).count(),
     })