Implement caching framework

This PR includes:

- Caching and fallback framework for listings (e.g. A-Z, or categories)
- Now troubleshooting option to invalidate caches

addon.py

@@ -29,6 +29,9 @@ def router(params_string):
     _kodi = kodiwrapper.KodiWrapper(_ADDON_HANDLE, _ADDON_URL, addon)
     _kodi.log_access(_ADDON_URL, params_string)
 
+    if action == actions.INVALIDATE_CACHES:
+        _kodi.invalidate_caches()
+        return
     if action == actions.CLEAR_COOKIES:
         from resources.lib.vrtplayer import tokenresolver
         _tokenresolver = tokenresolver.TokenResolver(_kodi)
@@ -52,7 +55,7 @@ def router(params_string):
         _favorites.unfollow(program=params.get('program'), path=params.get('path'))
         return
     if action == actions.REFRESH_FAVORITES:
-        _favorites.update_favorites()
+        _favorites.get_favorites(ttl=0)
         return
 
     from resources.lib.vrtplayer import vrtapihelper, vrtplayer

resources/language/resource.language.en_gb/strings.po

@@ -286,6 +286,10 @@ msgctxt "#30830"
 msgid "When enabled, menus are being cached so they work more quickly. Only in very specific use-cases this is problematic as new episodes may not appear when they are expected."
 msgstr ""
 
+msgctxt "#30831"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
+msgstr ""
+
 msgctxt "#30840"
 msgid "Playback"
 msgstr ""
@@ -339,26 +343,30 @@ msgid "Refresh favorites"
 msgstr ""
 
 msgctxt "#30867"
-msgid "Streaming"
+msgid "Invalidate HTTP caches"
 msgstr ""
 
 msgctxt "#30869"
-msgid "Use InputStream Adaptive"
+msgid "Streaming"
 msgstr ""
 
 msgctxt "#30871"
-msgid "InputStream Adaptive settings..."
+msgid "Use InputStream Adaptive"
 msgstr ""
 
 msgctxt "#30873"
-msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
+msgid "InputStream Adaptive settings..."
 msgstr ""
 
 msgctxt "#30875"
-msgid "Logging"
+msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr ""
 
 msgctxt "#30877"
+msgid "Logging"
+msgstr ""
+
+msgctxt "#30879"
 msgid "Log level"
 msgstr ""
 

resources/language/resource.language.nl_nl/strings.po

@@ -275,6 +275,10 @@ msgctxt "#30829"
 msgid "Enable menu caching"
 msgstr "Gebruik menu caching"
 
+msgctxt "#30831"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
+msgstr "Gebruik HTTP caching [COLOR gray][I](experimenteel)[/I][/COLOR]"
+
 msgctxt "#30840"
 msgid "Playback"
 msgstr "Afspelen"
@@ -315,27 +319,31 @@ msgctxt "#30065"
 msgid "Refresh favorites"
 msgstr "Ververs gevolgde programma's"
 
-msgctxt "#30067"
+msgctxt "#30867"
+msgid "Invalidate HTTP caches"
+msgstr "Invalideer HTTP caches"
+
+msgctxt "#30069"
 msgid "Streaming"
 msgstr "Streaming"
 
-msgctxt "#30869"
+msgctxt "#30870"
 msgid "Use InputStream Adaptive"
 msgstr "Gebruik InputStream Adaptive"
 
-msgctxt "#30071"
+msgctxt "#30073"
 msgid "InputStream Adaptive settings..."
 msgstr "InputStream Adaptive instellingen..."
 
-msgctxt "#30073"
+msgctxt "#30075"
 msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr "Installeer Widevine... [COLOR gray][I](nodig voor DRM content)[/I][/COLOR]"
 
-msgctxt "#30075"
+msgctxt "#30077"
 msgid "Logging"
 msgstr "Logboek"
 
-msgctxt "#30077"
+msgctxt "#30079"
 msgid "Log level"
 msgstr "Log level"
 

resources/lib/kodiwrappers/kodiwrapper.py

@@ -106,11 +106,11 @@ def __init__(self, handle, url, addon):
         self._addon_id = addon.getAddonInfo('id')
         self._max_log_level = log_levels.get(self.get_setting('max_log_level'), 3)
         self._usemenucaching = self.get_setting('usemenucaching') == 'true'
+        self._cache_path = self.get_userdata_path() + 'cache/'
         self._system_locale_works = self.set_locale()
 
     def install_widevine(self):
-        import xbmcgui
-        ok = xbmcgui.Dialog().yesno(self.localize(30971), self.localize(30972))
+        ok = self.show_yesno_dialog(heading=self.localize(30971), message=self.localize(30972))
         if not ok:
             return
         try:
@@ -235,6 +235,12 @@ def show_notification(self, heading='', message='', icon='info', time=4000):
             heading = self._addon.getAddonInfo('name')
         xbmcgui.Dialog().notification(heading=heading, message=message, icon=icon, time=time)
 
+    def show_yesno_dialog(self, heading='', message=''):
+        import xbmcgui
+        if not heading:
+            heading = self._addon.getAddonInfo('name')
+        return xbmcgui.Dialog().yesno(heading=self.localize(30971), line1=self.localize(30972))
+
     def set_locale(self):
         import locale
         locale_lang = self.get_global_setting('locale.language').split('.')[-1]
@@ -243,7 +249,7 @@ def set_locale(self):
             locale.setlocale(locale.LC_ALL, locale_lang)
             return True
         except Exception as e:
-            self.log_notice(e, 'Debug')
+            self.log_notice('Failed to set locale: %s' % e, 'Debug')
             return False
 
     def localize(self, string_id):
@@ -353,9 +359,19 @@ def get_addon_path(self):
     def get_path(self, path):
         return xbmc.translatePath(path)
 
-    def make_dir(self, path):
+    def listdir(self, path):
         import xbmcvfs
-        xbmcvfs.mkdir(path)
+        return xbmcvfs.listdir(path)
+
+    def mkdir(self, path):
+        import xbmcvfs
+        self.log_notice("Create directory '%s'." % path, 'Debug')
+        return xbmcvfs.mkdir(path)
+
+    def mkdirs(self, path):
+        import xbmcvfs
+        self.log_notice("Recursively create directory '%s'." % path, 'Debug')
+        return xbmcvfs.mkdirs(path)
 
     def check_if_path_exists(self, path):
         import xbmcvfs
@@ -374,8 +390,91 @@ def stat_file(self, path):
 
     def delete_file(self, path):
         import xbmcvfs
+        self.log_notice("Delete file '%s'." % path, 'Debug')
         return xbmcvfs.delete(path)
 
+    def md5(self, path):
+        import hashlib
+        with self.open_file(path) as f:
+            return hashlib.md5(f.read().encode('utf-8'))
+
+    def human_delta(self, seconds):
+        import math
+        days = int(math.floor(seconds / (24 * 60 * 60)))
+        seconds = seconds % (24 * 60 * 60)
+        hours = int(math.floor(seconds / (60 * 60)))
+        seconds = seconds % (60 * 60)
+        if days:
+            return '%d day%s and %d hour%s' % (days, 's' if days != 1 else '', hours, 's' if hours != 1 else '')
+        minutes = int(math.floor(seconds / 60))
+        seconds = seconds % 60
+        if hours:
+            return '%d hour%s and %d minute%s' % (hours, 's' if hours != 1 else '', minutes, 's' if minutes != 1 else '')
+        if minutes:
+            return '%d minute%s and %d second%s' % (minutes, 's' if minutes != 1 else '', seconds, 's' if seconds != 1 else '')
+        return '%d second%s' % (seconds, 's' if seconds != 1 else '')
+
+    def get_cache(self, path, ttl=None):
+        if self.get_setting('usehttpcaching') == 'false':
+            return None
+
+        fullpath = self._cache_path + path
+        if not self.check_if_path_exists(fullpath):
+            return None
+
+        import time
+        mtime = self.stat_file(fullpath).st_mtime()
+        now = time.mktime(time.localtime())
+        if ttl is None or now - mtime < ttl:
+            import json
+            if ttl is None:
+                self.log_notice("Cache '%s' is forced from cache." % path, 'Debug')
+            else:
+                self.log_notice("Cache '%s' is fresh, expires in %s." % (path, self.human_delta(mtime + ttl - now)), 'Debug')
+            with self.open_file(fullpath, 'r') as f:
+                try:
+                    # return json.load(f, encoding='utf-8')
+                    return json.load(f)
+                except ValueError:
+                    return None
+
+        return None
+
+    def update_cache(self, path, data):
+        if self.get_setting('usehttpcaching') == 'false':
+            return
+
+        import hashlib
+        import json
+        fullpath = self._cache_path + path
+        if self.check_if_path_exists(fullpath):
+            md5 = self.md5(fullpath)
+        else:
+            md5 = 0
+            # Create cache directory if missing
+            if not self.check_if_path_exists(self._cache_path):
+                self.mkdirs(self._cache_path)
+
+        # Avoid writes if possible (i.e. SD cards)
+        if md5 != hashlib.md5(json.dumps(data).encode('utf-8')):
+            self.log_notice("Write cache '%s'." % path, 'Debug')
+            with self.open_file(path, 'w') as f:
+                # json.dump(data, f, encoding='utf-8')
+                json.dump(data, f)
+        else:
+            # Update timestamp
+            import os
+            self.log_notice("Cache '%s' has not changed, updating mtime only." % path, 'Debug')
+            os.utime(path)
+
+    def invalidate_cache(self, path):
+        self.delete_file(self._cache_path + path)
+
+    def invalidate_caches(self):
+        _, files = self.listdir(self._cache_path)
+        for f in files:
+            self.delete_file(self._cache_path + f)
+
     def container_refresh(self):
         self.log_notice('Execute: Container.Refresh', 'Debug')
         xbmc.executebuiltin('Container.Refresh')

resources/lib/vrtplayer/__init__.py

@@ -119,6 +119,7 @@ class actions:
     CLEAR_COOKIES = 'clearcookies'
     FOLLOW = 'follow'
     INSTALL_WIDEVINE = 'installwidevine'
+    INVALIDATE_CACHES = 'invalidatecaches'
     LISTING_ALL_EPISODES = 'listingallepisodes'
     LISTING_AZ_TVSHOWS = 'listingaztvshows'
     LISTING_CATEGORIES = 'listingcategories'

resources/lib/vrtplayer/favorites.py

@@ -3,8 +3,6 @@
 # GNU General Public License v3.0 (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, unicode_literals
-import json
-import time
 
 from resources.lib.vrtplayer import tokenresolver
 
@@ -21,36 +19,38 @@ def __init__(self, _kodi):
         self._tokenresolver = tokenresolver.TokenResolver(_kodi)
         self._proxies = _kodi.get_proxies()
         install_opener(build_opener(ProxyHandler(self._proxies)))
-        self._cache_file = _kodi.get_userdata_path() + 'favorites.json'
         self._favorites = None
         if _kodi.get_setting('usefavorites') == 'true' and _kodi.has_credentials():
-            self.get_favorites()
+            # Get favorites from cache if fresh
+            self.get_favorites(ttl=60 * 60)
 
     def is_activated(self):
         return self._favorites is not None
 
-    def get_favorites(self):
-        if self._kodi.check_if_path_exists(self._cache_file):
-            if self._kodi.stat_file(self._cache_file).st_mtime() > time.mktime(time.localtime()) - (2 * 60):
-                self._kodi.log_notice('CACHE: %s vs %s' % (self._kodi.stat_file(self._cache_file).st_mtime(), time.mktime(time.localtime()) - (5 * 60)), 'Debug')
-                with self._kodi.open_file(self._cache_file) as f:
-                    self._favorites = json.loads(f.read())
-                return
-        self.update_favorites()
-
-    def update_favorites(self):
-        xvrttoken = self._tokenresolver.get_fav_xvrttoken()
-        headers = {
-            'authorization': 'Bearer ' + xvrttoken,
-            'content-type': 'application/json',
-            # 'Cookie': 'X-VRT-Token=' + xvrttoken,
-            'Referer': 'https://www.vrt.be/vrtnu',
-        }
-        req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
-        self._favorites = json.loads(urlopen(req).read())
-        self.write_favorites()
+    def get_favorites(self, ttl=None):
+        import json
+        api_json = self._kodi.get_cache('favorites.json', ttl)
+        if not api_json:
+            xvrttoken = self._tokenresolver.get_fav_xvrttoken()
+            headers = {
+                'authorization': 'Bearer ' + xvrttoken,
+                'content-type': 'application/json',
+                # 'Cookie': 'X-VRT-Token=' + xvrttoken,
+                'Referer': 'https://www.vrt.be/vrtnu',
+            }
+            req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
+            self._kodi.log_notice('URL post: https://video-user-data.vrt.be/favorites', 'Verbose')
+            try:
+                api_json = json.load(urlopen(req))
+            except Exception:
+                # Force favorites from cache
+                api_json = self._kodi.get_cache('favorites.json', ttl=None)
+            else:
+                self._kodi.update_cache('favorites.json', api_json)
+        self._favorites = api_json
 
     def set_favorite(self, program, path, value=True):
+        import json
         if value is not self.is_favorite(path):
             xvrttoken = self._tokenresolver.get_fav_xvrttoken()
             headers = {
@@ -69,11 +69,7 @@ def set_favorite(self, program, path, value=True):
                 self._kodi.log_error("Failed to follow program '%s' at VRT NU" % path)
             # NOTE: Updates to favorites take a longer time to take effect, so we keep our own cache and use it
             self._favorites[self.uuid(path)] = dict(value=payload)
-            self.write_favorites()
-
-    def write_favorites(self):
-        with self._kodi.open_file(self._cache_file, 'w') as f:
-            f.write(json.dumps(self._favorites))
+            self._kodi.update_cache('favorites.json', self._favorites)
 
     def is_favorite(self, path):
         value = False
@@ -103,3 +99,7 @@ def names(self):
 
     def titles(self):
         return [p.get('value').get('title') for p in self._favorites.values() if p.get('value').get('isFavorite')]
+
+    def invalidate_cache(self):
+        self._kodi.invalidate_cache('offline-filtered.json')
+        self._kodi.invalidate_cache('recent-filtered.json')

resources/lib/vrtplayer/streamservice.py

@@ -34,13 +34,13 @@ def __init__(self, _kodi, _tokenresolver):
         self._vualto_license_url = None
 
     def _get_vualto_license_url(self):
-        self._vualto_license_url = json.loads(urlopen(self._VUPLAY_API_URL).read()).get('drm_providers', dict()).get('widevine', dict()).get('la_url')
+        self._vualto_license_url = json.load(urlopen(self._VUPLAY_API_URL)).get('drm_providers', dict()).get('widevine', dict()).get('la_url')
         self._kodi.log_notice('URL get: ' + unquote(self._VUPLAY_API_URL), 'Verbose')
 
     def _create_settings_dir(self):
         settingsdir = self._kodi.get_userdata_path()
         if not self._kodi.check_if_path_exists(settingsdir):
-            self._kodi.make_dir(settingsdir)
+            self._kodi.mkdir(settingsdir)
 
     def _get_license_key(self, key_url, key_type='R', key_headers=None, key_value=None):
         ''' Generates a propery license key value
@@ -152,7 +152,7 @@ def _get_stream_json(self, api_data):
                 api_data.video_id + '?vrtPlayerToken=' + playertoken + '&client=' + api_data.client
             self._kodi.log_notice('URL get: ' + unquote(api_url), 'Verbose')
             try:
-                stream_json = json.loads(urlopen(api_url).read())
+                stream_json = json.load(urlopen(api_url))
             except HTTPError as e:
                 stream_json = json.loads(e.read())