Replace Mock with xbmc library replacement

This replaces a quite complex Mock() setup to mock kodiwrapper with an
alternative implementation of the xbmc libraries.

The benefit is that we don't have to fake every possible kodiwrapper
functionality, but only the xbmc functionality that kodiwrapper is
using. This means we are now also testing kodiwrapper to a greater
extent.

addon.py

@@ -52,7 +52,7 @@ def router(params_string):
         _favorites.unfollow(program=params.get('program'), path=params.get('path'))
         return
     if action == actions.REFRESH_FAVORITES:
-        _favorites.update_favorites()
+        _favorites.get_favorites(ttl=0)
         return
 
     from resources.lib.vrtplayer import vrtapihelper, vrtplayer

resources/language/resource.language.en_gb/strings.po

@@ -339,26 +339,30 @@ msgid "Refresh favorites"
 msgstr ""
 
 msgctxt "#30867"
-msgid "Streaming"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
 msgstr ""
 
 msgctxt "#30869"
-msgid "Use InputStream Adaptive"
+msgid "Streaming"
 msgstr ""
 
 msgctxt "#30871"
-msgid "InputStream Adaptive settings..."
+msgid "Use InputStream Adaptive"
 msgstr ""
 
 msgctxt "#30873"
-msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
+msgid "InputStream Adaptive settings..."
 msgstr ""
 
 msgctxt "#30875"
-msgid "Logging"
+msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr ""
 
 msgctxt "#30877"
+msgid "Logging"
+msgstr ""
+
+msgctxt "#30879"
 msgid "Log level"
 msgstr ""
 

resources/language/resource.language.nl_nl/strings.po

@@ -315,27 +315,31 @@ msgctxt "#30065"
 msgid "Refresh favorites"
 msgstr "Ververs gevolgde programma's"
 
-msgctxt "#30067"
+msgctxt "#30867"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
+msgstr "Gebruik HTTP caching [COLOR gray][I](experimenteel)[/I][/COLOR]"
+
+msgctxt "#30069"
 msgid "Streaming"
 msgstr "Streaming"
 
-msgctxt "#30869"
+msgctxt "#30870"
 msgid "Use InputStream Adaptive"
 msgstr "Gebruik InputStream Adaptive"
 
-msgctxt "#30071"
+msgctxt "#30073"
 msgid "InputStream Adaptive settings..."
 msgstr "InputStream Adaptive instellingen..."
 
-msgctxt "#30073"
+msgctxt "#30075"
 msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr "Installeer Widevine... [COLOR gray][I](nodig voor DRM content)[/I][/COLOR]"
 
-msgctxt "#30075"
+msgctxt "#30077"
 msgid "Logging"
 msgstr "Logboek"
 
-msgctxt "#30077"
+msgctxt "#30079"
 msgid "Log level"
 msgstr "Log level"
 

resources/lib/kodiwrappers/kodiwrapper.py

@@ -106,11 +106,11 @@ def __init__(self, handle, url, addon):
         self._addon_id = addon.getAddonInfo('id')
         self._max_log_level = log_levels.get(self.get_setting('max_log_level'), 3)
         self._usemenucaching = self.get_setting('usemenucaching') == 'true'
+        self._cache_path = self.get_userdata_path() + 'cache/'
         self._system_locale_works = self.set_locale()
 
     def install_widevine(self):
-        import xbmcgui
-        ok = xbmcgui.Dialog().yesno(self.localize(30971), self.localize(30972))
+        ok = self.show_yesno_dialog(heading=self.localize(30971), message=self.localize(30972))
         if not ok:
             return
         try:
@@ -227,14 +227,20 @@ def show_ok_dialog(self, heading='', message=''):
         import xbmcgui
         if not heading:
             heading = self._addon.getAddonInfo('name')
-        xbmcgui.Dialog().ok(heading=heading, message=message)
+        xbmcgui.Dialog().ok(heading=heading, line1=message)
 
     def show_notification(self, heading='', message='', icon='info', time=4000):
         import xbmcgui
         if not heading:
             heading = self._addon.getAddonInfo('name')
         xbmcgui.Dialog().notification(heading=heading, message=message, icon=icon, time=time)
 
+    def show_yesno_dialog(self, heading='', message=''):
+        import xbmcgui
+        if not heading:
+            heading = self._addon.getAddonInfo('name')
+        return xbmcgui.Dialog().yesno(heading=self.localize(30971), line1=self.localize(30972))
+
     def set_locale(self):
         import locale
         locale_lang = self.get_global_setting('locale.language').split('.')[-1]
@@ -376,6 +382,56 @@ def delete_file(self, path):
         import xbmcvfs
         return xbmcvfs.delete(path)
 
+    def md5(self, path):
+        import hashlib
+        with self.open_file(path) as f:
+            return hashlib.md5(f.read().encode('utf-8'))
+
+    def get_cache(self, path, ttl=None):
+        if self.get_setting('usehttpcaching') == 'false':
+            return None
+
+        path = self._cache_path + path
+        if not self.check_if_path_exists(path):
+            return None
+
+        import time
+        if ttl is None or self.stat_file(path).st_mtime() > time.mktime(time.localtime()) - ttl:
+            if ttl is None:
+                self.log_notice("Cache '%s' is forced from cache." % path, 'Debug')
+            else:
+                self.log_notice("Cache '%s' is fresh, within ttl of %s seconds." % (path, ttl), 'Debug')
+            with self.open_file(path) as f:
+                return f.read()
+
+        return None
+
+    def update_cache(self, path, data):
+        if self.get_setting('usehttpcaching') == 'false':
+            return
+
+        import hashlib
+        path = self._cache_path + path
+        if self.check_if_path_exists(path):
+            md5 = self.md5(path)
+        else:
+            md5 = 0
+            # Create cache directory if missing
+            if not self.check_if_path_exists(self._cache_path):
+                self.log_notice("Create path '%s'." % self._cache_path, 'Debug')
+                self.make_dir(self._cache_path)
+
+        # Avoid writes if possible (i.e. SD cards)
+        if md5 != hashlib.md5(data):
+            self.log_notice("Write cache '%s'." % path, 'Debug')
+            with self.open_file(path, 'wb') as f:
+                f.write(data)
+        else:
+            # Update timestamp
+            import os
+            self.log_notice("Cache '%s' has not changed, updating mtime only." % path, 'Debug')
+            os.utime(path)
+
     def container_refresh(self):
         self.log_notice('Execute: Container.Refresh', 'Debug')
         xbmc.executebuiltin('Container.Refresh')

resources/lib/vrtplayer/favorites.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, division, unicode_literals
 import json
-import time
 
 from resources.lib.vrtplayer import tokenresolver
 
@@ -21,34 +20,34 @@ def __init__(self, _kodi):
         self._tokenresolver = tokenresolver.TokenResolver(_kodi)
         self._proxies = _kodi.get_proxies()
         install_opener(build_opener(ProxyHandler(self._proxies)))
-        self._cache_file = _kodi.get_userdata_path() + 'favorites.json'
         self._favorites = None
         if _kodi.get_setting('usefavorites') == 'true' and _kodi.has_credentials():
-            self.get_favorites()
+            # Get favorites from cache if fresh
+            self.get_favorites(ttl=60 * 60)
 
     def is_activated(self):
         return self._favorites is not None
 
-    def get_favorites(self):
-        if self._kodi.check_if_path_exists(self._cache_file):
-            if self._kodi.stat_file(self._cache_file).st_mtime() > time.mktime(time.localtime()) - (2 * 60):
-                self._kodi.log_notice('CACHE: %s vs %s' % (self._kodi.stat_file(self._cache_file).st_mtime(), time.mktime(time.localtime()) - (5 * 60)), 'Debug')
-                with self._kodi.open_file(self._cache_file) as f:
-                    self._favorites = json.loads(f.read())
-                return
-        self.update_favorites()
-
-    def update_favorites(self):
-        xvrttoken = self._tokenresolver.get_fav_xvrttoken()
-        headers = {
-            'authorization': 'Bearer ' + xvrttoken,
-            'content-type': 'application/json',
-            # 'Cookie': 'X-VRT-Token=' + xvrttoken,
-            'Referer': 'https://www.vrt.be/vrtnu',
-        }
-        req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
-        self._favorites = json.loads(urlopen(req).read())
-        self.write_favorites()
+    def get_favorites(self, ttl=None):
+        data = self._kodi.get_cache('favorites.json', ttl)
+        if not data:
+            xvrttoken = self._tokenresolver.get_fav_xvrttoken()
+            headers = {
+                'authorization': 'Bearer ' + xvrttoken,
+                'content-type': 'application/json',
+                # 'Cookie': 'X-VRT-Token=' + xvrttoken,
+                'Referer': 'https://www.vrt.be/vrtnu',
+            }
+            req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
+            self._kodi.log_notice('URL post: https://video-user-data.vrt.be/favorites', 'Verbose')
+            try:
+                data = urlopen(req).read()
+            except Exception:
+                # Force favorites from cache
+                data = self._kodi.get_cache('favorites.json', ttl=None)
+            else:
+                self._kodi.update_cache('favorites.json', data)
+        self._favorites = json.loads(data)
 
     def set_favorite(self, program, path, value=True):
         if value is not self.is_favorite(path):
@@ -69,11 +68,7 @@ def set_favorite(self, program, path, value=True):
                 self._kodi.log_error("Failed to follow program '%s' at VRT NU" % path)
             # NOTE: Updates to favorites take a longer time to take effect, so we keep our own cache and use it
             self._favorites[self.uuid(path)] = dict(value=payload)
-            self.write_favorites()
-
-    def write_favorites(self):
-        with self._kodi.open_file(self._cache_file, 'w') as f:
-            f.write(json.dumps(self._favorites))
+            self._kodi.update_cache('favorites.json', json.dumps(self._favorites).encode('utf-8'))
 
     def is_favorite(self, path):
         value = False

resources/lib/vrtplayer/tokenresolver.py

@@ -100,10 +100,10 @@ def _get_cached_token(self, path, token_name):
             now = datetime.now(dateutil.tz.tzlocal())
             exp = dateutil.parser.parse(token.get('expirationDate'))
             if exp > now:
-                self._kodi.log_notice('Got cached token', 'Verbose')
+                self._kodi.log_notice("Got cached token '%s'" % path, 'Verbose')
                 cached_token = token.get(token_name)
             else:
-                self._kodi.log_notice('Cached token deleted', 'Verbose')
+                self._kodi.log_notice("Cached token '%s' deleted" % path, 'Verbose')
                 self._kodi.delete_file(path)
         return cached_token
 

resources/lib/vrtplayer/tvguide.py

@@ -126,8 +126,20 @@ def show_episodes(self, date, channel):
             epg += timedelta(days=-1)
         datelong = self._kodi.localize_datelong(epg)
         api_url = epg.strftime(self.VRT_TVGUIDE)
-        self._kodi.log_notice('URL get: ' + api_url, 'Verbose')
-        schedule = json.loads(urlopen(api_url).read())
+
+        if date in ('today', 'yesterday', 'tomorrow'):
+            cache_file = 'schedule.%s.json' % date
+            # Try the cache if it is fresh
+            data = self._kodi.get_cache(cache_file, ttl=60 * 60)
+            if data:
+                schedule = json.loads(data)
+            else:
+                self._kodi.log_notice('URL get: ' + api_url, 'Verbose')
+                schedule = json.loads(urlopen(api_url).read())
+                self._kodi.update_cache(cache_file, json.dumps(schedule))
+        else:
+            schedule = json.loads(urlopen(api_url).read())
+
         name = channel
         try:
             channel = next(c for c in CHANNELS if c.get('name') == name)
@@ -160,7 +172,7 @@ def show_episodes(self, date, channel):
             if url:
                 video_url = statichelper.add_https_method(url)
                 url_dict = dict(action=actions.PLAY, video_url=video_url)
-                if start_date < now <= end_date:  # Now playing
+                if start_date <= now <= end_date:  # Now playing
                     metadata.title = '[COLOR yellow]%s[/COLOR] %s' % (label, self._kodi.localize(30302))
                 else:
                     metadata.title = label

resources/lib/vrtplayer/vrtapihelper.py

@@ -34,17 +34,25 @@ def get_tvshow_items(self, category=None, channel=None, filtered=False):
 
         if category:
             params['facets[categories]'] = category
+            cache_file = 'category.%s.json' % category
 
         if channel:
             params['facets[programBrands]'] = channel
+            cache_file = 'channel.%s.json' % channel
 
         # If no facet-selection is done, we return the A-Z listing
         if not category and not channel:
             params['facets[transcodingStatus]'] = 'AVAILABLE'
+            cache_file = 'programs.json'
 
         api_url = self._VRTNU_SUGGEST_URL + '?' + urlencode(params)
-        self._kodi.log_notice('URL get: ' + unquote(api_url), 'Verbose')
-        api_json = json.loads(urlopen(api_url).read())
+        # Try the cache if it is fresh
+        data = self._kodi.get_cache(cache_file, ttl=60 * 60)
+        if not data:
+            self._kodi.log_notice('URL get: ' + unquote(api_url), 'Verbose')
+            data = urlopen(api_url).read()
+            self._kodi.update_cache(cache_file, data)
+        api_json = json.loads(data)
         return self._map_to_tvshow_items(api_json, filtered=statichelper.is_filtered(filtered))
 
     def _map_to_tvshow_items(self, tvshows, filtered=False):

resources/lib/vrtplayer/vrtplayer.py

@@ -109,7 +109,7 @@ def show_tvshow_menu_items(self, category=None, filtered=False):
         self._kodi.show_listing(tvshow_items, sort='label', content='tvshows')
 
     def show_category_menu_items(self):
-        category_items = self.__get_category_menu_items()
+        category_items = self.get_category_menu_items()
         self._kodi.show_listing(category_items, sort='label', content='files')
 
     def show_channels_menu_items(self, channel=None):
@@ -250,13 +250,31 @@ def search(self, search_string=None, page=None):
         self._kodi.container_update(replace=True)
         self._kodi.show_listing(search_items, sort=sort, ascending=ascending, content=content, cache=False)
 
-    def __get_category_menu_items(self):
-        try:
-            categories = self.get_categories(self._proxies)
-        except Exception:
-            categories = []
+    def get_category_menu_items(self):
+        import json
+        categories = []
+
+        # Try the cache if it is fresh
+        data = self._kodi.get_cache('categories.json', ttl=7 * 24 * 60 * 60)
+        if data:
+            categories = json.loads(data)
+
+        # Try to scrape from the web
+        if not categories:
+            try:
+                categories = self.get_categories(self._proxies)
+            except Exception:
+                categories = []
+            else:
+                self._kodi.update_cache('categories.json', json.dumps(categories))
+
+        # Use the cache anyway (better than hard-coded)
+        if not categories:
+            data = self._kodi.get_cache('categories.json', ttl=None)
+            if data:
+                categories = json.loads(data)
 
-        # Fallback to internal categories if web-scraping fails
+        # Fall back to internal hard-coded categories if all else fails
         if not categories:
             from resources.lib.vrtplayer import CATEGORIES
             categories = CATEGORIES

resources/settings.xml

@@ -23,11 +23,12 @@
         <setting label="30861" type="lsep"/> <!-- Cache -->
         <setting label="30863" help="30864" type="action" id="clear_tokens" action="RunPlugin(plugin://plugin.video.vrt.nu/?action=clearcookies)"/>
         <setting label="30865" help="30866" type="action" id="refresh_favorites" action="RunPlugin(plugin://plugin.video.vrt.nu/?action=refreshfavorites)"/>
-        <setting label="30867" type="lsep"/> <!-- InputStream Adaptive -->
-        <setting label="30869" help="30870" type="bool" id="useinputstreamadaptive" default="true" visible="System.HasAddon(inputstream.adaptive)"/>
-        <setting label="30871" help="30872" type="action" id="adaptive_settings" option="close" action="Addon.OpenSettings(inputstream.adaptive)" visible="System.HasAddon(inputstream.adaptive)" enable="eq(-1,true)" subsetting="true"/>
-        <setting label="30873" help="30874" type="action" id="widevine_install" option="close" action="RunPlugin(plugin://plugin.video.vrt.nu?action=installwidevine)" visible="System.HasAddon(inputstream.adaptive)" enable="eq(-2,true)" subsetting="true"/>
-        <setting label="30875" type="lsep"/> <!-- Logging -->
-        <setting label="30877" help="30878" type="select" id="max_log_level" values="Quiet|Info|Verbose|Debug" default="Info"/>
+        <setting label="30867" help="30868" type="bool" id="usehttpcaching" default="false"/>
+        <setting label="30869" type="lsep"/> <!-- InputStream Adaptive -->
+        <setting label="30871" help="30872" type="bool" id="useinputstreamadaptive" default="true" visible="System.HasAddon(inputstream.adaptive)"/>
+        <setting label="30873" help="30874" type="action" id="adaptive_settings" option="close" action="Addon.OpenSettings(inputstream.adaptive)" visible="System.HasAddon(inputstream.adaptive)" enable="eq(-1,true)" subsetting="true"/>
+        <setting label="30875" help="30876" type="action" id="widevine_install" option="close" action="RunPlugin(plugin://plugin.video.vrt.nu?action=installwidevine)" visible="System.HasAddon(inputstream.adaptive)" enable="eq(-2,true)" subsetting="true"/>
+        <setting label="30877" type="lsep"/> <!-- Logging -->
+        <setting label="30879" help="30880" type="select" id="max_log_level" values="Quiet|Info|Verbose|Debug" default="Info"/>
     </category>
 </settings>