Make code more robust (#117)

Make every dictionary lookup use get() which doesn't raise an exception
when missing. (If JSON returns something unexpected, or web scraping
fails unexpectedly)

We still have to look for each dictionary lookup what we should return
if the lookup fails. By default it returns None, which sometimes is
fine, but in some cases we better return an empty list or an empty dict.

Some parts now will undoubtedly fail because of None values where it
would have failed Kodi before.

In some cases a missing title would be better replaced with e.g. 'N/A'
or 'video #X' so the listing would still work. Either a complete review
is necessary, or future brokeness will guide us to make better decisions
;-)

plugin.video.vrt.nu/resources/lib/kodiwrappers/kodiwrapper.py

@@ -107,7 +107,7 @@ def open_settings(self):
 
     def get_global_setting(self, setting):
         json_result = xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "Settings.GetSettingValue", "params": {"setting": "%s"}, "id": 1}' % setting)
-        return json.loads(json_result)['result']['value']
+        return json.loads(json_result).get('result', dict()).get('value')
 
     def get_proxies(self):
         usehttpproxy = self.get_global_setting('network.usehttpproxy')

plugin.video.vrt.nu/resources/lib/vrtplayer/streamservice.py

@@ -27,7 +27,7 @@ def __init__(self, vrt_base, vrtnu_base_url, kodi_wrapper, token_resolver):
         self._license_url = self._get_license_url()
 
     def _get_license_url(self):
-        return requests.get(self._VUPLAY_API_URL, proxies=self._proxies).json()['drm_providers']['widevine']['la_url']
+        return requests.get(self._VUPLAY_API_URL, proxies=self._proxies).json().get('drm_providers', dict()).get('widevine', dict()).get('la_url')
 
     def _create_settings_dir(self):
         settingsdir = self._kodi_wrapper.get_userdata_path()
@@ -83,17 +83,17 @@ def _get_api_data(self, video_url):
         xvrttoken = None
 
         # Store required data attributes
-        client = video_data['data-client']
-        media_api_url = video_data['data-mediaapiurl']
-        if 'data-videoid' in list(video_data.keys()):
-            video_id = video_data['data-videoid']
+        client = video_data.get('data-client')
+        media_api_url = video_data.get('data-mediaapiurl')
+        video_id = video_data.get('data-videoid')
+        if video_id is not None:
             xvrttoken = self.token_resolver.get_xvrttoken()
         else:
-            video_id = video_data['data-livestream']
+            video_id = video_data.get('data-livestream')
             is_live_stream = True
-        publication_id = ''
-        if 'data-publicationid' in list(video_data.keys()):
-            publication_id = video_data['data-publicationid'] + requests.utils.quote('$')
+        publication_id = video_data.get('data-publicationid', '')
+        if publication_id:
+            publication_id += requests.utils.quote('$')
         return apidata.ApiData(client, media_api_url, video_id, publication_id, xvrttoken, is_live_stream)
 
     def _get_video_json(self, api_data):
@@ -111,28 +111,28 @@ def _get_video_json(self, api_data):
         return video_json
 
     def _handle_error(self, video_json):
-        self._kodi_wrapper.log_error(video_json['message'])
+        self._kodi_wrapper.log_error(video_json.get('message'))
         message = self._kodi_wrapper.get_localized_string(32054)
         self._kodi_wrapper.show_ok_dialog('', message)
 
     def get_stream(self, video, retry=False, api_data=None):
-        self._kodi_wrapper.log_notice('video_url ' + video['video_url'])
-        if video['video_id'] is not None and video['publication_id'] is not None and retry is False:
+        self._kodi_wrapper.log_notice('video_url ' + video.get('video_url'))
+        if video.get('video_id') is not None and video.get('publication_id') is not None and retry is False:
             xvrttoken = self.token_resolver.get_xvrttoken()
-            api_data = apidata.ApiData(self._CLIENT, self._VUALTO_API_URL, video['video_id'], video['publication_id'] + requests.utils.quote('$'), xvrttoken, False)
+            api_data = apidata.ApiData(self._CLIENT, self._VUALTO_API_URL, video.get('video_id'), video.get('publication_id') + requests.utils.quote('$'), xvrttoken, False)
         else:
-            api_data = api_data or self._get_api_data(video['video_url'])
+            api_data = api_data or self._get_api_data(video.get('video_url'))
         vudrm_token = None
         video_json = self._get_video_json(api_data)
 
         if 'drm' in video_json:
-            vudrm_token = video_json['drm']
-            target_urls = video_json['targetUrls']
-            stream_dict = dict(list([(x['type'], x['url']) for x in target_urls]))
+            vudrm_token = video_json.get('drm')
+            target_urls = video_json.get('targetUrls')
+            stream_dict = dict(list([(x.get('type'), x.get('url')) for x in target_urls]))
             return self._select_stream(stream_dict, vudrm_token)
 
-        if video_json['code'] == 'INVALID_LOCATION' or video_json['code'] == 'INCOMPLETE_ROAMING_CONFIG':
-            self._kodi_wrapper.log_notice(video_json['message'])
+        if video_json.get('code') in ('INCOMPLETE_ROAMING_CONFIG', 'INVALID_LOCATION'):
+            self._kodi_wrapper.log_notice(video_json.get('message'))
             roaming_xvrttoken = self.token_resolver.get_xvrttoken(True)
             if not retry and roaming_xvrttoken is not None:
                 # Delete cached playertokens

plugin.video.vrt.nu/resources/lib/vrtplayer/tokenresolver.py

@@ -63,15 +63,15 @@ def get_xvrttoken_from_cookiejar(cookiejar):
     def _get_new_playertoken(self, path, token_url, headers):
         playertoken = requests.post(token_url, proxies=self._proxies, headers=headers).json()
         json.dump(playertoken, open(path, 'w'))
-        return playertoken['vrtPlayerToken']
+        return playertoken.get('vrtPlayerToken')
 
     def _get_cached_token(self, path, token_name):
         cached_token = None
 
         if self._kodi_wrapper.check_if_path_exists(path):
             token = json.loads(open(path, 'r').read())
             now = datetime.datetime.utcnow()
-            exp = datetime.datetime(*(time.strptime(token['expirationDate'], '%Y-%m-%dT%H:%M:%S.%fZ')[0:6]))
+            exp = datetime.datetime(*(time.strptime(token.get('expirationDate'), '%Y-%m-%dT%H:%M:%S.%fZ')[0:6]))
             if exp > now:
                 self._kodi_wrapper.log_notice('Got cached token')
                 cached_token = token[token_name]
@@ -88,18 +88,18 @@ def _get_new_xvrttoken(self, path, get_roaming_token):
         data = {'loginID': cred.username, 'password': cred.password, 'sessionExpiration': '-1', 'APIKey': self._API_KEY, 'targetEnv': 'jssdk'}
         logon_json = requests.post(self._LOGIN_URL, data, proxies=self._proxies).json()
         token = None
-        if logon_json['errorCode'] == 0:
-            login_token = logon_json['sessionInfo']['login_token']
+        if logon_json.get('errorCode') == 0:
+            login_token = logon_json.get('sessionInfo', dict()).get('login_token')
             login_cookie = ''.join(('glt_', self._API_KEY, '=', login_token))
-            payload = {'uid': logon_json['UID'], 'uidsig': logon_json['UIDSignature'], 'ts': logon_json['signatureTimestamp'], 'email': cred.username}
+            payload = {'uid': logon_json.get('UID'), 'uidsig': logon_json.get('UIDSignature'), 'ts': logon_json.get('signatureTimestamp'), 'email': cred.username}
             headers = {'Content-Type': 'application/json', 'Cookie': login_cookie}
             cookie_jar = requests.post(self._TOKEN_GATEWAY_URL, proxies=self._proxies, headers=headers, json=payload).cookies
 
             xvrttoken = TokenResolver._create_token_dictionary(cookie_jar)
-            token = xvrttoken['X-VRT-Token']
+            token = xvrttoken.get('X-VRT-Token')
             if get_roaming_token:
                 xvrttoken = self._get_roaming_xvrttoken(login_cookie, xvrttoken)
-                token = xvrttoken['X-VRT-Token'] if xvrttoken is not None else None
+                token = xvrttoken.get('X-VRT-Token')
             json.dump(xvrttoken, open(path, 'w'))
         else:
             title = self._kodi_wrapper.get_localized_string(32051)
@@ -109,14 +109,14 @@ def _get_new_xvrttoken(self, path, get_roaming_token):
 
     def _get_roaming_xvrttoken(self, login_cookie, xvrttoken):
         url = 'https://token.vrt.be/vrtnuinitloginEU?destination=https://www.vrt.be/vrtnu/'
-        cookie_value = 'X-VRT-Token=' + xvrttoken['X-VRT-Token']
+        cookie_value = 'X-VRT-Token=' + xvrttoken.get('X-VRT-Token')
         headers = {'Cookie': cookie_value}
         r = requests.get(url, proxies=self._proxies, headers=headers, allow_redirects=False)
         url = r.headers.get('Location')
         r = requests.get(url, proxies=self._proxies, headers=headers, allow_redirects=False)
         url = r.headers.get('Location')
         headers = {'Cookie': login_cookie}
-        roaming_xvrttoken = None
+        roaming_xvrttoken = dict()
         if url is not None:
             cookie_jar = requests.get(url, proxies=self._proxies, headers=headers).cookies
             roaming_xvrttoken = TokenResolver._create_token_dictionary(cookie_jar)

plugin.video.vrt.nu/resources/lib/vrtplayer/vrtapihelper.py

@@ -48,7 +48,7 @@ def get_tvshow_items(self, path):
         return tvshow_items
 
     def _get_season_items(self, api_url, api_json):
-        season_items = None
+        season_items = []
         if api_json.get('results'):
             episode = api_json['results'][0]
         else:
@@ -62,31 +62,31 @@ def _get_season_items(self, api_url, api_json):
         return season_items
 
     def get_episode_items(self, path):
-        episode_items = None
-        sort_method = None
+        episode_items = []
+        sort = None
         if path == 'recent':
             api_url = ''.join((self._VRTNU_SEARCH_URL, '?i=video&size=100&facets[transcodingStatus]=AVAILABLE&facets[brands]=[een,canvas,sporza,vrtnws,vrtnxt,radio1,radio2,klara,stubru,mnm]'))
             api_json = requests.get(api_url, proxies=self._proxies).json()
-            episode_items, sort_method = self._map_to_episode_items(api_json.get('results', []), path)
+            episode_items, sort = self._map_to_episode_items(api_json.get('results', []), path)
         else:
             api_url = ''.join((self._VRTNU_SEARCH_URL, '?i=video&size=150&facets[programUrl]=//www.vrt.be', path.replace('.relevant', ''))) if '.relevant/' in path else path
             api_json = requests.get(api_url, proxies=self._proxies).json()
             # Look for seasons items if not yet done
             if 'facets[seasonTitle]' not in path:
                 episode_items = self._get_season_items(api_url, api_json)
             # No season items, generate episode items
-            if episode_items is None:
-                episode_items, sort_method = self._map_to_episode_items(api_json.get('results', []))
+            if not episode_items:
+                episode_items, sort = self._map_to_episode_items(api_json.get('results', []))
 
-        return episode_items, sort_method
+        return episode_items, sort
 
     def _map_to_episode_items(self, episodes, titletype=None):
         episode_items = []
         sort = None
         for episode in episodes:
             display_options = episode.get('displayOptions', dict())
 
-            if episode['programType'] == 'reeksoplopend' and titletype is None:
+            if episode.get('programType') == 'reeksoplopend' and titletype is None:
                 titletype = 'reeksoplopend'
 
             metadata_creator = metadatacreator.MetadataCreator()
@@ -109,9 +109,9 @@ def _map_to_episode_items(self, episodes, titletype=None):
             metadata_creator.season = episode.get('seasonName')
             metadata_creator.episode = episode.get('episodeNumber')
             metadata_creator.mediatype = episode.get('type', 'episode')
-            if episode.get('assetOnTime') is not None:
+            if episode.get('assetOnTime'):
                 metadata_creator.ontime = datetime(*time.strptime(episode.get('assetOnTime'), '%Y-%m-%dT%H:%M:%S+0000')[0:6])
-            if episode.get('assetOffTime') is not None:
+            if episode.get('assetOffTime'):
                 metadata_creator.offtime = datetime(*time.strptime(episode.get('assetOffTime'), '%Y-%m-%dT%H:%M:%S+0000')[0:6])
 
             # Add additional metadata to plot
@@ -190,11 +190,11 @@ def __get_crc32(string):
     def _make_title(self, result, titletype):
         sort = None
         if titletype == 'recent':
-            title = ''.join((result['program'], ' - ', BeautifulSoup(result['shortDescription'], 'html.parser').text))
-        elif titletype == 'reeksoplopend' or result['formattedBroadcastShortDate'] == '':
-            title = ''.join((self._kodi_wrapper.get_localized_string(32095), ' ', str(result['episodeNumber']), ' - ', BeautifulSoup(result['shortDescription'], 'html.parser').text if result['shortDescription'] != '' else BeautifulSoup(result['title'], 'html.parser').text))
+            title = ''.join((result.get('program'), ' - ', BeautifulSoup(result.get('shortDescription'), 'html.parser').text))
+        elif titletype == 'reeksoplopend' or result.get('formattedBroadcastShortDate') == '':
+            title = ''.join((self._kodi_wrapper.get_localized_string(32095), ' ', str(result.get('episodeNumber')), ' - ', BeautifulSoup(result.get('shortDescription'), 'html.parser').text if result.get('shortDescription') != '' else BeautifulSoup(result.get('title'), 'html.parser').text))
             sort = sortmethod.ALPHABET
         else:
-            title = ''.join((result['formattedBroadcastShortDate'], ' - ', BeautifulSoup(result['shortDescription'], 'html.parser').text if result['shortDescription'] != '' else BeautifulSoup(result['title'], 'html.parser').text))
+            title = ''.join((result.get('formattedBroadcastShortDate'), ' - ', BeautifulSoup(result.get('shortDescription'), 'html.parser').text if result.get('shortDescription') != '' else BeautifulSoup(result.get('title'), 'html.parser').text))
 
         return title, sort

plugin.video.vrt.nu/resources/lib/vrtplayer/vrtplayer.py

@@ -99,7 +99,7 @@ def __get_category_menu_items(self, url, soupstrainer_parser_selector, routing_a
         soup = BeautifulSoup(response.content, 'html.parser', parse_only=tiles)
         listing = []
         for tile in soup.find_all(class_='nui-tile'):
-            category = tile['href'].split('/')[-2]
+            category = tile.get('href').split('/')[-2]
             thumbnail, title = self.__get_category_thumbnail_and_title(tile)
             video_dict = None
             if video_dict_action is not None:
@@ -114,7 +114,7 @@ def __get_category_menu_items(self, url, soupstrainer_parser_selector, routing_a
 
     @staticmethod
     def __format_category_image_url(element):
-        raw_thumbnail = element.find(class_='media')['data-responsive-image']
+        raw_thumbnail = element.find(class_='media').get('data-responsive-image')
         return statichelper.add_https_method(raw_thumbnail)
 
     @staticmethod