Revised plugin: fixes/changes for Topics, Search, Speakers; adds TED Series and Other feeds

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 /dist.zip
 /plugin.video.ted.talks/
+#/plugin.video.ted.talks-*.zip

CHANGELOG.md

@@ -1,3 +1,19 @@
+## [5.1.0]
+### Changed
+- Reworked [5.0.1], with many fixes to keep up with ted.com changes
+- Fixed "Search", plus now searches for talks or playlists
+- Added "Other Feeds" to include other RSS feeds for TEDtalks
+- Added "TED Series" section to access TED series, seasons, and episodes
+- Reworked "Speakers" to provide more detail about a speaker 
+- Reworked "Topics" to show featured talks and playlists of a topic page
+- Added a context menu to use a topic as the search string, since there
+  seems no longer a good way to access the site's topic search
+- Honors the host's rate limiting responses when fetching urls
+- Reworked playback, improves subtitles selection
+- Playback uses the youtube.dl addon (YDStreamExtractor) when stream data
+  are missing, trying to extract stream urls from the "external" keys info
+- Requires Kodi v19.x (Matrix) or Kodi v20.x (Nexus)
+
 ## [5.0.1]
 ### Changed
 - Scraper fixes (pull#81 thanks to kevwag and kodaksmith)

Makefile

@@ -0,0 +1,56 @@
+## TEDTalks video addon for Kodi
+# simple Makefile to package the plugin based on addon.xml
+
+DISTRIBUTION = matrix
+
+ADDON_XML = ./addon.xml
+
+# explicitly identify the source files
+define SOURCE_FILES
+${ADDON_XML}
+./LICENSE.txt
+./default.py
+./resources/settings.xml
+./resources/icon.png
+./resources/speaker.png
+./resources/rss_feeds.opml
+./resources/language/resource.language.en_gb/strings.po
+./resources/language/resource.language.fi_fi/strings.po
+./resources/language/resource.language.hu_hu/strings.po
+./resources/language/resource.language.pt_pt/strings.po
+./resources/language/resource.language.sv_se/strings.po
+./resources/lib/settings.py
+./resources/lib/ted_talks.py
+./resources/lib/model/arguments.py
+./resources/lib/model/fetcher.py
+./resources/lib/model/rss_scraper.py
+./resources/lib/model/search_scraper.py
+./resources/lib/model/series_scraper.py
+./resources/lib/model/talk_scraper.py
+./resources/lib/model/topics_scraper.py
+./resources/lib/model/speakers_scraper.py
+endef
+
+# newline character
+define NL
+
+
+endef
+
+_AO = grep -o '<addon[^>]*>' ${ADDON_XML}
+ADDONID = $(subst id=",,$(shell ${_AO} | grep -o 'id="[^"]*'))
+VERSION = $(subst on=",,$(shell ${_AO} | grep -o 'on="[^"]*'))
+SOURCES = $(sort $(SOURCE_FILES)) # hack to remove newlines
+PACKAGE = ${ADDONID}-${VERSION}-${DISTRIBUTION}.zip
+
+.PHONY: clean
+
+${PACKAGE} : ${ADDONID} 
+	rm -f $@  # force new zip
+	zip -r $@ $<
+
+${ADDONID} : $(SOURCES)
+	$(foreach f,$^,install -p -D ${f} $@/${f}${NL})
+
+clean :
+	rm -fR ${ADDONID}

addon.xml

@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<addon id="plugin.video.ted.talks" name="TED Talks" version="5.0.1" provider-name="moreginger, rwparris2, kevwag, kodaksmith">
+<addon id="plugin.video.ted.talks" name="TED Talks" version="5.1.0" provider-name="rwparris2, moreginger, kevwag, kodaksmith, demitrios">
   <requires>
     <import addon="xbmc.python" version="3.0.0"/>
     <import addon="script.module.requests" version="2.22.0"/>
     <import addon="script.module.html5lib" version="1.0.1+matrix.2"/>
-    <import addon="script.module.m3u8" version="0.5.4+matrix.2"/>
+    <import addon="script.module.youtube.dl" version="21.303.0+matrix.1"/>
   </requires>
   <extension point="xbmc.python.pluginsource" library="default.py">
     <provides>video</provides>
@@ -23,13 +23,15 @@
     <platform>all</platform>
     <license>GPL-2.0-or-later</license>
     <news>
-v5.0.1
-- Scraper fixes (kevwag, kodaksmith)
-- Fix issue determining active language (issue#79)
-
-v.5.0.0
-- Requires Kodi 19+ (Matrix, Nexus, ...)
-- Playback uses m3u8, which respects the global bandwidth setting
+v5.1.0
+- More or less a rewrite of v5.0.1, many fixes
+- Fixed "Search", plus now searches for talks or playlists
+- Added "Other Feeds" to include other RSS feeds for TEDtalks
+- Reworked "Speakers" to provide more detail about a speaker 
+- Reworked "Topics" to show featured talks and playlists of a topic page
+- Reworked playback, improves subtitles selection
+- Playback uses youtube.dl for "external" (missing stream) TED video
+- Requires Kodi v19.x (Matrix) or Kodi v20.x (Nexus)
     </news>
     <forum>https://forum.kodi.tv/showthread.php?tid=36866</forum>
     <website>https://github.com/moreginger/xbmc-plugin.video.ted.talks</website>

build.sh

@@ -8,7 +8,8 @@ mkdir $BUILD
 # find -type f | grep -v "$BUILD\|$DIST\|build\.sh\|_test\.py\|test_utils\.py\|run_tests\.sh\|\.gitignore\|\.pyc\|\.travis.yml\|\.md\|README\|\.git\|\.pylintrc\|\testSupport\|requirements\.txt" | while read f; do cp "$f" "$BUILD/$f"; done
 rsync -aP\
   --exclude "$DIST"\
-  --exclude "$BUILD"\
+  --exclude "${BUILD}*"\
+  --exclude 'Makefile'\
   --exclude '.git'\
   --exclude '.gitignore'\
   --exclude '.pylintrc'\

default.py

@@ -1,13 +1,10 @@
-# Init block
-import resources.lib.plugin as plugin
-import resources.lib.settings as settings
-plugin.init()
-settings.init()
 
 import sys
-
 import resources.lib.model.arguments as arguments
 import resources.lib.ted_talks as ted_talks
+import resources.lib.settings as settings
+
+settings.initialize()
 
 if __name__ == "__main__":
     args_map = arguments.parse_arguments(sys.argv[2])

requirements.txt

@@ -1,4 +1,4 @@
 html5lib
 mock
-m3u8
 requests
+YDStreamExtractor

resources/language/resource.language.en_gb/strings.po

@@ -40,10 +40,38 @@ msgctxt "#30034"
 msgid "Search for Talks on TED.com"
 msgstr ""
 
+msgctxt "#30005"
+msgid "Search (Playlists)"
+msgstr ""
+
+msgctxt "#30035"
+msgid "Search for Playlists on TED.com"
+msgstr ""
+
 msgctxt "#30007"
 msgid "Topics"
 msgstr ""
 
+msgctxt "#30037"
+msgid "Browse featured Talks and Playlists for TED topics from A to Z"
+msgstr ""
+
+msgctxt "#30008"
+msgid "TED Series"
+msgstr ""
+
+msgctxt "#30038"
+msgid "Feed your curiosity with new TED series that go deeper into fascinating topics -- from the everyday objects that changed the world, to cutting-edge neuroscience on a shoestring budget."
+msgstr ""
+
+msgctxt "#30009"
+msgid "Other Feeds"
+msgstr ""
+
+msgctxt "#30039"
+msgid "More feeds that feature TEDTalks video."
+msgstr ""
+
 msgctxt "#30020"
 msgid "Next"
 msgstr ""
@@ -56,6 +84,26 @@ msgctxt "#30022"
 msgid "More..."
 msgstr ""
 
+msgctxt "#30023"
+msgid "Cancel"
+msgstr ""
+
+msgctxt "#30024"
+msgid "NEW Search"
+msgstr ""
+
+msgctxt "#30025"
+msgid "Search string: %s"
+msgstr ""
+
+msgctxt "#30091"
+msgid "Talk detail"
+msgstr ""
+
+msgctxt "#30094"
+msgid "Search with Topic"
+msgstr ""
+
 msgctxt "#30097"
 msgid "Queue"
 msgstr ""

resources/lib/model/arguments.py

@@ -6,6 +6,6 @@ def parse_arguments(custom_args):
         args = custom_args[1:].split('&')
         for arg in args:
             if len(arg) > 0:
-                split = arg.split('=')
+                split = arg.split('=') + [''] # hack to ensure a value
                 args_map[split[0]] = urllib.parse.unquote_plus(split[1])
-    return args_map
+    return args_map

resources/lib/model/fetcher.py

@@ -1,20 +1,27 @@
 import requests
+import time
 
 class Fetcher:
 
     def __init__(self, logger):
         self.logger = logger
 
     def get(self, url):
-        self.logger('GET {}'.format(url))
-
-        r = requests.get(url)
-        if r.ok:
-            return r
-        else:
-            self.logger('%s\n%s\n%s'.format(r.status_code, r.headers, r.text))
-            raise Exception('Failed to GET {}: {}'.format(url, r.status_code))
-
+        #self.logger('GET {}'.format(url))
+        r = requests.get(url, timeout=(6,12))
+        if not r.ok:
+            wait = int(r.headers.get('Retry-After', 0))
+            if r.status_code == 429 and wait > 0 and wait < 10:
+                # too many requests, respect RetryAfter and try again
+                time.sleep(wait+1)
+                r = requests.get(url, timeout=(6,12))
+                if not r.ok:
+                    hdrs = '\n'.join([k +'='+ v for k, v in r.headers.items()])
+                    self.logger('{0}\n{1}\n{2}'.format(
+                        r.status_code, hdrs, r.text), level='warn')
+                    raise Exception('Failed to GET {}: {}'.format(
+                        url, r.status_code))
+        return r
 
     def get_HTML(self, url):
         return self.get(url).text

resources/lib/model/rss_scraper.py

@@ -4,7 +4,6 @@
 so keep it for now.
 """
 
-from datetime import timedelta
 import time
 import urllib.request, urllib.error, urllib.parse
 
@@ -37,43 +36,47 @@ def get_talk_details(self, item):
         """
         Return the details from an RSS <item> tag soup.
         """
-        title = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}subtitle').text  # <title> tag has unecessary padding strings
-        author = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}author').text
-        pic = item.find('./{http://search.yahoo.com/mrss/}thumbnail').get('url')
+        # TODO: Upgrade link to user's bitrate, looks like can synthesize
+        # URL of form https://download.ted.com/talks/{lastPathSegment.mp4}
 
-        self.logger('%s = %s' % ('pic', str(pic)), level='debug')
-
-        duration = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}duration').text
-        duration = time.strptime(duration, '%H:%M:%S')
-        duration_seconds = self.__total_seconds__(timedelta(hours=duration.tm_hour, minutes=duration.tm_min, seconds=duration.tm_sec))
-        plot = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}summary').text
-        link = item.find('./link').text
-
-        # TODO: Upgrade link to user's bitrate, looks like can synthesize URL of form https://download.ted.com/talks/{lastPathSegment.mp4}
-
-        # Get date as XBMC wants it
+        # get date as XBMC wants it
         pub_date = item.find('./pubDate').text[:-6]  # strptime can't handle timezone info.
         try:
             date = time.strptime(pub_date, "%a, %d %b %Y %H:%M:%S")
         except ValueError as e:
             self.logger("Could not parse date '%s': %s" % (pub_date, e))
             date = time.localtime()
-        date = time.strftime("%d.%m.%Y", date)
 
-        return {'title':title, 'author':author, 'thumb':pic, 'plot':plot, 'duration':duration_seconds, 'date':date, 'link':link, 'mediatype': "video"}
-
-    def __total_seconds__(self, delta):
-        return delta.total_seconds()
-
-    def get_new_talks(self):
+        # convert H:M:S duration to seconds
+        duration = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}duration').text
+        duration = [0, 0] + duration.split(':')
+        duration = (int(duration[-3])*60 + int(duration[-2])*60) + int(duration[-1])
+
+        return {
+            'title': item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}subtitle').text.strip(),
+            'cast': [item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}author').text],
+            'thumb': item.find('./{http://search.yahoo.com/mrss/}thumbnail').get('url'),
+            'plot': item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}summary').text,
+            'duration': duration,
+            'date': time.strftime("%d.%m.%Y", date),
+            #'aired': time.strftime("%Y-%m-%d", date),
+            'dateadded': time.strftime("%Y-%m-%d %H:%M:%S", date),
+            'url': item.find('./link').text,
+            'media': item.find('./{http://search.yahoo.com/mrss/}content').get('url'),
+            'mediatype': 'video'
+        }
+
+    def get_new_talks(self, feed=None):
+        if not feed:
+            feed = 'http://feeds.feedburner.com/tedtalks_video'
         """
-        Returns talks as dicts {title:, author:, thumb:, date:, duration:, link:}.
+        Returns talks as dicts:
+        {title:, author:, thumb:, date:, duration:, link: ...}.
         """
         talks_by_title = {}
-        rss = get_document('http://feeds.feedburner.com/tedtalks_video')
+        rss = get_document(feed)
         for item in fromstring(rss).findall('channel/item'):
             talk = self.get_talk_details(item)
             talks_by_title[talk['title']] = talk
 
         return iter(talks_by_title.values())
-