Adapt url scraper to site changes

addon.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.0.9" provider-name="AddonScriptorDE">
+<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.1.1" provider-name="AddonScriptorDE">
     <requires>
         <import addon="xbmc.python" version="2.1.0"/>
     </requires>

changelog.txt

@@ -22,3 +22,7 @@
 - Fixed site changes (concerts)
 2.0.9
 - Fixed geoblocking filter
+2.1.0
+- Fixed concert livestreams
+2.1.1
+- Added support for concert collections

default.py

@@ -20,6 +20,8 @@
 forceViewMode = addon.getSetting("forceView") == "true"
 useThumbAsFanart = addon.getSetting("useThumbAsFanart") == "true"
 viewMode = str(addon.getSetting("viewIDNew"))
+addonDir = xbmc.translatePath(addon.getAddonInfo('path'))
+defaultFanart = os.path.join(addonDir ,'fanart.png')
 icon = xbmc.translatePath('special://home/addons/'+addonID+'/icon.png')
 baseUrl = "http://www.arte.tv"
 baseUrlConcert = "http://concert.arte.tv"
@@ -39,7 +41,7 @@ def index():
     for filter in regionFilters:
         regionFilter += filter.replace("'","").strip()+"%2C"
     regionFilter = regionFilter[:-3]
-    addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7/plus_recentes.json?regions="+regionFilter, "listVideosNew", "")
+    addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7.json", "listVideosNew", "")
     addDir(translation(30002), baseUrl+"/guide/"+language+"/plus7/selection.json?regions="+regionFilter, "listVideosNew", "")
     addDir(translation(30003), baseUrl+"/guide/"+language+"/plus7/plus_vues.json?regions="+regionFilter, "listVideosNew", "")
     addDir(translation(30004), baseUrl+"/guide/"+language+"/plus7/derniere_chance.json?regions="+regionFilter, "listVideosNew", "")
@@ -106,24 +108,27 @@ def listSearchVideos(urlMain):
     xbmcplugin.setContent(pluginhandle, "episodes")
     try:
         content = getUrl(urlMain)
-        spl = content.split('<div class="video-block has-play"')
+        spl = content.split('has-play\'')
         for i in range(1, len(spl), 1):
             entry = spl[i]
-            match = re.compile('alt="(.+?)"', re.DOTALL).findall(entry)
+            match = re.compile('<h4>.+?<a href=\'.+?\'>\n(.+?) \| ARTE', re.DOTALL).findall(entry)
             title = cleanTitle(match[0])
+            # The search keyword is wrapped in <b> </b> in the tile, remove these
+            title = re.compile('<b>').sub('',title)
+            title = re.compile('</b>').sub('',title)
             match = re.compile('data-description="(.+?)"', re.DOTALL).findall(entry)
             desc = ""
             if match:
                 desc = cleanTitle(match[0])
-            match = re.compile('<p class="time-row">.+?<span class=".+?">.+?</span>(.+?)</p>.+?<p class=".+?">.+?<span class=".+?">.+?</span>(.+?)</p>', re.DOTALL).findall(entry)
+            match = re.compile('<p class=\'time-row\'>.+?<span class=\'.+?\'>.+?</span>(.+?) \((.+?)\).+?</p>', re.DOTALL).findall(entry)
             if match:
                 date = match[0][0].strip()
                 desc = date+"\n"+desc
                 duration = match[0][1].strip()
                 duration = duration.split(" ")[0]
-            match = re.compile('href="(.+?)"', re.DOTALL).findall(entry)
+            match = re.compile('href=\'(.+?)\'', re.DOTALL).findall(entry)
             url = match[0]
-            match = re.compile('src="(.+?)"', re.DOTALL).findall(entry)
+            match = re.compile('src=\'(.+?)\'', re.DOTALL).findall(entry)
             thumb = match[0]
             addLink(title, url, 'playVideoNew', thumb, desc, duration)
         currentPage = urlMain[urlMain.find("page=")+5:]
@@ -138,9 +143,9 @@ def listSearchVideos(urlMain):
 
 def listCats(type, regionFilter):
     content = getUrl(baseUrl+"/guide/"+language+"/plus7")
-    content = content[content.find('<ul class="span12" data-filter="'+type+'">'):]
-    content = content[:content.find('</ul>')]
-    match = re.compile('<a href="(.+?)" data-controller="catchup" data-action="refresh" >(.+?)</a>', re.DOTALL).findall(content)
+    content = content[content.find('<div class=\'choice-'+type+' choices row\' data-filter=\''+type+'\'>'):]
+    content = content[:content.find('</div>\n</div>\n</div>')]
+    match = re.compile('<a class="ellipsis" data-action="refresh" data-controller="catchup" href="(.+?)">(.+?)</a>', re.DOTALL).findall(content)
     for url, title in match:
         title = cleanTitle(title)
         url = baseUrl+url.replace("?", ".json?").replace("&amp;", "&")+"&regions="+regionFilter
@@ -164,7 +169,7 @@ def search():
 
 def listConcertsMain():
     addDir(translation(30002), "", "listConcerts", "")
-    addDir(translation(30003), baseUrlConcert+"/"+language+"/videos/all?sort=mostviewed", "listConcerts", "")
+    addDir("Collections", "", "listCollections", "")
     addDir(translation(30011), baseUrlConcert+"/"+language+"/videos/all", "listConcerts", "")
     addDir(translation(30013), baseUrlConcert+"/"+language+"/videos/rockpop", "listConcerts", "")
     if language=="de":
@@ -199,7 +204,10 @@ def listConcerts(url=""):
         url = baseUrlConcert+match[0]
         match = re.compile('src="(.+?)"', re.DOTALL).findall(entry)
         thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
-        addLink(title, url, 'playVideoNew', thumb, "")
+        if "node-eventp" in entry:
+            addDir(title, url, "listConcerts", thumb)
+        elif "node-videop" in entry:
+            addLink(title, url, 'playVideoNew', thumb, "")
     match = re.compile('<li class="pager-next">.+?href="(.+?)"', re.DOTALL).findall(content)
     if match:
         addDir(translation(30010), baseUrlConcert+match[0], "listConcerts", "")
@@ -208,23 +216,49 @@ def listConcerts(url=""):
         xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')
 
 
+def listCollections():
+    content = getUrl("http://concert.arte.tv/"+language+"/collections.xml")
+    spl = content.split('<item>')
+    for i in range(1, len(spl), 1):
+        entry = spl[i]
+        match = re.compile('<title>(.+?)</title>', re.DOTALL).findall(entry)
+        title = cleanTitle(match[0])
+        match = re.compile('field-name-eventp-videos-count.*?&gt;(.+?)&lt;', re.DOTALL).findall(entry)
+        if match:
+            count = match[0].strip()
+            if language=="de":
+                count = count.replace("vidéos","Videos")
+            title += " ("+count+")"
+        match = re.compile('<link>(.+?)</link>', re.DOTALL).findall(entry)
+        url = match[0]
+        match = re.compile('data-src=&quot;(.+?)&quot;', re.DOTALL).findall(entry)
+        thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
+        addDir(title, url, "listConcerts", thumb)
+    xbmcplugin.endOfDirectory(pluginhandle)
+    if forceViewMode:
+        xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')
+
+
 def playVideoNew(url):
     listitem = xbmcgui.ListItem(path=getStreamUrlNew(url))
     xbmcplugin.setResolvedUrl(pluginhandle, True, listitem)
 
 
 def getStreamUrlNew(url):
     content = getUrl(url)
-    match = re.compile('arte_vp_url="(.+?)"', re.DOTALL).findall(content)
+    match = re.compile('arte_vp_url=[\'"](.+?)[\'"]', re.DOTALL).findall(content)
     if "concert.arte.tv" in url:
         url = match[0]
         content = getUrl(url)
         match1 = re.compile('"HTTP_SQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
         match2 = re.compile('"HTTP_EQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
+        match3 = re.compile('"RMTP_HQ":.*?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content)
         if match1 and maxVideoQuality == "720p":
             return match1[0].replace("\\","")
         elif match2:
             return match2[0].replace("\\","")
+        elif match3:
+            return match3[0][0].replace("\\","")+match3[0][1].replace("\\","")+" swfUrl=http://www.arte.tv/flash/mediaplayer/mediaplayer.swf live=1 swfVfy=1"
     elif streamingType=="HTTP":
         url = match[0].replace("/player/","/")
         content = getUrl(url)
@@ -306,6 +340,8 @@ def addLink(name, url, mode, iconimage, desc="", duration=""):
     liz.setProperty('IsPlayable', 'true')
     if useThumbAsFanart and iconimage!=icon:
         liz.setProperty("fanart_image", iconimage)
+    else:
+        liz.setProperty("fanart_image", defaultFanart)
     liz.addContextMenuItems([(translation(30020), 'RunPlugin(plugin://'+addonID+'/?mode=queueVideo&url='+urllib.quote_plus(u)+'&name='+urllib.quote_plus(name)+')',)])
     ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz)
     return ok
@@ -316,6 +352,10 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
     ok = True
     liz = xbmcgui.ListItem(name, iconImage=icon, thumbnailImage=iconimage)
     liz.setInfo(type="Video", infoLabels={"Title": name})
+    if useThumbAsFanart and iconimage and iconimage!=icon:
+        liz.setProperty("fanart_image", iconimage)
+    else:
+        liz.setProperty("fanart_image", defaultFanart)
     ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, isFolder=True)
     return ok
 
@@ -341,6 +381,8 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
     playLiveStream()
 elif mode == 'listConcerts':
     listConcerts(url)
+elif mode == 'listCollections':
+    listCollections()
 elif mode == 'listConcertsMain':
     listConcertsMain()
 elif mode == 'search':