AddonManager: Better support of non-github addons

src/Mod/AddonManager/addonmanager_utilities.py

@@ -25,6 +25,7 @@
 import sys
 import FreeCAD
 import shutil
+import re
 
 # check for SSL support
 
@@ -234,7 +235,7 @@ def restartFreeCAD():
         QtCore.QProcess.startDetached(QtGui.QApplication.applicationFilePath(),args)
 
 
-def getzipurl(baseurl):
+def getZipUrl(baseurl):
 
     "Returns the location of a zip file from a repo, if available"
 
@@ -246,45 +247,53 @@ def getzipurl(baseurl):
         reponame = baseurl.strip("/").split("/")[-1]
         return baseurl+"/-/archive/master/"+reponame+"-master.zip"
     else:
-        print("Debug: addonmanager_utilities.getzipurl: Unknown git host:",url)
+        print("Debug: addonmanager_utilities.getZipUrl: Unknown git host:",url)
         return None
 
 
-def getreadmeurl(baseurl):
+def getReadmeUrl(url):
 
     "Returns the location of a readme file"
 
-    url = getserver(baseurl).strip("/")
-    if url.endswith("github.com") or url.endswith("framagit.org"):
-        return baseurl+"/blob/master/README.md"
-    else:
-        print("Debug: addonmanager_utilities.getreadmeurl: Unknown git host:",url)
-        return None
+    if ("github" in url) or ("framagit" in url):
+        return url+"/blob/master/README.md"
+    print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url)
+    return None
 
 
-def getreadmeregex(baseurl):
+def getReadmeRegex(url):
 
     """Return a regex string that extracts the contents to be displayed in the description
     panel of the Addon manager, from raw HTML data (the readme's html rendering usually)"""
 
-    url = getserver(baseurl).strip("/")
-    if url.endswith("github.com"):
+    if ("github" in url):
         return "<article.*?>(.*?)</article>"
-    elif url.endswith("framagit.org"):
+    elif ("framagit" in url):
         return None # the readme content on framagit is generated by javascript so unretrievable by urlopen
-    else:
-        print("Debug: addonmanager_utilities.getreadmeregex: Unknown git host:",url)
-        return None
+    print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url)
+    return None
 
 
-def getdescregex(baseurl):
+def getDescRegex(url):
 
     """Returns a regex string that extracts a WB description to be displayed in the description
     panel of the Addon manager, if the README could not be found"""
-
-    url = getserver(baseurl).strip("/")
-    if url.endswith("github.com") or url.endswith("framagit.org"):
+
+    if ("github" in url):
         return "<meta property=\"og:description\" content=\"(.*?)\""
-    else:
-        print("Debug: addonmanager_utilities.getdescregex: Unknown git host:",url)
-        return None
+    elif ("framagit" in url):
+        return "<meta.*?content=\"(.*?)\".*?og\:description.*?>"
+    print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url)
+    return None
+
+def getRepoUrl(text):
+
+    "finds an URL in a given piece of text extracted from github's HTML"
+
+    if ("github" in text) and ("href" in text):
+        return "https://github.com/" + re.findall("href=\"\/(.*?)\/tree",text)[0]
+    elif ("MOOC" in text):
+        # Bad hack for now... We need to do better
+        return "https://framagit.org/freecad-france/mooc-workbench"
+    print("Debug: addonmanager_utilities.getRepoUrl: Unkable to find repo:",text)
+    return None

src/Mod/AddonManager/addonmanager_workers.py

@@ -84,14 +84,15 @@ def run(self):
             name = re.findall("title=\"(.*?) @",l)[0]
             self.info_label.emit(name)
             #url = re.findall("title=\"(.*?) @",l)[0]
-            url = "https://github.com/" + re.findall("href=\"\/(.*?)\/tree",l)[0]
-            addondir = moddir + os.sep + name
-            #print ("found:",name," at ",url)
-            if not os.path.exists(addondir):
-                state = 0
-            else:
-                state = 1
-            repos.append([name,url,state])
+            url = utils.getRepoUrl(l)
+            if url:
+                addondir = moddir + os.sep + name
+                #print ("found:",name," at ",url)
+                if not os.path.exists(addondir):
+                    state = 0
+                else:
+                    state = 1
+                repos.append([name,url,state])
         # querying custom addons
         customaddons = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons").GetString("CustomRepositories","").split("\n")
         for url in customaddons:
@@ -317,7 +318,7 @@ def run(self):
             self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + ' ' + str(url))
             desc = ""
             # get the README if possible
-            readmeurl = utils.getreadmeurl(url)
+            readmeurl = utils.getReadmeUrl(url)
             if not readmeurl:
                 print("Debug: README not found for",url)
             u = utils.urlopen(readmeurl)
@@ -328,12 +329,12 @@ def run(self):
                 if sys.version_info.major >= 3 and isinstance(p, bytes):
                     p = p.decode("utf-8")
                 u.close()
-                readmeregex = utils.getreadmeregex(url)
+                readmeregex = utils.getReadmeRegex(url)
                 if readmeregex:
                     readme = re.findall(readmeregex,p,flags=re.MULTILINE|re.DOTALL)
                     if readme:
                         desc += readme[0]
-            else:
+            if not desc:
                 # fall back to the description text
                 u = utils.urlopen(url)
                 if not u:
@@ -344,11 +345,11 @@ def run(self):
                 if sys.version_info.major >= 3 and isinstance(p, bytes):
                     p = p.decode("utf-8")
                 u.close()
-                descregex = utils.getdescregex(url)
+                descregex = utils.getDescRegex(url)
                 if descregex:
                     desc = re.findall(descregex,p)
                     if desc:
-                        desc = desc[0]
+                        desc = "<br/>"+desc[0]
             if not desc:
                 desc = "Unable to retrieve addon description"
             self.repos[self.idx].append(desc)
@@ -676,7 +677,7 @@ def download(self,baseurl,clonedir):
                 shutil.rmtree(bakdir)
             os.rename(clonedir,bakdir)
         os.makedirs(clonedir)
-        zipurl = utils.getzipurl(baseurl)
+        zipurl = utils.getZipUrl(baseurl)
         if not zipurl:
             return translate("AddonsInstaller", "Error: Unable to locate zip from") + " " + baseurl
         try: