[processing] first implementation of github-based resources manager

Author: volaya

python/plugins/processing/gui/GetScriptsAndModels.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+"""
+***************************************************************************
+    GetScriptsAndModels.py
+    ---------------------
+    Date                 : June 2014
+    Copyright            : (C) 2014 by Victor Olaya
+    Email                : volayaf at gmail dot com
+***************************************************************************
+*                                                                         *
+*   This program is free software; you can redistribute it and/or modify  *
+*   it under the terms of the GNU General Public License as published by  *
+*   the Free Software Foundation; either version 2 of the License, or     *
+*   (at your option) any later version.                                   *
+*                                                                         *
+***************************************************************************
+"""
+import os
+from PyQt4 import QtGui
+from processing.ui.ui_DlgGetScriptsAndModels import Ui_DlgGetScriptsAndModels
+from processing.script.ScriptUtils import ScriptUtils
+from processing.modeler.ModelerUtils import ModelerUtils
+import json
+from processing.gui import Help2Html
+from qgis.utils import iface
+from processing.gui.Help2Html import getDescription, ALG_DESC, ALG_VERSION,\
+    ALG_CREATOR
+
+__author__ = 'Victor Olaya'
+__date__ = 'June 2014'
+__copyright__ = '(C) 201, Victor Olaya'
+
+# This will get replaced with a git SHA1 when you do a git archive
+
+__revision__ = '$Format:%H$'
+
+import urllib2
+from urllib2 import HTTPError
+from PyQt4.QtCore import *
+from PyQt4.QtGui import *
+from processing.gui.ToolboxAction import ToolboxAction
+
+
+class GetScriptsAction(ToolboxAction):
+
+    def __init__(self):
+        self.name = "Get scripts from on-line scripts collection"
+        self.group = 'Tools'
+
+    def getIcon(self):
+        return QIcon(':/processing/images/script.png')
+
+    def execute(self):
+        try:
+            dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.SCRIPTS)
+            dlg.exec_()
+            if dlg.updateToolbox:                        
+                self.toolbox.updateProvider('script')
+                
+        except HTTPError:
+            QMessageBox.critical(iface.mainWindow(), "Connection problem", "Could not connect to scripts/models repository")
+        
+class GetModelsAction(ToolboxAction):
+
+    def __init__(self):
+        self.name = "Get models from on-line scripts collection"
+        self.group = 'Tools'
+
+    def getIcon(self):
+        return QIcon(':/processing/images/model.png')
+
+    def execute(self):
+        try:
+            dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.MODELS)
+            dlg.exec_()
+            if dlg.updateToolbox:                        
+                self.toolbox.updateProvider('model')                
+        except HTTPError:
+            QMessageBox.critical(iface.mainWindow(), "Connection problem", "Could not connect to scripts/models repository")
+                    
+   
+def readUrl(url):        
+    try:
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+        return urllib2.urlopen(url).read()
+    finally:        
+        QApplication.restoreOverrideCursor()
+        
+
+class GetScriptsAndModelsDialog(QDialog,  Ui_DlgGetScriptsAndModels):
+
+    HELP_TEXT = ("<h3> Processing resources manager </h3>"
+                "<p>Check/uncheck algorithms in the tree to select the ones that you want to install or remove</p>"
+                "<p>Algorithms are divided in 3 groups:</p>"
+                "<ul><li><b>Installed:</b> Algorihms already in your system, with the latest version available</li>"
+                "<li><b>Upgradable:</b> Algorihms already in your system, but with a newer version available in the server</li>"
+                "<li><b>Not installed:</b> Algorithms not installed in your system</li></ul>")    
+    MODELS = 0
+    SCRIPTS = 1
+    
+    def __init__(self, resourceType):
+        QDialog.__init__(self, iface.mainWindow())
+        self.resourceType = resourceType  
+        if self.resourceType == self.MODELS:
+            self.folder = ModelerUtils.modelsFolder()                    
+            self.urlBase = "https://raw.githubusercontent.com/qgis/QGIS-Processing/master/models/"
+            self.icon = QtGui.QIcon(os.path.dirname(__file__) + '/../images/model.png')
+        else:
+            self.folder = ScriptUtils.scriptsFolder()      
+            self.urlBase = "https://raw.githubusercontent.com/qgis/QGIS-Processing/master/scripts/"
+            self.icon = QtGui.QIcon(os.path.dirname(__file__) + '/../images/script.png')                    
+        self.lastSelectedItem = None        
+        self.setupUi(self)
+        self.populateTree()  
+        self.updateToolbox = False                                       
+        self.buttonBox.accepted.connect(self.okPressed)
+        self.buttonBox.rejected.connect(self.cancelPressed)
+        self.tree.currentItemChanged .connect(self.currentItemChanged)
+       
+    def populateTree(self):   
+        self.uptodateItem = QTreeWidgetItem()         
+        self.uptodateItem.setText(0, "Installed")
+        self.toupdateItem = QTreeWidgetItem()
+        self.toupdateItem.setText(0, "Upgradable")
+        self.notinstalledItem = QTreeWidgetItem()
+        self.notinstalledItem.setText(0, "Not installed")        
+        self.toupdateItem.setIcon(0, self.icon)
+        self.uptodateItem.setIcon(0, self.icon)
+        self.notinstalledItem.setIcon(0, self.icon)                                             
+        resources = readUrl(self.urlBase + "list.txt").splitlines()
+        resources = [r.split(",") for r in resources]
+        for filename, version, name in resources:
+            treeBranch = self.getTreeBranchForState(filename, float(version))            
+            item = TreeItem(filename, name, self.icon)                
+            treeBranch.addChild(item)
+            if treeBranch != self.notinstalledItem:
+                item.setCheckState(0, Qt.Checked)
+
+        self.tree.addTopLevelItem(self.toupdateItem)
+        self.tree.addTopLevelItem(self.notinstalledItem) 
+        self.tree.addTopLevelItem(self.uptodateItem)
+        
+        self.webView.setHtml(self.HELP_TEXT)
+        
+    def currentItemChanged(self, item, prev):        
+        if isinstance(item, TreeItem):
+            try:
+                url = self.urlBase + item.filename.replace(" ","%20") + ".help"
+                helpContent = readUrl(url)
+                descriptions = json.loads(helpContent)
+                html = "<h2>%s</h2>" % item.name
+                html+="<p><b>Description:</b> " + getDescription(ALG_DESC, descriptions)+"</p>"
+                html+="<p><b>Created by:</b> " + getDescription(ALG_CREATOR, descriptions)+"</p>"
+                html+="<p><b>Version:</b> " + getDescription(ALG_VERSION, descriptions)+"</p>"               
+            except HTTPError, e:
+                html = "<h2>No detailed description available for this script</h2>"
+            self.webView.setHtml(html)
+        else:
+            self.webView.setHtml(self.HELP_TEXT)
+        
+    def getTreeBranchForState(self, filename, version):       
+        if not os.path.exists(os.path.join(self.folder, filename)):
+            return self.notinstalledItem
+        else:
+            helpFile = os.path.join(self.folder, filename + ".help")
+            if not os.path.exists(helpFile):
+                currentVersion = 1
+            else:
+                with open(helpFile) as f:
+                    helpContent = json.load(f)
+                    try:
+                        currentVersion = float(helpContent[Help2Html.ALG_VERSION])
+                    except:
+                        currentVersion = 1
+            print filename, currentVersion, version
+            if version > currentVersion:
+                print version - currentVersion
+                return self.toupdateItem
+            else:
+                return self.uptodateItem
+
+    
+    def cancelPressed(self):
+        self.close() 
+        
+    def okPressed(self):
+        toDownload = []
+        for i in xrange(self.toupdateItem.childCount()):
+            item = self.toupdateItem.child(i)
+            if item.checkState(0) == Qt.Checked:
+                toDownload.append(item.filename)
+        for i in xrange(self.notinstalledItem.childCount()):
+            item = self.notinstalledItem.child(i)
+            if item.checkState(0) == Qt.Checked:
+                toDownload.append(item.filename)                
+        
+        if toDownload:                        
+            self.progressBar.setMaximum(len(toDownload))
+            for i, filename in enumerate(toDownload):
+                QCoreApplication.processEvents()
+                url = self.urlBase + filename.replace(" ","%20")
+                code = readUrl(url)                                                    
+                path = os.path.join(self.folder, filename)
+                with open(path, "w") as f:
+                    f.write(code)   
+                self.progressBar.setValue(i + 1)
+        
+        toDelete = []
+        for i in xrange(self.uptodateItem.childCount()):
+            item = self.uptodateItem.child(i)
+            if item.checkState(0) == Qt.Unchecked:
+                toDelete.append(item.filename)
+        for filename in toDelete:
+            path = os.path.join(self.folder, filename)
+            os.remove(path)
+               
+        self.updateToolbox = len(toDownload) + len(toDelete)> 0
+        self.close()         
+        
+                    
+class TreeItem(QTreeWidgetItem):
+
+    def __init__(self, filename, name, icon):
+        QTreeWidgetItem.__init__(self)
+        self.name = name   
+        self.filename = filename        
+        self.setText(0, name)   
+        self.setIcon(0, icon)     
+        self.setCheckState(0, Qt.Unchecked)
+
+
+
+
+
+
+        

python/plugins/processing/gui/Help2Html.py

@@ -33,6 +33,7 @@
 ALG_DESC = 'ALG_DESC'
 ALG_CREATOR = 'ALG_CREATOR'
 ALG_HELP_CREATOR = 'ALG_HELP_CREATOR'
+ALG_VERSION = 'ALG_VERSION'
 
 exps = [(r"\*(.*?)\*", r"<i>\1</i>"),
         ("``(.*?)``", r'<FONT FACE="courier">\1</FONT>'),
@@ -56,7 +57,6 @@ def getHtmlFromRstFile(rst):
 def getHtmlFromHelpFile(alg, helpFile):
     if not os.path.exists(helpFile):
         return None
-    alg = alg
     with open(helpFile) as f:    
         descriptions = json.load(f)
     s = '<html><body><h2>Algorithm description</h2>\n'
@@ -72,6 +72,7 @@ def getHtmlFromHelpFile(alg, helpFile):
     s += '<br>'
     s += '<p align="right">Algorithm author: ' + getDescription(ALG_CREATOR, descriptions) + '</p>'
     s += '<p align="right">Help author: ' + getDescription(ALG_HELP_CREATOR, descriptions) + '</p>'
+    s += '<p align="right">Algorithm version: ' + getDescription(ALG_VERSION, descriptions) + '</p>'
     s += '</body></html>'
     return s
 
@@ -80,3 +81,4 @@ def getDescription(name, descriptions):
         return descriptions[name].replace("\n", "<br>")
     else:
         return ''
+

python/plugins/processing/gui/HelpEditionDialog.py

@@ -40,6 +40,7 @@ class HelpEditionDialog(QDialog, Ui_DlgHelpEdition):
     ALG_DESC = 'ALG_DESC'
     ALG_CREATOR = 'ALG_CREATOR'
     ALG_HELP_CREATOR = 'ALG_HELP_CREATOR'
+    ALG_VERSION = 'ALG_VERSION'
 
     def __init__(self, alg):
         QDialog.__init__(self)
@@ -117,6 +118,9 @@ def fillTree(self):
         item = TreeDescriptionItem('Algorithm help written by',
                                    self.ALG_HELP_CREATOR)
         self.tree.addTopLevelItem(item)
+        item = TreeDescriptionItem('Algorithm version',
+                                   self.ALG_VERSION)
+        self.tree.addTopLevelItem(item)
 
     def changeItem(self):
         item = self.tree.currentItem()

python/plugins/processing/modeler/ModelerAlgorithmProvider.py

@@ -40,13 +40,14 @@
 from processing.modeler.CreateNewModelAction import CreateNewModelAction
 from processing.modeler.DeleteModelAction import DeleteModelAction
 from processing.modeler.AddModelFromFileAction import AddModelFromFileAction
+from processing.gui.GetScriptsAndModels import GetModelsAction
 
 
 class ModelerAlgorithmProvider(AlgorithmProvider):
 
     def __init__(self):
         AlgorithmProvider.__init__(self)
-        self.actions = [CreateNewModelAction(), AddModelFromFileAction()]
+        self.actions = [CreateNewModelAction(), AddModelFromFileAction(), GetModelsAction()]
         self.contextMenuActions = [EditModelAction(), DeleteModelAction(),
                                    SaveAsPythonScriptAction()]
 

python/plugins/processing/script/ScriptAlgorithmProvider.py

@@ -38,7 +38,7 @@
 from processing.script.ScriptUtils import ScriptUtils
 from processing.script.WrongScriptException import WrongScriptException
 from processing.script.AddScriptFromFileAction import AddScriptFromFileAction
-
+from processing.gui.GetScriptsAndModels import GetScriptsAction
 import processing.resources_rc
 
 
@@ -48,7 +48,8 @@ def __init__(self):
         AlgorithmProvider.__init__(self)
         self.actions.extend([CreateNewScriptAction('Create new script',
                             CreateNewScriptAction.SCRIPT_PYTHON),
-                            AddScriptFromFileAction()])
+                            AddScriptFromFileAction(),
+                            GetScriptsAction()])
         self.contextMenuActions = \
             [EditScriptAction(EditScriptAction.SCRIPT_PYTHON),
              DeleteScriptAction(DeleteScriptAction.SCRIPT_PYTHON)]

python/plugins/processing/script/scripts/Unique_values_count.py

@@ -1,4 +1,4 @@
-##[Example scripts]aster processing=group
+##[Example scripts]=group
 ##input=raster
 ##round_values_to_ndigits=number 3
 ##output_file=output html