[processing] update script provider to use QgsProcessingAlgorithm

subclasses instead of custom script code

Author: alexbruy

python/plugins/processing/algs/qgis/QgisAlgorithmProvider.py

@@ -36,7 +36,7 @@
 from qgis.core import (QgsApplication,
                        QgsProcessingProvider)
 
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 from .QgisAlgorithm import QgisAlgorithm
 
@@ -288,11 +288,11 @@ def getAlgs(self):
                          VectorLayerScatterplot3D()])
 
         # to store algs added by 3rd party plugins as scripts
-        folder = os.path.join(os.path.dirname(__file__), 'scripts')
-        scripts = ScriptUtils.loadFromFolder(folder)
-        for script in scripts:
-            script.allowEdit = False
-        algs.extend(scripts)
+        #folder = os.path.join(os.path.dirname(__file__), 'scripts')
+        #scripts = ScriptUtils.loadFromFolder(folder)
+        #for script in scripts:
+        #    script.allowEdit = False
+        #algs.extend(scripts)
 
         return algs
 

python/plugins/processing/core/Processing.py

@@ -45,12 +45,12 @@
                        QgsProcessingOutputMapLayer)
 
 import processing
-from processing.script.ScriptUtils import ScriptUtils
 from processing.core.ProcessingConfig import ProcessingConfig
 from processing.gui.MessageBarProgress import MessageBarProgress
 from processing.gui.RenderingStyles import RenderingStyles
 from processing.gui.Postprocessing import handleAlgorithmResults
 from processing.gui.AlgorithmExecutor import execute
+from processing.script import ScriptUtils
 from processing.tools import dataobjects
 
 from processing.algs.qgis.QgisAlgorithmProvider import QgisAlgorithmProvider  # NOQA
@@ -105,25 +105,25 @@ def deinitialize():
 
     @staticmethod
     def addScripts(folder):
-        Processing.initialize()
-        provider = QgsApplication.processingRegistry().providerById("qgis")
-        scripts = ScriptUtils.loadFromFolder(folder)
-        # fix_print_with_import
-        print(scripts)
-        for script in scripts:
-            script.allowEdit = False
-            script._icon = provider.icon()
-        provider.externalAlgs.extend(scripts)
-        provider.refreshAlgorithms()
+        #Processing.initialize()
+        #provider = QgsApplication.processingRegistry().providerById("qgis")
+        #scripts = ScriptUtils.loadFromFolder(folder)
+        #for script in scripts:
+        #    script.allowEdit = False
+        #    script._icon = provider.icon()
+        #provider.externalAlgs.extend(scripts)
+        #provider.refreshAlgorithms()
+        pass
 
     @staticmethod
     def removeScripts(folder):
-        provider = QgsApplication.processingRegistry().providerById("qgis")
-        for alg in provider.externalAlgs[::-1]:
-            path = os.path.dirname(alg.descriptionFile)
-            if path == folder:
-                provider.externalAlgs.remove(alg)
-        provider.refreshAlgorithms()
+        #provider = QgsApplication.processingRegistry().providerById("qgis")
+        #for alg in provider.externalAlgs[::-1]:
+        #    path = os.path.dirname(alg.descriptionFile)
+        #    if path == folder:
+        #        provider.externalAlgs.remove(alg)
+        #provider.refreshAlgorithms()
+        pass
 
     @staticmethod
     def runAlgorithm(algOrName, parameters, onFinish=None, feedback=None, context=None):

python/plugins/processing/script/AddScriptFromFileAction.py

@@ -36,7 +36,7 @@
 
 from processing.script.ScriptAlgorithm import ScriptAlgorithm
 from processing.script.WrongScriptException import WrongScriptException
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 pluginPath = os.path.split(os.path.dirname(__file__))[0]
 

python/plugins/processing/script/ScriptAlgorithmProvider.py

@@ -31,6 +31,7 @@
                        QgsProcessingProvider)
 
 from processing.core.ProcessingConfig import ProcessingConfig, Setting
+
 from processing.gui.ProviderActions import (ProviderActions,
                                             ProviderContextMenuActions)
 
@@ -39,9 +40,7 @@
 from processing.script.DeleteScriptAction import DeleteScriptAction
 from processing.script.EditScriptAction import EditScriptAction
 from processing.script.CreateScriptCollectionPluginAction import CreateScriptCollectionPluginAction
-from processing.script.ScriptUtils import ScriptUtils
-
-pluginPath = os.path.split(os.path.dirname(__file__))[0]
+from processing.script import ScriptUtils
 
 
 class ScriptAlgorithmProvider(QgsProcessingProvider):
@@ -59,17 +58,22 @@ def __init__(self):
     def load(self):
         ProcessingConfig.settingIcons[self.name()] = self.icon()
         ProcessingConfig.addSetting(Setting(self.name(),
-                                            ScriptUtils.SCRIPTS_FOLDER,
-                                            self.tr('Scripts folder', 'ScriptAlgorithmProvider'),
-                                            ScriptUtils.defaultScriptsFolder(), valuetype=Setting.MULTIPLE_FOLDERS))
+                                            ScriptUtils.SCRIPTS_FOLDERS,
+                                            self.tr("Scripts folder(s)"),
+                                            ScriptUtils.defaultScriptsFolder(),
+                                            valuetype=Setting.MULTIPLE_FOLDERS))
+
         ProviderActions.registerProviderActions(self, self.actions)
         ProviderContextMenuActions.registerProviderContextMenuActions(self.contextMenuActions)
+
         ProcessingConfig.readSettings()
         self.refreshAlgorithms()
+
         return True
 
     def unload(self):
-        ProcessingConfig.removeSetting(ScriptUtils.SCRIPTS_FOLDER)
+        ProcessingConfig.removeSetting(ScriptUtils.SCRIPTS_FOLDERS)
+
         ProviderActions.deregisterProviderActions(self)
         ProviderContextMenuActions.deregisterProviderContextMenuActions(self.contextMenuActions)
 
@@ -80,26 +84,30 @@ def svgIconPath(self):
         return QgsApplication.iconPath("processingScript.svg")
 
     def id(self):
-        return 'script'
+        return "script"
 
     def name(self):
-        return self.tr('Scripts', 'ScriptAlgorithmProvider')
-
-    def loadAlgorithms(self):
-        self.algs = []
-        folders = ScriptUtils.scriptsFolders()
-        for f in folders:
-            self.algs.extend(ScriptUtils.loadFromFolder(f))
-        self.algs.extend(self.folder_algorithms)
-        for a in self.algs:
-            self.addAlgorithm(a)
-
-    def addAlgorithmsFromFolder(self, folder):
-        self.folder_algorithms.extend(ScriptUtils.loadFromFolder(folder))
+        return self.tr("Scripts")
 
     def supportsNonFileBasedOutput(self):
         # TODO - this may not be strictly true. We probably need a way for scripts
         # to indicate whether individual outputs support non-file based outputs,
         # but for now allow it. At best we expose nice features to users, at worst
         # they'll get an error if they use them with incompatible outputs...
         return True
+
+    def loadAlgorithms(self):
+        self.algs = []
+        folders = ScriptUtils.scriptsFolders()
+        for folder in folders:
+            items = os.scandir(folder)
+            for entry in items:
+                if entry.name.lower().endswith("py") and entry.is_file():
+                    algName = os.path.splitext(entry.name)[0]
+                    filePath = os.path.abspath(os.path.join(folder, entry.name))
+                    alg = ScriptUtils.loadAlgorithm(algName, filePath)
+                    if alg is not None:
+                        self.algs.append(alg)
+
+        for a in self.algs:
+            self.addAlgorithm(a)

python/plugins/processing/script/ScriptEditorDialog.py

@@ -44,7 +44,7 @@
 from processing.gui.HelpEditionDialog import HelpEditionDialog
 
 from processing.script.ScriptAlgorithm import ScriptAlgorithm
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 pluginPath = os.path.split(os.path.dirname(__file__))[0]
 WIDGET, BASE = uic.loadUiType(

python/plugins/processing/script/ScriptUtils.py

@@ -26,53 +26,48 @@
 __revision__ = '$Format:%H$'
 
 import os
-from qgis.core import (QgsProcessingUtils,
-                       QgsMessageLog)
+import inspect
+import importlib
+
+from qgis.PyQt.QtCore import QCoreApplication
+
+from qgis.core import QgsProcessingAlgorithm, QgsMessageLog
+
 from processing.core.ProcessingConfig import ProcessingConfig
+
 from processing.script.ScriptAlgorithm import ScriptAlgorithm
 from processing.script.WrongScriptException import WrongScriptException
+
 from processing.tools.system import mkdir, userFolder
 
-from qgis.PyQt.QtCore import QCoreApplication
+
+SCRIPTS_FOLDERS = "SCRIPTS_FOLDERS"
+
+
+def defaultScriptsFolder():
+    folder = str(os.path.join(userFolder(), "scripts"))
+    mkdir(folder)
+    return os.path.abspath(folder)
+
+
+def scriptsFolders():
+    folder = ProcessingConfig.getSetting(SCRIPTS_FOLDERS)
+    if folder is not None:
+        return folder.split(";")
+    else:
+        return [ScriptUtils.defaultScriptsFolder()]
 
 
-class ScriptUtils:
-
-    SCRIPTS_FOLDER = 'SCRIPTS_FOLDER'
-
-    @staticmethod
-    def defaultScriptsFolder():
-        folder = str(os.path.join(userFolder(), 'scripts'))
-        mkdir(folder)
-        return os.path.abspath(folder)
-
-    @staticmethod
-    def scriptsFolders():
-        folder = ProcessingConfig.getSetting(ScriptUtils.SCRIPTS_FOLDER)
-        if folder is not None:
-            return folder.split(';')
-        else:
-            return [ScriptUtils.defaultScriptsFolder()]
-
-    @staticmethod
-    def loadFromFolder(folder):
-        if not os.path.exists(folder):
-            return []
-        algs = []
-        for path, subdirs, files in os.walk(folder):
-            for descriptionFile in files:
-                if descriptionFile.endswith('py'):
-                    try:
-                        fullpath = os.path.join(path, descriptionFile)
-                        alg = ScriptAlgorithm(fullpath)
-                        if alg.name().strip() != '':
-                            algs.append(alg)
-                    except WrongScriptException as e:
-                        QgsMessageLog.logMessage(e.msg, QCoreApplication.translate('Processing', 'Processing'), QgsMessageLog.CRITICAL)
-                    except Exception as e:
-                        QgsMessageLog.logMessage(
-                            QCoreApplication.translate('Processing', 'Could not load script: {0}\n{1}').format(descriptionFile, str(e)),
-                            QCoreApplication.translate('Processing', 'Processing'),
-                            QgsMessageLog.CRITICAL
-                        )
-        return algs
+def loadAlgorithm(moduleName, filePath):
+    try:
+        spec = importlib.util.spec_from_file_location(moduleName, filePath)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        for x in dir(module):
+            obj = getattr(module, x)
+            if inspect.isclass(obj) and issubclass(obj, QgsProcessingAlgorithm) and obj.__name__ == moduleName:
+                return obj()
+    except ImportError as e:
+        QgsMessageLog.logMessage("Could not import script algorithm '{}' from '{}'\n{}".format(moduleName, filePath, str(e)),
+                                 "Processing",
+                                 QgsMessageLog.CRITICAL)