Merge pull request #4835 from nyalldawson/more_processing

[processing] Restore some gdal provider algorithms

python/plugins/processing/algs/gdal/AssignProjection.py

@@ -29,10 +29,10 @@
 
 from qgis.PyQt.QtGui import QIcon
 
+from qgis.core import (QgsProcessingParameterRasterLayer,
+                       QgsProcessingParameterCrs,
+                       QgsProcessingOutputRasterLayer)
 from processing.algs.gdal.GdalAlgorithm import GdalAlgorithm
-from processing.core.parameters import ParameterRaster
-from processing.core.parameters import ParameterCrs
-from processing.core.outputs import OutputRaster
 from processing.algs.gdal.GdalUtils import GdalUtils
 
 from processing.tools.system import isWindows
@@ -41,18 +41,19 @@
 
 
 class AssignProjection(GdalAlgorithm):
-
     INPUT = 'INPUT'
     CRS = 'CRS'
     OUTPUT = 'OUTPUT'
 
     def __init__(self):
         super().__init__()
-        self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer'), False))
-        self.addParameter(ParameterCrs(self.CRS,
-                                       self.tr('Desired CRS'), ''))
 
-        self.addOutput(OutputRaster(self.OUTPUT, self.tr('Layer with projection'), True))
+    def initAlgorithm(self, config=None):
+        self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT, self.tr('Input layer'), optional=False))
+        self.addParameter(QgsProcessingParameterCrs(self.CRS,
+                                                    self.tr('Desired CRS')))
+
+        self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT, self.tr('Layer with projection')))
 
     def name(self):
         return 'assignprojection'
@@ -66,10 +67,11 @@ def icon(self):
     def group(self):
         return self.tr('Raster projections')
 
-    def getConsoleCommands(self, parameters):
-        fileName = self.getParameterValue(self.INPUT)
-        crs = self.getParameterValue(self.CRS)
-        output = self.getOutputValue(self.OUTPUT)  # NOQA
+    def getConsoleCommands(self, parameters, context, feedback):
+        inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
+        fileName = inLayer.source()
+
+        crs = self.parameterAsCrs(parameters, self.CRS, context).authid()
 
         arguments = []
         arguments.append('-a_srs')

python/plugins/processing/algs/gdal/ClipByExtent.py

@@ -55,6 +55,8 @@ class ClipByExtent(GdalAlgorithm):
 
     def __init__(self):
         super().__init__()
+
+    def initAlgorithm(self, config=None):
         self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer')))
         self.addParameter(ParameterString(self.NO_DATA,
                                           self.tr("Nodata value, leave blank to take the nodata value from input"),
@@ -83,7 +85,7 @@ def icon(self):
     def group(self):
         return self.tr('Raster extraction')
 
-    def getConsoleCommands(self, parameters):
+    def getConsoleCommands(self, parameters, context, feedback):
         out = self.getOutputValue(self.OUTPUT)
         noData = self.getParameterValue(self.NO_DATA)
         opts = self.getParameterValue(self.OPTIONS)

python/plugins/processing/algs/gdal/ClipByMask.py

@@ -45,7 +45,6 @@
 from processing.algs.gdal.GdalUtils import GdalUtils
 
 from processing.tools import dataobjects
-from processing.tools.vector import ogrConnectionString
 
 pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0]
 
@@ -65,6 +64,8 @@ class ClipByMask(GdalAlgorithm):
 
     def __init__(self):
         super().__init__()
+
+    def initAlgorithm(self, config=None):
         self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer'), False))
         self.addParameter(ParameterVector(self.MASK, self.tr('Mask layer'),
                                           [dataobjects.TYPE_VECTOR_POLYGON]))
@@ -102,12 +103,12 @@ def icon(self):
     def group(self):
         return self.tr('Raster extraction')
 
-    def getConsoleCommands(self, parameters):
+    def getConsoleCommands(self, parameters, context, feedback):
         out = self.getOutputValue(self.OUTPUT)
         mask = self.getParameterValue(self.MASK)
         context = dataobjects.createContext()
         maskLayer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.MASK), context)
-        ogrMask = ogrConnectionString(mask)[1:-1]
+        ogrMask = GdalUtils.ogrConnectionString(mask, context)[1:-1]
         noData = self.getParameterValue(self.NO_DATA)
         opts = self.getParameterValue(self.OPTIONS)
 

python/plugins/processing/algs/gdal/ColorRelief.py

@@ -26,13 +26,13 @@
 
 __revision__ = '$Format:%H$'
 
+from qgis.core import (QgsProcessingParameterRasterLayer,
+                       QgsProcessingParameterBand,
+                       QgsProcessingParameterBoolean,
+                       QgsProcessingParameterEnum,
+                       QgsProcessingParameterFile,
+                       QgsProcessingParameterRasterDestination)
 from processing.algs.gdal.GdalAlgorithm import GdalAlgorithm
-from processing.core.parameters import ParameterRaster
-from processing.core.parameters import ParameterBoolean
-from processing.core.parameters import ParameterNumber
-from processing.core.parameters import ParameterFile
-from processing.core.parameters import ParameterSelection
-from processing.core.outputs import OutputRaster
 from processing.algs.gdal.GdalUtils import GdalUtils
 
 
@@ -49,17 +49,19 @@ class ColorRelief(GdalAlgorithm):
 
     def __init__(self):
         super().__init__()
-        self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer')))
-        self.addParameter(ParameterNumber(
-            self.BAND, self.tr('Band number'), 1, 99, 1))
-        self.addParameter(ParameterBoolean(self.COMPUTE_EDGES,
-                                           self.tr('Compute edges'), False))
-        self.addParameter(ParameterFile(self.COLOR_TABLE,
-                                        self.tr('Color configuration file'), optional=False))
-        self.addParameter(ParameterSelection(self.MATCH_MODE,
-                                             self.tr('Matching mode'), self.MATCHING_MODES, 0))
-
-        self.addOutput(OutputRaster(self.OUTPUT, self.tr('Color relief')))
+
+    def initAlgorithm(self, config=None):
+        self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT, self.tr('Input layer')))
+        self.addParameter(QgsProcessingParameterBand(
+            self.BAND, self.tr('Band number'), parentLayerParameterName=self.INPUT))
+        self.addParameter(QgsProcessingParameterBoolean(self.COMPUTE_EDGES,
+                                                        self.tr('Compute edges'), defaultValue=False))
+        self.addParameter(QgsProcessingParameterFile(self.COLOR_TABLE,
+                                                     self.tr('Color configuration file'), optional=False))
+        self.addParameter(QgsProcessingParameterEnum(self.MATCH_MODE,
+                                                     self.tr('Matching mode'), options=self.MATCHING_MODES, defaultValue=0))
+
+        self.addParameter(QgsProcessingParameterRasterDestination(self.OUTPUT, self.tr('Color relief')))
 
     def name(self):
         return 'colorrelief'
@@ -70,23 +72,25 @@ def displayName(self):
     def group(self):
         return self.tr('Raster analysis')
 
-    def getConsoleCommands(self, parameters):
+    def getConsoleCommands(self, parameters, context, feedback):
         arguments = ['color-relief']
-        arguments.append(str(self.getParameterValue(self.INPUT)))
-        arguments.append(str(self.getParameterValue(self.COLOR_TABLE)))
+        inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
+        arguments.append(inLayer.source())
+        arguments.append(self.parameterAsFile(parameters, self.COLOR_TABLE, context))
         #filePath = unicode(self.getParameterValue(self.COLOR_TABLE))
         #if filePath is None or filePath == '':
         #    filePath = os.path.join(os.path.dirname(__file__), 'terrain.txt')
         #arguments.append(filePath)
-        arguments.append(str(self.getOutputValue(self.OUTPUT)))
+        out = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
+        arguments.append(out)
 
         arguments.append('-b')
-        arguments.append(str(self.getParameterValue(self.BAND)))
+        arguments.append(str(self.parameterAsInt(parameters, self.BAND, context)))
 
-        if self.getParameterValue(self.COMPUTE_EDGES):
+        if self.parameterAsBool(parameters, self.COMPUTE_EDGES, context):
             arguments.append('-compute_edges')
 
-        mode = self.getParameterValue(self.MATCH_MODE)
+        mode = self.parameterAsEnum(parameters, self.MATCH_MODE, context)
         if mode == 1:
             arguments.append('-exact_color_entry')
         elif mode == 2:

python/plugins/processing/algs/gdal/GdalAlgorithm.py

@@ -28,58 +28,85 @@
 import os
 import re
 
-from qgis.PyQt.QtCore import QUrl
+from qgis.PyQt.QtCore import QUrl, QCoreApplication
 
 from qgis.core import (QgsApplication,
                        QgsVectorFileWriter,
-                       QgsProcessingUtils,
-                       QgsProject)
+                       QgsProcessingAlgorithm)
 
-from processing.core.GeoAlgorithm import GeoAlgorithm
 from processing.algs.gdal.GdalAlgorithmDialog import GdalAlgorithmDialog
 from processing.algs.gdal.GdalUtils import GdalUtils
-from processing.tools import dataobjects
 
 pluginPath = os.path.normpath(os.path.join(
     os.path.split(os.path.dirname(__file__))[0], os.pardir))
 
 
-class GdalAlgorithm(GeoAlgorithm):
+class GdalAlgorithm(QgsProcessingAlgorithm):
 
     def __init__(self):
-        GeoAlgorithm.__init__(self)
+        super().__init__()
+        self.output_values = {}
 
     def icon(self):
         return QgsApplication.getThemeIcon("/providerGdal.svg")
 
     def svgIconPath(self):
         return QgsApplication.iconPath("providerGdal.svg")
 
+    def createInstance(self, config={}):
+        return self.__class__()
+
     def createCustomParametersWidget(self, parent):
         return GdalAlgorithmDialog(self)
 
-    def getConsoleCommands(self, parameters):
+    def getConsoleCommands(self, parameters, context, feedback):
         return None
 
+    def getOgrCompatibleSource(self, parameter_name, parameters, context, feedback):
+        """
+        Interprets a parameter as an OGR compatible source and layer name
+        """
+        input_layer = self.parameterAsVectorLayer(parameters, parameter_name, context)
+        ogr_data_path = None
+        ogr_layer_name = None
+        if input_layer is None:
+            # parameter is not a vector layer - try to convert to a source compatible with OGR
+            # and extract selection if required
+            ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
+                                                                      QgsVectorFileWriter.supportedFormatExtensions(),
+                                                                      feedback=feedback)
+            ogr_layer_name = GdalUtils.ogrLayerName(ogr_data_path)
+        elif input_layer.dataProvider().name() == 'ogr':
+            # parameter is a vector layer, with OGR data provider
+            # so extract selection if required
+            ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
+                                                                      QgsVectorFileWriter.supportedFormatExtensions(),
+                                                                      feedback=feedback)
+            ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
+        else:
+            # vector layer, but not OGR - get OGR compatible path
+            # TODO - handle "selected features only" mode!!
+            ogr_data_path = GdalUtils.ogrConnectionString(input_layer.dataProvider().dataSourceUri(), context)[1:-1]
+            ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
+        return ogr_data_path, ogr_layer_name
+
+    def setOutputValue(self, name, value):
+        self.output_values[name] = value
+
     def processAlgorithm(self, parameters, context, feedback):
-        commands = self.getConsoleCommands(parameters)
-        layers = QgsProcessingUtils.compatibleVectorLayers(QgsProject.instance())
-        supported = QgsVectorFileWriter.supportedFormatExtensions()
-        for i, c in enumerate(commands):
-            for layer in layers:
-                if layer.source() in c:
-                    exported = dataobjects.exportVectorLayer(layer, supported)
-                    exportedFileName = os.path.splitext(os.path.split(exported)[1])[0]
-                    c = c.replace(layer.source(), exported)
-                    if os.path.isfile(layer.source()):
-                        fileName = os.path.splitext(os.path.split(layer.source())[1])[0]
-                        c = re.sub('[\s]{}[\s]'.format(fileName), ' ' + exportedFileName + ' ', c)
-                        c = re.sub('[\s]{}'.format(fileName), ' ' + exportedFileName, c)
-                        c = re.sub('["\']{}["\']'.format(fileName), "'" + exportedFileName + "'", c)
-
-            commands[i] = c
+        commands = self.getConsoleCommands(parameters, context, feedback)
         GdalUtils.runGdal(commands, feedback)
 
+        # auto generate outputs
+        results = {}
+        for o in self.outputDefinitions():
+            if o.name() in parameters:
+                results[o.name()] = parameters[o.name()]
+        for k, v in self.output_values.items():
+            results[k] = v
+
+        return results
+
     def helpUrl(self):
         helpPath = GdalUtils.gdalHelpPath()
         if helpPath == '':
@@ -100,3 +127,8 @@ def commandName(self):
         if name.endswith(".py"):
             name = name[:-3]
         return name
+
+    def tr(self, string, context=''):
+        if context == '':
+            context = self.__class__.__name__
+        return QCoreApplication.translate(context, string)

python/plugins/processing/algs/gdal/GdalAlgorithmDialog.py

@@ -37,13 +37,17 @@
                                  QSizePolicy,
                                  QDialogButtonBox)
 
-from qgis.gui import QgsMessageBar
+from qgis.core import (QgsProcessingFeedback,
+                       QgsProcessingParameterDefinition)
+from qgis.gui import (QgsMessageBar,
+                      QgsProjectionSelectionWidget)
 
 from processing.gui.AlgorithmDialog import AlgorithmDialog
 from processing.gui.AlgorithmDialogBase import AlgorithmDialogBase
 from processing.gui.ParametersPanel import ParametersPanel
 from processing.gui.MultipleInputPanel import MultipleInputPanel
 from processing.gui.NumberInputPanel import NumberInputPanel
+from processing.tools.dataobjects import createContext
 
 
 class GdalAlgorithmDialog(AlgorithmDialog):
@@ -61,7 +65,8 @@ def __init__(self, alg):
 
         self.runAsBatchButton = QPushButton(QCoreApplication.translate("AlgorithmDialog", "Run as Batch Process…"))
         self.runAsBatchButton.clicked.connect(self.runAsBatch)
-        self.buttonBox.addButton(self.runAsBatchButton, QDialogButtonBox.ResetRole) # reset role to ensure left alignment
+        self.buttonBox.addButton(self.runAsBatchButton,
+                                 QDialogButtonBox.ResetRole)  # reset role to ensure left alignment
 
         self.mainWidget.parametersHaveChanged()
 
@@ -90,27 +95,41 @@ def __init__(self, parent, alg):
     def connectParameterSignals(self):
         for wrapper in list(self.wrappers.values()):
             w = wrapper.widget
-            if isinstance(w, QLineEdit):
-                w.textChanged.connect(self.parametersHaveChanged)
-            elif isinstance(w, QComboBox):
-                w.currentIndexChanged.connect(self.parametersHaveChanged)
-            elif isinstance(w, QCheckBox):
-                w.stateChanged.connect(self.parametersHaveChanged)
-            elif isinstance(w, MultipleInputPanel):
-                w.selectionChanged.connect(self.parametersHaveChanged)
-            elif isinstance(w, NumberInputPanel):
-                w.hasChanged.connect(self.parametersHaveChanged)
+            self.connectWidgetChangedSignals(w)
+            for c in w.children():
+                self.connectWidgetChangedSignals(c)
+
+    def connectWidgetChangedSignals(self, w):
+        if isinstance(w, QLineEdit):
+            w.textChanged.connect(self.parametersHaveChanged)
+        elif isinstance(w, QComboBox):
+            w.currentIndexChanged.connect(self.parametersHaveChanged)
+        elif isinstance(w, QgsProjectionSelectionWidget):
+            w.crsChanged.connect(self.parametersHaveChanged)
+        elif isinstance(w, QCheckBox):
+            w.stateChanged.connect(self.parametersHaveChanged)
+        elif isinstance(w, MultipleInputPanel):
+            w.selectionChanged.connect(self.parametersHaveChanged)
+        elif isinstance(w, NumberInputPanel):
+            w.hasChanged.connect(self.parametersHaveChanged)
 
     def parametersHaveChanged(self):
+        context = createContext()
+        feedback = QgsProcessingFeedback()
         try:
             parameters = self.parent.getParamValues()
             for output in self.alg.destinationParameterDefinitions():
-                if parameters[output.name()] is None:
+                if not output.name() in parameters or parameters[output.name()] is None:
                     parameters[output.name()] = self.tr("[temporary file]")
-            commands = self.alg.getConsoleCommands(parameters)
+            for p in self.alg.parameterDefinitions():
+                if (not p.name() in parameters and not p.flags() & QgsProcessingParameterDefinition.FlagOptional) \
+                        or (not p.checkValueIsAcceptable(parameters[p.name()], context)):
+                    # not ready yet
+                    self.text.setPlainText('')
+                    return
+
+            commands = self.alg.getConsoleCommands(parameters, context, feedback)
             commands = [c for c in commands if c not in ['cmd.exe', '/C ']]
             self.text.setPlainText(" ".join(commands))
         except AlgorithmDialogBase.InvalidParameterValue as e:
             self.text.setPlainText(self.tr("Invalid value for parameter '{0}'").format(e.parameter.description()))
-        except:
-            self.text.setPlainText("")