Skip to content
Permalink
Browse files

Merge pull request #4835 from nyalldawson/more_processing

[processing] Restore some gdal provider algorithms
  • Loading branch information
nyalldawson committed Aug 13, 2017
2 parents 7768f06 + f9750dd commit 8d615543b7b0d2f79f4f19724fb45202167048fe
Showing with 979 additions and 825 deletions.
  1. +14 −12 python/plugins/processing/algs/gdal/AssignProjection.py
  2. +3 −1 python/plugins/processing/algs/gdal/ClipByExtent.py
  3. +4 −3 python/plugins/processing/algs/gdal/ClipByMask.py
  4. +28 −24 python/plugins/processing/algs/gdal/ColorRelief.py
  5. +56 −24 python/plugins/processing/algs/gdal/GdalAlgorithm.py
  6. +35 −16 python/plugins/processing/algs/gdal/GdalAlgorithmDialog.py
  7. +16 −15 python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py
  8. +144 −2 python/plugins/processing/algs/gdal/GdalUtils.py
  9. +3 −1 python/plugins/processing/algs/gdal/GridAverage.py
  10. +3 −1 python/plugins/processing/algs/gdal/GridDataMetrics.py
  11. +3 −1 python/plugins/processing/algs/gdal/GridInvDist.py
  12. +3 −1 python/plugins/processing/algs/gdal/GridNearest.py
  13. +32 −28 python/plugins/processing/algs/gdal/aspect.py
  14. +3 −1 python/plugins/processing/algs/gdal/buildvrt.py
  15. +3 −1 python/plugins/processing/algs/gdal/contour.py
  16. +3 −1 python/plugins/processing/algs/gdal/extractprojection.py
  17. +3 −1 python/plugins/processing/algs/gdal/fillnodata.py
  18. +3 −1 python/plugins/processing/algs/gdal/gdal2tiles.py
  19. +3 −1 python/plugins/processing/algs/gdal/gdal2xyz.py
  20. +3 −1 python/plugins/processing/algs/gdal/gdaladdo.py
  21. +3 −1 python/plugins/processing/algs/gdal/gdalcalc.py
  22. +3 −1 python/plugins/processing/algs/gdal/gdaltindex.py
  23. +3 −1 python/plugins/processing/algs/gdal/hillshade.py
  24. +3 −1 python/plugins/processing/algs/gdal/information.py
  25. +3 −1 python/plugins/processing/algs/gdal/merge.py
  26. +3 −1 python/plugins/processing/algs/gdal/nearblack.py
  27. +7 −6 python/plugins/processing/algs/gdal/offsetcurve.py
  28. +6 −5 python/plugins/processing/algs/gdal/ogr2ogr.py
  29. +6 −5 python/plugins/processing/algs/gdal/ogr2ogrbuffer.py
  30. +8 −7 python/plugins/processing/algs/gdal/ogr2ogrclip.py
  31. +6 −5 python/plugins/processing/algs/gdal/ogr2ogrclipextent.py
  32. +6 −5 python/plugins/processing/algs/gdal/ogr2ogrdissolve.py
  33. +32 −30 python/plugins/processing/algs/gdal/ogr2ogrpointsonlines.py
  34. +6 −5 python/plugins/processing/algs/gdal/ogr2ogrtabletopostgislist.py
  35. +129 −129 python/plugins/processing/algs/gdal/ogr2ogrtopostgis.py
  36. +6 −5 python/plugins/processing/algs/gdal/ogr2ogrtopostgislist.py
  37. +4 −4 python/plugins/processing/algs/gdal/ogrinfo.py
  38. +4 −4 python/plugins/processing/algs/gdal/ogrsql.py
  39. +7 −6 python/plugins/processing/algs/gdal/onesidebuffer.py
  40. +3 −1 python/plugins/processing/algs/gdal/pct2rgb.py
  41. +3 −1 python/plugins/processing/algs/gdal/polygonize.py
  42. +3 −1 python/plugins/processing/algs/gdal/proximity.py
  43. +6 −6 python/plugins/processing/algs/gdal/rasterize.py
  44. +6 −5 python/plugins/processing/algs/gdal/rasterize_over.py
  45. +3 −1 python/plugins/processing/algs/gdal/retile.py
  46. +3 −1 python/plugins/processing/algs/gdal/rgb2pct.py
  47. +3 −1 python/plugins/processing/algs/gdal/roughness.py
  48. +3 −1 python/plugins/processing/algs/gdal/sieve.py
  49. +3 −1 python/plugins/processing/algs/gdal/slope.py
  50. +3 −1 python/plugins/processing/algs/gdal/tpi.py
  51. +3 −1 python/plugins/processing/algs/gdal/translate.py
  52. +19 −15 python/plugins/processing/algs/gdal/tri.py
  53. +6 −6 python/plugins/processing/algs/gdal/ui/RasterOptionsWidget.py
  54. +76 −77 python/plugins/processing/algs/gdal/warp.py
  55. +1 −1 python/plugins/processing/core/Processing.py
  56. +3 −0 python/plugins/processing/gui/AlgorithmDialog.py
  57. +54 −8 python/plugins/processing/tests/GdalAlgorithmsTest.py
  58. +1 −42 python/plugins/processing/tests/ToolsTest.py
  59. +158 −156 python/plugins/processing/tests/testdata/gdal_algorithm_tests.yaml
  60. +1 −136 python/plugins/processing/tools/vector.py
  61. +5 −4 python/testing/__init__.py
  62. +1 −1 src/core/processing/qgsprocessingparameters.cpp
  63. +2 −0 tests/src/core/testqgsprocessing.cpp
@@ -29,10 +29,10 @@

from qgis.PyQt.QtGui import QIcon

from qgis.core import (QgsProcessingParameterRasterLayer,
QgsProcessingParameterCrs,
QgsProcessingOutputRasterLayer)
from processing.algs.gdal.GdalAlgorithm import GdalAlgorithm
from processing.core.parameters import ParameterRaster
from processing.core.parameters import ParameterCrs
from processing.core.outputs import OutputRaster
from processing.algs.gdal.GdalUtils import GdalUtils

from processing.tools.system import isWindows
@@ -41,18 +41,19 @@


class AssignProjection(GdalAlgorithm):

INPUT = 'INPUT'
CRS = 'CRS'
OUTPUT = 'OUTPUT'

def __init__(self):
super().__init__()
self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer'), False))
self.addParameter(ParameterCrs(self.CRS,
self.tr('Desired CRS'), ''))

self.addOutput(OutputRaster(self.OUTPUT, self.tr('Layer with projection'), True))
def initAlgorithm(self, config=None):
self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT, self.tr('Input layer'), optional=False))
self.addParameter(QgsProcessingParameterCrs(self.CRS,
self.tr('Desired CRS')))

self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT, self.tr('Layer with projection')))

def name(self):
return 'assignprojection'
@@ -66,10 +67,11 @@ def icon(self):
def group(self):
return self.tr('Raster projections')

def getConsoleCommands(self, parameters):
fileName = self.getParameterValue(self.INPUT)
crs = self.getParameterValue(self.CRS)
output = self.getOutputValue(self.OUTPUT) # NOQA
def getConsoleCommands(self, parameters, context, feedback):
inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
fileName = inLayer.source()

crs = self.parameterAsCrs(parameters, self.CRS, context).authid()

arguments = []
arguments.append('-a_srs')
@@ -55,6 +55,8 @@ class ClipByExtent(GdalAlgorithm):

def __init__(self):
super().__init__()

def initAlgorithm(self, config=None):
self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer')))
self.addParameter(ParameterString(self.NO_DATA,
self.tr("Nodata value, leave blank to take the nodata value from input"),
@@ -83,7 +85,7 @@ def icon(self):
def group(self):
return self.tr('Raster extraction')

def getConsoleCommands(self, parameters):
def getConsoleCommands(self, parameters, context, feedback):
out = self.getOutputValue(self.OUTPUT)
noData = self.getParameterValue(self.NO_DATA)
opts = self.getParameterValue(self.OPTIONS)
@@ -45,7 +45,6 @@
from processing.algs.gdal.GdalUtils import GdalUtils

from processing.tools import dataobjects
from processing.tools.vector import ogrConnectionString

pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0]

@@ -65,6 +64,8 @@ class ClipByMask(GdalAlgorithm):

def __init__(self):
super().__init__()

def initAlgorithm(self, config=None):
self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer'), False))
self.addParameter(ParameterVector(self.MASK, self.tr('Mask layer'),
[dataobjects.TYPE_VECTOR_POLYGON]))
@@ -102,12 +103,12 @@ def icon(self):
def group(self):
return self.tr('Raster extraction')

def getConsoleCommands(self, parameters):
def getConsoleCommands(self, parameters, context, feedback):
out = self.getOutputValue(self.OUTPUT)
mask = self.getParameterValue(self.MASK)
context = dataobjects.createContext()
maskLayer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.MASK), context)
ogrMask = ogrConnectionString(mask)[1:-1]
ogrMask = GdalUtils.ogrConnectionString(mask, context)[1:-1]
noData = self.getParameterValue(self.NO_DATA)
opts = self.getParameterValue(self.OPTIONS)

@@ -26,13 +26,13 @@

__revision__ = '$Format:%H$'

from qgis.core import (QgsProcessingParameterRasterLayer,
QgsProcessingParameterBand,
QgsProcessingParameterBoolean,
QgsProcessingParameterEnum,
QgsProcessingParameterFile,
QgsProcessingParameterRasterDestination)
from processing.algs.gdal.GdalAlgorithm import GdalAlgorithm
from processing.core.parameters import ParameterRaster
from processing.core.parameters import ParameterBoolean
from processing.core.parameters import ParameterNumber
from processing.core.parameters import ParameterFile
from processing.core.parameters import ParameterSelection
from processing.core.outputs import OutputRaster
from processing.algs.gdal.GdalUtils import GdalUtils


@@ -49,17 +49,19 @@ class ColorRelief(GdalAlgorithm):

def __init__(self):
super().__init__()
self.addParameter(ParameterRaster(self.INPUT, self.tr('Input layer')))
self.addParameter(ParameterNumber(
self.BAND, self.tr('Band number'), 1, 99, 1))
self.addParameter(ParameterBoolean(self.COMPUTE_EDGES,
self.tr('Compute edges'), False))
self.addParameter(ParameterFile(self.COLOR_TABLE,
self.tr('Color configuration file'), optional=False))
self.addParameter(ParameterSelection(self.MATCH_MODE,
self.tr('Matching mode'), self.MATCHING_MODES, 0))

self.addOutput(OutputRaster(self.OUTPUT, self.tr('Color relief')))

def initAlgorithm(self, config=None):
self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT, self.tr('Input layer')))
self.addParameter(QgsProcessingParameterBand(
self.BAND, self.tr('Band number'), parentLayerParameterName=self.INPUT))
self.addParameter(QgsProcessingParameterBoolean(self.COMPUTE_EDGES,
self.tr('Compute edges'), defaultValue=False))
self.addParameter(QgsProcessingParameterFile(self.COLOR_TABLE,
self.tr('Color configuration file'), optional=False))
self.addParameter(QgsProcessingParameterEnum(self.MATCH_MODE,
self.tr('Matching mode'), options=self.MATCHING_MODES, defaultValue=0))

self.addParameter(QgsProcessingParameterRasterDestination(self.OUTPUT, self.tr('Color relief')))

def name(self):
return 'colorrelief'
@@ -70,23 +72,25 @@ def displayName(self):
def group(self):
return self.tr('Raster analysis')

def getConsoleCommands(self, parameters):
def getConsoleCommands(self, parameters, context, feedback):
arguments = ['color-relief']
arguments.append(str(self.getParameterValue(self.INPUT)))
arguments.append(str(self.getParameterValue(self.COLOR_TABLE)))
inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
arguments.append(inLayer.source())
arguments.append(self.parameterAsFile(parameters, self.COLOR_TABLE, context))
#filePath = unicode(self.getParameterValue(self.COLOR_TABLE))
#if filePath is None or filePath == '':
# filePath = os.path.join(os.path.dirname(__file__), 'terrain.txt')
#arguments.append(filePath)
arguments.append(str(self.getOutputValue(self.OUTPUT)))
out = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
arguments.append(out)

arguments.append('-b')
arguments.append(str(self.getParameterValue(self.BAND)))
arguments.append(str(self.parameterAsInt(parameters, self.BAND, context)))

if self.getParameterValue(self.COMPUTE_EDGES):
if self.parameterAsBool(parameters, self.COMPUTE_EDGES, context):
arguments.append('-compute_edges')

mode = self.getParameterValue(self.MATCH_MODE)
mode = self.parameterAsEnum(parameters, self.MATCH_MODE, context)
if mode == 1:
arguments.append('-exact_color_entry')
elif mode == 2:
@@ -28,58 +28,85 @@
import os
import re

from qgis.PyQt.QtCore import QUrl
from qgis.PyQt.QtCore import QUrl, QCoreApplication

from qgis.core import (QgsApplication,
QgsVectorFileWriter,
QgsProcessingUtils,
QgsProject)
QgsProcessingAlgorithm)

from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.algs.gdal.GdalAlgorithmDialog import GdalAlgorithmDialog
from processing.algs.gdal.GdalUtils import GdalUtils
from processing.tools import dataobjects

pluginPath = os.path.normpath(os.path.join(
os.path.split(os.path.dirname(__file__))[0], os.pardir))


class GdalAlgorithm(GeoAlgorithm):
class GdalAlgorithm(QgsProcessingAlgorithm):

def __init__(self):
GeoAlgorithm.__init__(self)
super().__init__()
self.output_values = {}

def icon(self):
return QgsApplication.getThemeIcon("/providerGdal.svg")

def svgIconPath(self):
return QgsApplication.iconPath("providerGdal.svg")

def createInstance(self, config={}):
return self.__class__()

def createCustomParametersWidget(self, parent):
return GdalAlgorithmDialog(self)

def getConsoleCommands(self, parameters):
def getConsoleCommands(self, parameters, context, feedback):
return None

def getOgrCompatibleSource(self, parameter_name, parameters, context, feedback):
"""
Interprets a parameter as an OGR compatible source and layer name
"""
input_layer = self.parameterAsVectorLayer(parameters, parameter_name, context)
ogr_data_path = None
ogr_layer_name = None
if input_layer is None:
# parameter is not a vector layer - try to convert to a source compatible with OGR
# and extract selection if required
ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
QgsVectorFileWriter.supportedFormatExtensions(),
feedback=feedback)
ogr_layer_name = GdalUtils.ogrLayerName(ogr_data_path)
elif input_layer.dataProvider().name() == 'ogr':
# parameter is a vector layer, with OGR data provider
# so extract selection if required
ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
QgsVectorFileWriter.supportedFormatExtensions(),
feedback=feedback)
ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
else:
# vector layer, but not OGR - get OGR compatible path
# TODO - handle "selected features only" mode!!
ogr_data_path = GdalUtils.ogrConnectionString(input_layer.dataProvider().dataSourceUri(), context)[1:-1]
ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
return ogr_data_path, ogr_layer_name

def setOutputValue(self, name, value):
self.output_values[name] = value

def processAlgorithm(self, parameters, context, feedback):
commands = self.getConsoleCommands(parameters)
layers = QgsProcessingUtils.compatibleVectorLayers(QgsProject.instance())
supported = QgsVectorFileWriter.supportedFormatExtensions()
for i, c in enumerate(commands):
for layer in layers:
if layer.source() in c:
exported = dataobjects.exportVectorLayer(layer, supported)
exportedFileName = os.path.splitext(os.path.split(exported)[1])[0]
c = c.replace(layer.source(), exported)
if os.path.isfile(layer.source()):
fileName = os.path.splitext(os.path.split(layer.source())[1])[0]
c = re.sub('[\s]{}[\s]'.format(fileName), ' ' + exportedFileName + ' ', c)
c = re.sub('[\s]{}'.format(fileName), ' ' + exportedFileName, c)
c = re.sub('["\']{}["\']'.format(fileName), "'" + exportedFileName + "'", c)

commands[i] = c
commands = self.getConsoleCommands(parameters, context, feedback)
GdalUtils.runGdal(commands, feedback)

# auto generate outputs
results = {}
for o in self.outputDefinitions():
if o.name() in parameters:
results[o.name()] = parameters[o.name()]
for k, v in self.output_values.items():
results[k] = v

return results

def helpUrl(self):
helpPath = GdalUtils.gdalHelpPath()
if helpPath == '':
@@ -100,3 +127,8 @@ def commandName(self):
if name.endswith(".py"):
name = name[:-3]
return name

def tr(self, string, context=''):
if context == '':
context = self.__class__.__name__
return QCoreApplication.translate(context, string)
@@ -37,13 +37,17 @@
QSizePolicy,
QDialogButtonBox)

from qgis.gui import QgsMessageBar
from qgis.core import (QgsProcessingFeedback,
QgsProcessingParameterDefinition)
from qgis.gui import (QgsMessageBar,
QgsProjectionSelectionWidget)

from processing.gui.AlgorithmDialog import AlgorithmDialog
from processing.gui.AlgorithmDialogBase import AlgorithmDialogBase
from processing.gui.ParametersPanel import ParametersPanel
from processing.gui.MultipleInputPanel import MultipleInputPanel
from processing.gui.NumberInputPanel import NumberInputPanel
from processing.tools.dataobjects import createContext


class GdalAlgorithmDialog(AlgorithmDialog):
@@ -61,7 +65,8 @@ def __init__(self, alg):

self.runAsBatchButton = QPushButton(QCoreApplication.translate("AlgorithmDialog", "Run as Batch Process…"))
self.runAsBatchButton.clicked.connect(self.runAsBatch)
self.buttonBox.addButton(self.runAsBatchButton, QDialogButtonBox.ResetRole) # reset role to ensure left alignment
self.buttonBox.addButton(self.runAsBatchButton,
QDialogButtonBox.ResetRole) # reset role to ensure left alignment

self.mainWidget.parametersHaveChanged()

@@ -90,27 +95,41 @@ def __init__(self, parent, alg):
def connectParameterSignals(self):
for wrapper in list(self.wrappers.values()):
w = wrapper.widget
if isinstance(w, QLineEdit):
w.textChanged.connect(self.parametersHaveChanged)
elif isinstance(w, QComboBox):
w.currentIndexChanged.connect(self.parametersHaveChanged)
elif isinstance(w, QCheckBox):
w.stateChanged.connect(self.parametersHaveChanged)
elif isinstance(w, MultipleInputPanel):
w.selectionChanged.connect(self.parametersHaveChanged)
elif isinstance(w, NumberInputPanel):
w.hasChanged.connect(self.parametersHaveChanged)
self.connectWidgetChangedSignals(w)
for c in w.children():
self.connectWidgetChangedSignals(c)

def connectWidgetChangedSignals(self, w):
if isinstance(w, QLineEdit):
w.textChanged.connect(self.parametersHaveChanged)
elif isinstance(w, QComboBox):
w.currentIndexChanged.connect(self.parametersHaveChanged)
elif isinstance(w, QgsProjectionSelectionWidget):
w.crsChanged.connect(self.parametersHaveChanged)
elif isinstance(w, QCheckBox):
w.stateChanged.connect(self.parametersHaveChanged)
elif isinstance(w, MultipleInputPanel):
w.selectionChanged.connect(self.parametersHaveChanged)
elif isinstance(w, NumberInputPanel):
w.hasChanged.connect(self.parametersHaveChanged)

def parametersHaveChanged(self):
context = createContext()
feedback = QgsProcessingFeedback()
try:
parameters = self.parent.getParamValues()
for output in self.alg.destinationParameterDefinitions():
if parameters[output.name()] is None:
if not output.name() in parameters or parameters[output.name()] is None:
parameters[output.name()] = self.tr("[temporary file]")
commands = self.alg.getConsoleCommands(parameters)
for p in self.alg.parameterDefinitions():
if (not p.name() in parameters and not p.flags() & QgsProcessingParameterDefinition.FlagOptional) \
or (not p.checkValueIsAcceptable(parameters[p.name()], context)):
# not ready yet
self.text.setPlainText('')
return

commands = self.alg.getConsoleCommands(parameters, context, feedback)
commands = [c for c in commands if c not in ['cmd.exe', '/C ']]
self.text.setPlainText(" ".join(commands))
except AlgorithmDialogBase.InvalidParameterValue as e:
self.text.setPlainText(self.tr("Invalid value for parameter '{0}'").format(e.parameter.description()))
except:
self.text.setPlainText("")

0 comments on commit 8d61554

Please sign in to comment.
You can’t perform that action at this time.