Skip to content
Permalink
Browse files

Merge pull request #4702 from nyalldawson/processing_pt31

More processing goodness, restore algs
  • Loading branch information
nyalldawson committed Jun 12, 2017
2 parents 0ceeb29 + 2779260 commit 40cae2922809ff4cf6b685e5cf99920e3983d472
Showing with 1,020 additions and 349 deletions.
  1. +30 −4 python/core/__init__.py
  2. +10 −0 python/core/processing/qgsprocessingalgorithm.sip
  3. +22 −0 python/core/processing/qgsprocessingcontext.sip
  4. +2 −0 python/plugins/processing/algs/help/qgis.yaml
  5. +4 −3 python/plugins/processing/algs/qgis/BasicStatistics.py
  6. +15 −2 python/plugins/processing/algs/qgis/CheckValidity.py
  7. +1 −1 python/plugins/processing/algs/qgis/Clip.py
  8. +56 −15 python/plugins/processing/algs/qgis/ExtractByExpression.py
  9. +58 −40 python/plugins/processing/algs/qgis/GridPolygon.py
  10. +26 −13 python/plugins/processing/algs/qgis/Merge.py
  11. +11 −8 python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py
  12. +0 −20 python/plugins/processing/core/GeoAlgorithm.py
  13. +1 −1 python/plugins/processing/core/Processing.py
  14. +1 −1 python/plugins/processing/core/parameters.py
  15. +25 −18 python/plugins/processing/gui/AlgorithmDialog.py
  16. +19 −10 python/plugins/processing/gui/AlgorithmDialogBase.py
  17. +2 −2 python/plugins/processing/gui/BatchAlgorithmDialog.py
  18. +2 −2 python/plugins/processing/gui/DestinationSelectionPanel.py
  19. +8 −8 python/plugins/processing/gui/ExtentSelectionPanel.py
  20. +0 −1 python/plugins/processing/gui/Postprocessing.py
  21. +1 −0 python/plugins/processing/gui/ProcessingToolbox.py
  22. +8 −2 python/plugins/processing/gui/wrappers.py
  23. +2 −2 python/plugins/processing/script/ScriptAlgorithm.py
  24. +114 −114 python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
  25. +5 −0 python/plugins/processing/tools/dataobjects.py
  26. +13 −64 python/plugins/processing/ui/DlgAlgorithmBase.ui
  27. +326 −5 src/core/processing/qgsnativealgorithms.cpp
  28. +46 −0 src/core/processing/qgsnativealgorithms.h
  29. +68 −0 src/core/processing/qgsprocessingalgorithm.cpp
  30. +9 −0 src/core/processing/qgsprocessingalgorithm.h
  31. +36 −0 src/core/processing/qgsprocessingcontext.h
  32. +21 −12 src/core/processing/qgsprocessingparameters.cpp
  33. +2 −0 src/core/processing/qgsprocessingutils.cpp
  34. +1 −0 src/core/processing/qgsprocessingutils.h
  35. +75 −1 tests/src/core/testqgsprocessing.cpp
@@ -33,6 +33,7 @@
import functools
from qgis._core import *


# Boolean evaluation of QgsGeometry


@@ -44,7 +45,8 @@ def _geometryNonZero(self):
QgsGeometry.__bool__ = _geometryNonZero


def register_function(function, arg_count, group, usesgeometry=False, referenced_columns=[QgsFeatureRequest.ALL_ATTRIBUTES], **kwargs):
def register_function(function, arg_count, group, usesgeometry=False,
referenced_columns=[QgsFeatureRequest.ALL_ATTRIBUTES], **kwargs):
"""
Register a Python function to be used as a expression function.
@@ -71,9 +73,11 @@ def myfunc(values, *args):
:param usesgeometry:
:return:
"""

class QgsPyExpressionFunction(QgsExpressionFunction):

def __init__(self, func, name, args, group, helptext='', usesGeometry=True, referencedColumns=QgsFeatureRequest.ALL_ATTRIBUTES, expandargs=False):
def __init__(self, func, name, args, group, helptext='', usesGeometry=True,
referencedColumns=QgsFeatureRequest.ALL_ATTRIBUTES, expandargs=False):
QgsExpressionFunction.__init__(self, name, args, group, helptext)
self.function = func
self.expandargs = expandargs
@@ -126,13 +130,16 @@ def referencedColumns(self, node):
if register and QgsExpression.isFunctionName(name):
if not QgsExpression.unregisterFunction(name):
msgtitle = QCoreApplication.translate("UserExpressions", "User expressions")
msg = QCoreApplication.translate("UserExpressions", "The user expression {0} already exists and could not be unregistered.").format(name)
msg = QCoreApplication.translate("UserExpressions",
"The user expression {0} already exists and could not be unregistered.").format(
name)
QgsMessageLog.logMessage(msg + "\n", msgtitle, QgsMessageLog.WARNING)
return None

function.__name__ = name
helptext = helptemplate.safe_substitute(name=name, doc=helptext)
f = QgsPyExpressionFunction(function, name, arg_count, group, helptext, usesgeometry, referenced_columns, expandargs)
f = QgsPyExpressionFunction(function, name, arg_count, group, helptext, usesgeometry, referenced_columns,
expandargs)

# This doesn't really make any sense here but does when used from a decorator context
# so it can stay.
@@ -163,6 +170,7 @@ def add(values, *args):

def wrapper(func):
return register_function(func, args, group, **kwargs)

return wrapper


@@ -174,6 +182,7 @@ def __init__(self, value):
def __str__(self):
return repr(self.value)


# Define a `with edit(layer)` statement


@@ -264,3 +273,20 @@ def calculation_finished(exception, value=None):


QgsTask.fromFunction = fromFunction


# add some __repr__ methods to processing classes
def processing_source_repr(self):
return "<QgsProcessingFeatureSourceDefinition {{'source':{}, 'selectedFeaturesOnly': {}}}>".format(
self.source.staticValue(), self.selectedFeaturesOnly)


QgsProcessingFeatureSourceDefinition.__repr__ = processing_source_repr


def processing_output_layer_repr(self):
return "<QgsProcessingOutputLayerDefinition {{'sink':{}, 'createOptions': {}}}>".format(self.sink.staticValue(),
self.createOptions)


QgsProcessingOutputLayerDefinition.__repr__ = processing_output_layer_repr
@@ -29,6 +29,7 @@ class QgsProcessingAlgorithm
FlagHideFromModeler,
FlagSupportsBatch,
FlagCanCancel,
FlagRequiresMatchingCrs,
FlagDeprecated,
};
typedef QFlags<QgsProcessingAlgorithm::Flag> Flags;
@@ -244,6 +245,15 @@ class QgsProcessingAlgorithm
:rtype: QgsExpressionContext
%End

virtual bool validateInputCrs( const QVariantMap &parameters,
QgsProcessingContext &context ) const;
%Docstring
Checks whether the coordinate reference systems for the specified set of ``parameters``
are valid for the algorithm. For instance, the base implementation performs
checks to ensure that all input CRS are equal
Returns true if ``parameters`` have passed the CRS check.
:rtype: bool
%End

protected:

@@ -163,6 +163,28 @@ Destination project
%End


void setTransformErrorCallback( SIP_PYCALLABLE / AllowNone / );
%Docstring
Sets a callback function to use when encountering a transform error when iterating
features. This function will be
called using the feature which encountered the transform error as a parameter.
.. versionadded:: 3.0
.. seealso:: transformErrorCallback()
%End
%MethodCode
Py_BEGIN_ALLOW_THREADS

sipCpp->setTransformErrorCallback( [a0]( const QgsFeature &arg )
{
SIP_BLOCK_THREADS
Py_XDECREF( sipCallMethod( NULL, a0, "D", &arg, sipType_QgsFeature, NULL ) );
SIP_UNBLOCK_THREADS
} );

Py_END_ALLOW_THREADS
%End


QString defaultEncoding() const;
%Docstring
Returns the default encoding to use for newly created files.
@@ -314,6 +314,8 @@ qgis:mergevectorlayers: >

If attributes tables are different, the attribute table of the resulting layer will contain the attributes from both input layers.

The layers will all be reprojected to match the coordinate reference system of the first input layer.

qgis:multiparttosingleparts: >
This algorithm takes a vector layer with multipart geometries and generates a new one in which all geometries contain a single part. Features with multipart geometries are divided in as many different features as parts the geometry contain, and the same attributes are used for each of them.

@@ -94,7 +94,7 @@ def __init__(self):
self.tr('Field to calculate statistics on'),
None, self.INPUT_LAYER, QgsProcessingParameterTableField.Any))

self.addParameter(QgsProcessingParameterFileOutput(self.OUTPUT_HTML_FILE, self.tr('Statistics'), self.tr('HTML files (*.html)')))
self.addParameter(QgsProcessingParameterFileOutput(self.OUTPUT_HTML_FILE, self.tr('Statistics'), self.tr('HTML files (*.html)'), None, True))
self.addOutput(QgsProcessingOutputHtml(self.OUTPUT_HTML_FILE, self.tr('Statistics')))

self.addOutput(QgsProcessingOutputNumber(self.COUNT, self.tr('Count')))
@@ -149,9 +149,10 @@ def processAlgorithm(self, parameters, context, feedback):
d, results = self.calcStringStats(features, feedback, field, count)
data.extend(d)

self.createHTML(output_file, data)
if output_file:
self.createHTML(output_file, data)
results[self.OUTPUT_HTML_FILE] = output_file

results[self.OUTPUT_HTML_FILE] = output_file
return results

def calcNumericStats(self, features, feedback, field, count):
@@ -41,7 +41,8 @@
QgsProcessingParameterEnum,
QgsProcessingParameterFeatureSink,
QgsProcessingOutputVectorLayer,
QgsProcessingParameterDefinition
QgsProcessingParameterDefinition,
QgsProcessingOutputNumber
)
from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm

@@ -54,8 +55,11 @@ class CheckValidity(QgisAlgorithm):
INPUT_LAYER = 'INPUT_LAYER'
METHOD = 'METHOD'
VALID_OUTPUT = 'VALID_OUTPUT'
VALID_COUNT = 'VALID_COUNT'
INVALID_OUTPUT = 'INVALID_OUTPUT'
INVALID_COUNT = 'INVALID_COUNT'
ERROR_OUTPUT = 'ERROR_OUTPUT'
ERROR_COUNT = 'ERROR_COUNT'

def icon(self):
return QIcon(os.path.join(pluginPath, 'images', 'ftools', 'check_geometry.png'))
@@ -76,10 +80,15 @@ def __init__(self):

self.addParameter(QgsProcessingParameterFeatureSink(self.VALID_OUTPUT, self.tr('Valid output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
self.addOutput(QgsProcessingOutputVectorLayer(self.VALID_OUTPUT, self.tr('Valid output')))
self.addOutput(QgsProcessingOutputNumber(self.VALID_COUNT, self.tr('Count of valid features')))

self.addParameter(QgsProcessingParameterFeatureSink(self.INVALID_OUTPUT, self.tr('Invalid output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
self.addOutput(QgsProcessingOutputVectorLayer(self.INVALID_OUTPUT, self.tr('Invalid output')))
self.addOutput(QgsProcessingOutputNumber(self.INVALID_COUNT, self.tr('Count of invalid features')))

self.addParameter(QgsProcessingParameterFeatureSink(self.ERROR_OUTPUT, self.tr('Error output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
self.addOutput(QgsProcessingOutputVectorLayer(self.ERROR_OUTPUT, self.tr('Error output')))
self.addOutput(QgsProcessingOutputNumber(self.ERROR_COUNT, self.tr('Count of errors')))

def name(self):
return 'checkvalidity'
@@ -168,7 +177,11 @@ def doCheck(self, method, parameters, context, feedback):

feedback.setProgress(int(current * total))

results = {}
results = {
self.VALID_COUNT: valid_count,
self.INVALID_COUNT: invalid_count,
self.ERROR_COUNT: error_count
}
if valid_output_sink:
results[self.VALID_OUTPUT] = valid_output_dest_id
if invalid_output_sink:
@@ -84,7 +84,7 @@ def processAlgorithm(self, parameters, context, feedback):

# first build up a list of clip geometries
clip_geoms = []
for mask_feature in mask_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([])):
for mask_feature in mask_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(feature_source.sourceCrs())):
clip_geoms.append(mask_feature.geometry())

# are we clipping against a single feature? if so, we can show finer progress reports
@@ -27,7 +27,12 @@
from qgis.core import (QgsExpression,
QgsFeatureRequest,
QgsApplication,
QgsProcessingUtils)
QgsProcessingUtils,
QgsProcessingParameterFeatureSource,
QgsProcessingParameterExpression,
QgsProcessingParameterFeatureSink,
QgsProcessingOutputVectorLayer,
QgsProcessingParameterDefinition)

from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
from processing.core.parameters import ParameterVector
@@ -41,6 +46,7 @@ class ExtractByExpression(QgisAlgorithm):
INPUT = 'INPUT'
EXPRESSION = 'EXPRESSION'
OUTPUT = 'OUTPUT'
FAIL_OUTPUT = 'FAIL_OUTPUT'

def icon(self):
return QgsApplication.getThemeIcon("/providerQgis.svg")
@@ -56,11 +62,16 @@ def group(self):

def __init__(self):
super().__init__()
self.addParameter(ParameterVector(self.INPUT,
self.tr('Input Layer')))
self.addParameter(ParameterExpression(self.EXPRESSION,
self.tr("Expression"), parent_layer=self.INPUT))
self.addOutput(OutputVector(self.OUTPUT, self.tr('Extracted (expression)')))
self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT,
self.tr('Input layer')))
self.addParameter(QgsProcessingParameterExpression(self.EXPRESSION,
self.tr('Expression'), None, self.INPUT))

self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Matching features')))
self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Matching (expression)')))
self.addParameter(QgsProcessingParameterFeatureSink(self.FAIL_OUTPUT, self.tr('Non-matching'),
QgsProcessingParameterDefinition.TypeVectorAny, None, True))
self.addOutput(QgsProcessingOutputVectorLayer(self.FAIL_OUTPUT, self.tr('Non-matching (expression)')))

def name(self):
return 'extractbyexpression'
@@ -69,18 +80,48 @@ def displayName(self):
return self.tr('Extract by expression')

def processAlgorithm(self, parameters, context, feedback):
layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context)
expression_string = self.getParameterValue(self.EXPRESSION)
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
context)
source = self.parameterAsSource(parameters, self.INPUT, context)
expression_string = self.parameterAsExpression(parameters, self.EXPRESSION, context)

(matching_sink, matching_sink_id) = self.parameterAsSink(parameters, self.OUTPUT, context,
source.fields(), source.wkbType(), source.sourceCrs())
(nonmatching_sink, non_matching_sink_id) = self.parameterAsSink(parameters, self.FAIL_OUTPUT, context,
source.fields(), source.wkbType(), source.sourceCrs())

expression = QgsExpression(expression_string)
if not expression.hasParserError():
if expression.hasParserError():
raise GeoAlgorithmExecutionException(expression.parserErrorString())
expression_context = self.createExpressionContext(parameters, context)

if not nonmatching_sink:
# not saving failing features - so only fetch good features
req = QgsFeatureRequest().setFilterExpression(expression_string)
req.setExpressionContext(expression_context)

for f in source.getFeatures(req):
if feedback.isCanceled():
break
matching_sink.addFeature(f)
else:
raise GeoAlgorithmExecutionException(expression.parserErrorString())
# saving non-matching features, so we need EVERYTHING
expression_context.setFields(source.fields())
expression.prepare(expression_context)

total = 100.0 / source.featureCount()

for current, f in enumerate(source.getFeatures()):
if feedback.isCanceled():
break

expression_context.setFeature(f)
if expression.evaluate(expression_context):
matching_sink.addFeature(f)
else:
nonmatching_sink.addFeature(f)

for f in layer.getFeatures(req):
writer.addFeature(f)
feedback.setProgress(int(current * total))

del writer
results = {self.OUTPUT: matching_sink_id}
if nonmatching_sink:
results[self.FAIL_OUTPUT] = non_matching_sink_id
return results

0 comments on commit 40cae29

Please sign in to comment.
You can’t perform that action at this time.