From 16629b406b63607abf38f51ff38c49924081a90e Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:05:59 +1000 Subject: [PATCH 01/23] Register QgsFeatureIds metatype Fixes warnings when using signals which use this type, like QgsVectorLayer::selectionChanged --- src/core/qgsapplication.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/qgsapplication.cpp b/src/core/qgsapplication.cpp index 49dda7879658..2e5f0610ffe4 100644 --- a/src/core/qgsapplication.cpp +++ b/src/core/qgsapplication.cpp @@ -147,6 +147,7 @@ void QgsApplication::init( QString profileFolder ) qRegisterMetaType( "QgsProcessingFeatureSourceDefinition" ); qRegisterMetaType( "QgsProcessingOutputLayerDefinition" ); qRegisterMetaType( "QgsUnitTypes::LayoutUnit" ); + qRegisterMetaType( "QgsFeatureIds" ); QString prefixPath( getenv( "QGIS_PREFIX_PATH" ) ? getenv( "QGIS_PREFIX_PATH" ) : applicationDirPath() ); // QgsDebugMsg( QString( "prefixPath(): %1" ).arg( prefixPath ) ); From a64d199e6f216cdaeb99d2e5cc6b56af93371452 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:07:22 +1000 Subject: [PATCH 02/23] [processing] If an error occurs while running an algorith, always keep the algorithm dialog open after execution Otherwise it's hard to see the error - you have to know to check the python log. Keeping the dialog open at the log makes the error immediately visible to the user --- python/plugins/processing/gui/AlgorithmDialog.py | 8 ++++---- python/plugins/processing/gui/AlgorithmDialogBase.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/plugins/processing/gui/AlgorithmDialog.py b/python/plugins/processing/gui/AlgorithmDialog.py index 4c7d96c14ba6..fad8255f9c09 100644 --- a/python/plugins/processing/gui/AlgorithmDialog.py +++ b/python/plugins/processing/gui/AlgorithmDialog.py @@ -229,7 +229,7 @@ def accept(self): feedback.pushInfo( self.tr('Execution completed in {0:0.2f} seconds'.format(time.time() - start_time))) self.buttonCancel.setEnabled(False) - self.finish(parameters, context, feedback) + self.finish(True, parameters, context, feedback) else: self.buttonCancel.setEnabled(False) self.resetGUI() @@ -250,7 +250,7 @@ def on_complete(ok, results): feedback.pushInfo('') self.buttonCancel.setEnabled(False) - self.finish(results, context, feedback) + self.finish(ok, results, context, feedback) task = QgsProcessingAlgRunnerTask(self.alg, parameters, context, feedback) task.executed.connect(on_complete) @@ -269,8 +269,8 @@ def on_complete(ok, results): self.bar.pushMessage("", self.tr("Wrong or missing parameter value: {0}").format(e.parameter.description()), level=QgsMessageBar.WARNING, duration=5) - def finish(self, result, context, feedback): - keepOpen = ProcessingConfig.getSetting(ProcessingConfig.KEEP_DIALOG_OPEN) + def finish(self, successful, result, context, feedback): + keepOpen = not successful or ProcessingConfig.getSetting(ProcessingConfig.KEEP_DIALOG_OPEN) if self.iterateParam is None: diff --git a/python/plugins/processing/gui/AlgorithmDialogBase.py b/python/plugins/processing/gui/AlgorithmDialogBase.py index a73ba82daa18..f34f51d95025 100644 --- a/python/plugins/processing/gui/AlgorithmDialogBase.py +++ b/python/plugins/processing/gui/AlgorithmDialogBase.py @@ -238,7 +238,7 @@ def reject(self): self._saveGeometry() super(AlgorithmDialogBase, self).reject() - def finish(self, context, feedback): + def finish(self, successful, result, context, feedback): pass def toggleCollapsed(self): From 7ab82444f1108fba8a26629821a13138a8b50a72 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:09:47 +1000 Subject: [PATCH 03/23] Port random selection algorithms to new API And heavily optimise random selection within subsets alg --- .../algs/qgis/QGISAlgorithmProvider.py | 7 +- .../processing/algs/qgis/RandomSelection.py | 45 ++++---- .../algs/qgis/RandomSelectionWithinSubsets.py | 102 +++++++++--------- 3 files changed, 75 insertions(+), 79 deletions(-) diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 13c8704e4394..6615f66f4c77 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -99,6 +99,8 @@ from .RandomPointsExtent import RandomPointsExtent from .RandomPointsLayer import RandomPointsLayer from .RandomPointsPolygons import RandomPointsPolygons +from .RandomSelection import RandomSelection +from .RandomSelectionWithinSubsets import RandomSelectionWithinSubsets from .RasterLayerStatistics import RasterLayerStatistics from .RegularPoints import RegularPoints from .ReverseLineDirection import ReverseLineDirection @@ -135,8 +137,6 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation -# from .RandomSelection import RandomSelection -# from .RandomSelectionWithinSubsets import RandomSelectionWithinSubsets # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin # from .GridLine import GridLine @@ -185,7 +185,6 @@ def __init__(self): def getAlgs(self): # algs = [ - # RandomSelection(), RandomSelectionWithinSubsets(), # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), @@ -270,6 +269,8 @@ def getAlgs(self): RandomPointsExtent(), RandomPointsLayer(), RandomPointsPolygons(), + RandomSelection(), + RandomSelectionWithinSubsets(), RasterLayerStatistics(), RegularPoints(), ReverseLineDirection(), diff --git a/python/plugins/processing/algs/qgis/RandomSelection.py b/python/plugins/processing/algs/qgis/RandomSelection.py index 167c3c6729f3..31af20a29f85 100644 --- a/python/plugins/processing/algs/qgis/RandomSelection.py +++ b/python/plugins/processing/algs/qgis/RandomSelection.py @@ -30,13 +30,16 @@ import random from qgis.PyQt.QtGui import QIcon -from qgis.core import QgsFeatureSink, QgsProcessingUtils +from qgis.core import (QgsFeatureSink, + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessingParameterNumber, + QgsProcessingParameterFeatureSink, + QgsProcessingOutputVectorLayer) + from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterSelection -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.outputs import OutputVector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -61,13 +64,14 @@ def initAlgorithm(self, config=None): self.methods = [self.tr('Number of selected features'), self.tr('Percentage of selected features')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'))) - self.addParameter(ParameterSelection(self.METHOD, - self.tr('Method'), self.methods, 0)) - self.addParameter(ParameterNumber(self.NUMBER, - self.tr('Number/percentage of selected features'), 0, None, 10)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Selection'), True)) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterEnum(self.METHOD, + self.tr('Method'), self.methods, False, 0)) + self.addParameter(QgsProcessingParameterNumber(self.NUMBER, + self.tr('Number/percentage of selected features'), QgsProcessingParameterNumber.Integer, + 10, False, 0.0, 999999999999.0)) + self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Selected (random)'))) def name(self): return 'randomselection' @@ -76,23 +80,20 @@ def displayName(self): return self.tr('Random selection') def processAlgorithm(self, parameters, context, feedback): - filename = self.getParameterValue(self.INPUT) - layer = QgsProcessingUtils.mapLayerFromString(filename, context) - method = self.getParameterValue(self.METHOD) + layer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + method = self.parameterAsEnum(parameters, self.METHOD, context) featureCount = layer.featureCount() - value = int(self.getParameterValue(self.NUMBER)) - - layer.removeSelection() + value = self.parameterAsInt(parameters, self.NUMBER, context) if method == 0: if value > featureCount: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Selected number is greater than feature count. ' 'Choose a lower value and try again.')) else: if value > 100: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr("Percentage can't be greater than 100. Set a " "different value and try again.")) value = int(round(value / 100.0, 4) * featureCount) @@ -100,4 +101,4 @@ def processAlgorithm(self, parameters, context, feedback): selran = random.sample(list(range(featureCount)), value) layer.selectByIds(selran) - self.setOutputValue(self.OUTPUT, filename) + return {self.OUTPUT: parameters[self.INPUT]} diff --git a/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py b/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py index 6086c31efcfa..c72390a49764 100644 --- a/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py +++ b/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py @@ -31,15 +31,17 @@ from qgis.PyQt.QtGui import QIcon -from qgis.core import QgsFeature, QgsFeatureSink, QgsProcessingUtils - +from qgis.core import (QgsFeatureRequest, + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessingParameterField, + QgsProcessingParameterNumber, + QgsProcessingParameterFeatureSink, + QgsProcessingOutputVectorLayer) +from collections import defaultdict from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterSelection -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -65,16 +67,17 @@ def initAlgorithm(self, config=None): self.methods = [self.tr('Number of selected features'), self.tr('Percentage of selected features')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('ID Field'), self.INPUT)) - self.addParameter(ParameterSelection(self.METHOD, - self.tr('Method'), self.methods, 0)) - self.addParameter(ParameterNumber(self.NUMBER, - self.tr('Number/percentage of selected features'), 1, None, 10)) - - self.addOutput(OutputVector(self.OUTPUT, self.tr('Selection stratified'), True)) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('ID field'), None, self.INPUT)) + self.addParameter(QgsProcessingParameterEnum(self.METHOD, + self.tr('Method'), self.methods, False, 0)) + self.addParameter(QgsProcessingParameterNumber(self.NUMBER, + self.tr('Number/percentage of selected features'), + QgsProcessingParameterNumber.Integer, + 10, False, 0.0, 999999999999.0)) + self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Selected (stratified random)'))) def name(self): return 'randomselectionwithinsubsets' @@ -83,61 +86,52 @@ def displayName(self): return self.tr('Random selection within subsets') def processAlgorithm(self, parameters, context, feedback): - filename = self.getParameterValue(self.INPUT) - - layer = QgsProcessingUtils.mapLayerFromString(filename, context) - field = self.getParameterValue(self.FIELD) - method = self.getParameterValue(self.METHOD) + layer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + method = self.parameterAsEnum(parameters, self.METHOD, context) + field = self.parameterAsString(parameters, self.FIELD, context) - layer.removeSelection() index = layer.fields().lookupField(field) - unique = QgsProcessingUtils.uniqueValues(layer, index, context) + unique = layer.uniqueValues(index) featureCount = layer.featureCount() - value = int(self.getParameterValue(self.NUMBER)) + value = self.parameterAsInt(parameters, self.NUMBER, context) if method == 0: if value > featureCount: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Selected number is greater that feature count. ' 'Choose lesser value and try again.')) else: if value > 100: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr("Percentage can't be greater than 100. Set a " "different value and try again.")) value = value / 100.0 - selran = [] - inFeat = QgsFeature() - - current = 0 total = 100.0 / (featureCount * len(unique)) if featureCount else 1 if not len(unique) == featureCount: - for i in unique: - features = QgsProcessingUtils.getFeatures(layer, context) - FIDs = [] - for inFeat in features: - attrs = inFeat.attributes() - if attrs[index] == i: - FIDs.append(inFeat.id()) - current += 1 - feedback.setProgress(int(current * total)) - - if method == 1: - selValue = int(round(value * len(FIDs), 0)) - else: - selValue = value - - if selValue >= len(FIDs): - selFeat = FIDs - else: - selFeat = random.sample(FIDs, selValue) - - selran.extend(selFeat) + classes = defaultdict(list) + + features = layer.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry).setSubsetOfAttributes([index])) + + for i, feature in enumerate(features): + if feedback.isCanceled(): + break + + classes[feature.attributes()[index]].append(feature.id()) + feedback.setProgress(int(i * total)) + + selran = [] + for subset in classes.values(): + if feedback.isCanceled(): + break + + selValue = value if method != 1 else int(round(value * len(subset), 0)) + selran.extend(random.sample(subset, selValue)) + layer.selectByIds(selran) else: layer.selectByIds(list(range(featureCount))) # FIXME: implies continuous feature ids - self.setOutputValue(self.OUTPUT, filename) + return {self.OUTPUT: parameters[self.INPUT]} From 54be72048516896d9022da4fa24369d5ac4aeb0c Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:22:30 +1000 Subject: [PATCH 04/23] Port grid lines to new API --- .../plugins/processing/algs/qgis/GridLine.py | 81 +++++++++-------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../tests/testdata/qgis_algorithm_tests.yaml | 88 +++++++++---------- 3 files changed, 93 insertions(+), 81 deletions(-) diff --git a/python/plugins/processing/algs/qgis/GridLine.py b/python/plugins/processing/algs/qgis/GridLine.py index bcf5cabae4d3..3bb87b8d9f59 100644 --- a/python/plugins/processing/algs/qgis/GridLine.py +++ b/python/plugins/processing/algs/qgis/GridLine.py @@ -39,14 +39,15 @@ QgsPoint, QgsLineString, QgsWkbTypes, + QgsProcessing, + QgsProcessingException, + QgsProcessingParameterEnum, + QgsProcessingParameterExtent, + QgsProcessingParameterNumber, + QgsProcessingParameterCrs, + QgsProcessingParameterFeatureSink, QgsFields) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterExtent -from processing.core.parameters import ParameterNumber -from processing.core.parameters import ParameterCrs -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -73,19 +74,24 @@ def __init__(self): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterExtent(self.EXTENT, - self.tr('Grid extent'), optional=False)) - self.addParameter(ParameterNumber(self.HSPACING, - self.tr('Horizontal spacing'), 0.0, 1000000000.0, default=0.0001)) - self.addParameter(ParameterNumber(self.VSPACING, - self.tr('Vertical spacing'), 0.0, 1000000000.0, default=0.0001)) - self.addParameter(ParameterNumber(self.HOVERLAY, - self.tr('Horizontal overlay'), 0.0, 1000000000.0, default=0.0)) - self.addParameter(ParameterNumber(self.VOVERLAY, - self.tr('Vertical overlay'), 0.0, 1000000000.0, default=0.0)) - self.addParameter(ParameterCrs(self.CRS, 'Grid CRS', 'EPSG:4326')) - - self.addOutput(OutputVector(self.OUTPUT, self.tr('Grid'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + self.addParameter(QgsProcessingParameterExtent(self.EXTENT, self.tr('Grid extent'))) + + self.addParameter(QgsProcessingParameterNumber(self.HSPACING, + self.tr('Horizontal spacing'), QgsProcessingParameterNumber.Double, + 0.0001, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.VSPACING, + self.tr('Vertical spacing'), QgsProcessingParameterNumber.Double, + 0.0001, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.HOVERLAY, + self.tr('Horizontal overlay'), QgsProcessingParameterNumber.Double, + 0.0, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.VOVERLAY, + self.tr('Vertical overlay'), QgsProcessingParameterNumber.Double, + 0.0, False, 0, 1000000000.0)) + + self.addParameter(QgsProcessingParameterCrs(self.CRS, 'Grid CRS', 'ProjectCrs')) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Grid'), type=QgsProcessing.TypeVectorLine)) def name(self): return 'creategridlines' @@ -94,33 +100,31 @@ def displayName(self): return self.tr('Create grid (lines)') def processAlgorithm(self, parameters, context, feedback): - extent = self.getParameterValue(self.EXTENT).split(',') - hSpacing = self.getParameterValue(self.HSPACING) - vSpacing = self.getParameterValue(self.VSPACING) - hOverlay = self.getParameterValue(self.HOVERLAY) - vOverlay = self.getParameterValue(self.VOVERLAY) - crs = QgsCoordinateReferenceSystem(self.getParameterValue(self.CRS)) + hSpacing = self.parameterAsDouble(parameters, self.HSPACING, context) + vSpacing = self.parameterAsDouble(parameters, self.VSPACING, context) + hOverlay = self.parameterAsDouble(parameters, self.HOVERLAY, context) + vOverlay = self.parameterAsDouble(parameters, self.VOVERLAY, context) - bbox = QgsRectangle(float(extent[0]), float(extent[2]), - float(extent[1]), float(extent[3])) + bbox = self.parameterAsExtent(parameters, self.EXTENT, context) + crs = self.parameterAsCrs(parameters, self.CRS, context) width = bbox.width() height = bbox.height() if hSpacing <= 0 or vSpacing <= 0: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Invalid grid spacing: {0}/{1}').format(hSpacing, vSpacing)) if hSpacing <= hOverlay or vSpacing <= vOverlay: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Invalid overlay: {0}/{1}').format(hOverlay, vOverlay)) if width < hSpacing: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Horizontal spacing is too small for the covered area')) if height < vSpacing: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Vertical spacing is too small for the covered area')) fields = QgsFields() @@ -131,7 +135,8 @@ def processAlgorithm(self, parameters, context, feedback): fields.append(QgsField('id', QVariant.Int, '', 10, 0)) fields.append(QgsField('coord', QVariant.Double, '', 24, 15)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.LineString, crs, context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, crs) if hOverlay > 0: hSpace = [hSpacing - hOverlay, hOverlay] @@ -154,6 +159,9 @@ def processAlgorithm(self, parameters, context, feedback): count_update = count_max * 0.10 y = bbox.yMaximum() while y >= bbox.yMinimum(): + if feedback.isCanceled(): + break + pt1 = QgsPoint(bbox.xMinimum(), y) pt2 = QgsPoint(bbox.xMaximum(), y) line = QgsLineString() @@ -165,7 +173,7 @@ def processAlgorithm(self, parameters, context, feedback): y, id, y]) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) y = y - vSpace[count % 2] id += 1 count += 1 @@ -181,6 +189,9 @@ def processAlgorithm(self, parameters, context, feedback): count_update = count_max * 0.10 x = bbox.xMinimum() while x <= bbox.xMaximum(): + if feedback.isCanceled(): + break + pt1 = QgsPoint(x, bbox.yMaximum()) pt2 = QgsPoint(x, bbox.yMinimum()) line = QgsLineString() @@ -192,11 +203,11 @@ def processAlgorithm(self, parameters, context, feedback): bbox.yMinimum(), id, x]) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) x = x + hSpace[count % 2] id += 1 count += 1 if int(math.fmod(count, count_update)) == 0: feedback.setProgress(50 + int(count / count_max * 50)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 6615f66f4c77..53397648311c 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -70,6 +70,7 @@ from .FixedDistanceBuffer import FixedDistanceBuffer from .FixGeometry import FixGeometry from .GeometryByExpression import GeometryByExpression +from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade @@ -139,7 +140,6 @@ # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .GridLine import GridLine # from .Gridify import Gridify # from .HubDistancePoints import HubDistancePoints # from .HubDistanceLines import HubDistanceLines @@ -188,7 +188,7 @@ def getAlgs(self): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # GridLine(), Gridify(), HubDistancePoints(), + # Gridify(), HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -240,6 +240,7 @@ def getAlgs(self): FixedDistanceBuffer(), FixGeometry(), GeometryByExpression(), + GridLine(), GridPolygon(), Heatmap(), Hillshade(), diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 1c73df8b9086..64b881dd059e 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -1795,21 +1795,21 @@ tests: name: expected/dropped_geometry.csv type: vector -# - algorithm: qgis:creategridlines -# name: Create grid (lines) -# params: -# CRS: EPSG:4326 -# EXTENT: -1,11.2,-4,6.5 -# HSPACING: 5.0 -# VSPACING: 3.0 -# results: -# OUTPUT: -# name: expected/grid_lines.gml -# type: vector -# compare: -# geometry: -# precision: 7 -# + - algorithm: qgis:creategridlines + name: Create grid (lines) + params: + CRS: EPSG:4326 + EXTENT: -1,11.2,-4,6.5 + HSPACING: 5.0 + VSPACING: 3.0 + results: + OUTPUT: + name: expected/grid_lines.gml + type: vector + compare: + geometry: + precision: 7 + - algorithm: qgis:creategridpolygon name: Create grid (rectangles) params: @@ -1858,20 +1858,20 @@ tests: geometry: precision: 7 -# - algorithm: qgis:creategridlines -# name: Create grid (lines with overlay) -# params: -# CRS: EPSG:4326 -# EXTENT: -1,11.2,-4,6.5 -# HOVERLAY: 2.0 -# HSPACING: 5.0 -# VOVERLAY: 1.0 -# VSPACING: 3.0 -# results: -# OUTPUT: -# name: expected/grid_lines_overlay.gml -# type: vector -# + - algorithm: qgis:creategridlines + name: Create grid (lines with overlay) + params: + CRS: EPSG:4326 + EXTENT: -1,11.2,-4,6.5 + HOVERLAY: 2.0 + HSPACING: 5.0 + VOVERLAY: 1.0 + VSPACING: 3.0 + results: + OUTPUT: + name: expected/grid_lines_overlay.gml + type: vector + - algorithm: qgis:creategridpolygon name: Create grid (rectangle with overlay) params: @@ -2586,21 +2586,21 @@ tests: # OUTPUT_LAYER: # name: expected/buffer_ovals.gml # type: vector -# -# - algorithm: qgis:creategridlines -# name: Lines grid 0.1 degree spacing -# params: -# CRS: EPSG:4326 -# EXTENT: -0.10453905405405395,8.808021567567568,-2.5010055337837844,4.058021763513514 -# HOVERLAY: 0.0 -# HSPACING: 0.1 -# VOVERLAY: 0.0 -# VSPACING: 0.1 -# results: -# OUTPUT: -# name: expected/create_grid_lines.gml -# type: vector -# + + - algorithm: qgis:creategridlines + name: Lines grid 0.1 degree spacing + params: + CRS: EPSG:4326 + EXTENT: -0.10453905405405395,8.808021567567568,-2.5010055337837844,4.058021763513514 + HOVERLAY: 0.0 + HSPACING: 0.1 + VOVERLAY: 0.0 + VSPACING: 0.1 + results: + OUTPUT: + name: expected/create_grid_lines.gml + type: vector + # - algorithm: qgis:convertgeometrytype # name: polygon to centroid # params: From 591de92b071e72466c6b78225ae1652256c5e009 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 00:33:27 +1000 Subject: [PATCH 05/23] Port gridify to new API --- .../plugins/processing/algs/qgis/Gridify.py | 92 ++++++++----------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- 2 files changed, 43 insertions(+), 54 deletions(-) diff --git a/python/plugins/processing/algs/qgis/Gridify.py b/python/plugins/processing/algs/qgis/Gridify.py index df782427b3aa..2605ccd9cd0c 100644 --- a/python/plugins/processing/algs/qgis/Gridify.py +++ b/python/plugins/processing/algs/qgis/Gridify.py @@ -31,36 +31,29 @@ QgsPointXY, QgsWkbTypes, QgsApplication, - QgsMessageLog, - QgsProcessingUtils) -from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.outputs import OutputVector + QgsProcessingException, + QgsProcessingParameterNumber) +from processing.algs.qgis.QgisAlgorithm import QgisFeatureBasedAlgorithm -class Gridify(QgisAlgorithm): - INPUT = 'INPUT' +class Gridify(QgisFeatureBasedAlgorithm): + HSPACING = 'HSPACING' VSPACING = 'VSPACING' - OUTPUT = 'OUTPUT' def group(self): return self.tr('Vector general tools') def __init__(self): super().__init__() + self.h_spacing = None + self.v_spacing = None - def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input Layer'))) - self.addParameter(ParameterNumber(self.HSPACING, - self.tr('Horizontal spacing'), default=0.1)) - self.addParameter(ParameterNumber(self.VSPACING, - self.tr('Vertical spacing'), default=0.1)) - - self.addOutput(OutputVector(self.OUTPUT, self.tr('Snapped'))) + def initParameters(self, config=None): + self.addParameter(QgsProcessingParameterNumber(self.HSPACING, + self.tr('Horizontal spacing'), type=QgsProcessingParameterNumber.Double, minValue=0.0, defaultValue=0.1)) + self.addParameter(QgsProcessingParameterNumber(self.VSPACING, + self.tr('Vertical spacing'), type=QgsProcessingParameterNumber.Double, minValue=0.0, defaultValue=0.1)) def name(self): return 'snappointstogrid' @@ -68,46 +61,45 @@ def name(self): def displayName(self): return self.tr('Snap points to grid') - def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context) - hSpacing = self.getParameterValue(self.HSPACING) - vSpacing = self.getParameterValue(self.VSPACING) - - if hSpacing <= 0 or vSpacing <= 0: - raise GeoAlgorithmExecutionException( - self.tr('Invalid grid spacing: {0}/{1}').format(hSpacing, vSpacing)) + def outputName(self): + return self.tr('Snapped') - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(), - context) + def prepareAlgorithm(self, parameters, context, feedback): + self.h_spacing = self.parameterAsDouble(parameters, self.HSPACING, context) + self.v_spacing = self.parameterAsDouble(parameters, self.VSPACING, context) + if self.h_spacing <= 0 or self.v_spacing <= 0: + raise QgsProcessingException( + self.tr('Invalid grid spacing: {0}/{1}').format(self.h_spacing, self.v_spacing)) - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + return True - for current, f in enumerate(features): - geom = f.geometry() - geomType = geom.wkbType() + def processFeature(self, feature, feedback): + if feature.hasGeometry(): + geom = feature.geometry() + geomType = QgsWkbTypes.flatType(geom.wkbType()) + newGeom = None if geomType == QgsWkbTypes.Point: - points = self._gridify([geom.asPoint()], hSpacing, vSpacing) + points = self._gridify([geom.asPoint()], self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromPoint(points[0]) elif geomType == QgsWkbTypes.MultiPoint: - points = self._gridify(geom.aMultiPoint(), hSpacing, vSpacing) + points = self._gridify(geom.aMultiPoint(), self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromMultiPoint(points) elif geomType == QgsWkbTypes.LineString: - points = self._gridify(geom.asPolyline(), hSpacing, vSpacing) + points = self._gridify(geom.asPolyline(), self.h_spacing, self.v_spacing) if len(points) < 2: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromPolyline(points) elif geomType == QgsWkbTypes.MultiLineString: polyline = [] for line in geom.asMultiPolyline(): - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 1: polyline.append(points) if len(polyline) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromMultiPolyline(polyline) @@ -115,11 +107,11 @@ def processAlgorithm(self, parameters, context, feedback): elif geomType == QgsWkbTypes.Polygon: polygon = [] for line in geom.asPolygon(): - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 1: polygon.append(points) if len(polygon) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromPolygon(polygon) @@ -128,7 +120,7 @@ def processAlgorithm(self, parameters, context, feedback): for polygon in geom.asMultiPolygon(): newPolygon = [] for line in polygon: - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 2: newPolygon.append(points) @@ -136,20 +128,16 @@ def processAlgorithm(self, parameters, context, feedback): multipolygon.append(newPolygon) if len(multipolygon) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromMultiPolygon(multipolygon) if newGeom is not None: - feat = QgsFeature() - feat.setGeometry(newGeom) - feat.setAttributes(f.attributes()) - writer.addFeature(feat, QgsFeatureSink.FastInsert) - - feedback.setProgress(int(current * total)) - - del writer + feature.setGeometry(newGeom) + else: + feature.clearGeometry() + return feature def _gridify(self, points, hSpacing, vSpacing): nPoints = [] diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 53397648311c..97d113597dc8 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -70,6 +70,7 @@ from .FixedDistanceBuffer import FixedDistanceBuffer from .FixGeometry import FixGeometry from .GeometryByExpression import GeometryByExpression +from .Gridify import Gridify from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap @@ -140,7 +141,6 @@ # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .Gridify import Gridify # from .HubDistancePoints import HubDistancePoints # from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines @@ -188,7 +188,7 @@ def getAlgs(self): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # Gridify(), HubDistancePoints(), + # HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -240,6 +240,7 @@ def getAlgs(self): FixedDistanceBuffer(), FixGeometry(), GeometryByExpression(), + Gridify(), GridLine(), GridPolygon(), Heatmap(), From 0930e18bf929bd6487382c1c89ce8800299f9eff Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 01:26:42 +1000 Subject: [PATCH 06/23] Add tests for gridify --- .../tests/testdata/expected/gridify_lines.gfs | 16 ++++++ .../tests/testdata/expected/gridify_lines.gml | 46 +++++++++++++++ .../tests/testdata/expected/gridify_polys.gfs | 32 +++++++++++ .../tests/testdata/expected/gridify_polys.gml | 57 +++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 26 +++++++++ 5 files changed, 177 insertions(+) create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_lines.gml create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_polys.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_polys.gml diff --git a/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs b/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs new file mode 100644 index 000000000000..ae929a271369 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs @@ -0,0 +1,16 @@ + + + gridify_lines + gridify_lines + + 2 + EPSG:4326 + + 7 + 2.00000 + 12.00000 + -4.00000 + 4.00000 + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_lines.gml b/python/plugins/processing/tests/testdata/expected/gridify_lines.gml new file mode 100644 index 000000000000..961702eaa01f --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_lines.gml @@ -0,0 +1,46 @@ + + + + + 2-4 + 124 + + + + + + 6,2 8,2 8,4 12,4 + + + + + + + + + 2,0 2,2 4,2 4,4 + + + + + + + + + 8,-4 10,-4 + + + + + 6,-4 10,0 + + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs b/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs new file mode 100644 index 000000000000..cc44b1a90478 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs @@ -0,0 +1,32 @@ + + + gridify_polys + gridify_polys + + 3 + EPSG:4326 + + 6 + 0.00000 + 10.00000 + -4.00000 + 6.00000 + + + name + name + String + 5 + + + intval + intval + Integer + + + floatval + floatval + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_polys.gml b/python/plugins/processing/tests/testdata/expected/gridify_polys.gml new file mode 100644 index 000000000000..c255726ba4e7 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_polys.gml @@ -0,0 +1,57 @@ + + + + + -0-4 + 106 + + + + + + 0,0 0,4 4,4 4,2 2,2 2,0 0,0 + aaaaa + 33 + 44.123456 + + + + + Aaaaa + -33 + 0 + + + + + 2,4 2,6 4,6 4,4 2,4 + bbaaa + 0.123 + + + + + 6,0 10,0 10,-4 6,-4 6,0 + ASDF + 0 + + + + + 120 + -100291.43213 + + + + + 4,2 6,0 6,-4 2,0 2,2 4,2 + elim + 2 + 3.33 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 64b881dd059e..ac3d7d2929a6 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2158,6 +2158,32 @@ tests: name: expected/lines_to_polygon.gml type: vector + - algorithm: qgis:snappointstogrid + name: Gridify polys + params: + INPUT: + name: polys.gml + type: vector + HSPACING: 2 + VSPACING: 2 + results: + OUTPUT: + name: expected/gridify_polys.gml + type: vector + + - algorithm: qgis:snappointstogrid + name: Gridify lines + params: + INPUT: + name: lines.gml + type: vector + HSPACING: 2 + VSPACING: 2 + results: + OUTPUT: + name: expected/gridify_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From fc1746e7706e35ff54a75c664071a2c86732386d Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 01:34:28 +1000 Subject: [PATCH 07/23] Port Hub Distance (points) to new API Improvements: - handle different CRS between points and hubs - add unit test --- .../processing/algs/qgis/HubDistancePoints.py | 111 +++++++++--------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../tests/testdata/custom/hub_points.gfs | 22 ++++ .../tests/testdata/custom/hub_points.gml | 32 +++++ .../testdata/expected/hub_distance_points.gfs | 37 ++++++ .../testdata/expected/hub_distance_points.gml | 95 +++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 7 files changed, 260 insertions(+), 58 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/custom/hub_points.gfs create mode 100644 python/plugins/processing/tests/testdata/custom/hub_points.gml create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_points.gml diff --git a/python/plugins/processing/algs/qgis/HubDistancePoints.py b/python/plugins/processing/algs/qgis/HubDistancePoints.py index ea670b39b4d2..32effadfc14a 100644 --- a/python/plugins/processing/algs/qgis/HubDistancePoints.py +++ b/python/plugins/processing/algs/qgis/HubDistancePoints.py @@ -33,34 +33,32 @@ QgsDistanceArea, QgsFeature, QgsFeatureRequest, + QgsSpatialIndex, QgsWkbTypes, - QgsApplication, - QgsProject, - QgsProcessingUtils) + QgsUnitTypes, + QgsProcessing, + QgsProcessingUtils, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink, + QgsProcessingException) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects - -from math import sqrt class HubDistancePoints(QgisAlgorithm): - POINTS = 'POINTS' + INPUT = 'INPUT' HUBS = 'HUBS' FIELD = 'FIELD' UNIT = 'UNIT' OUTPUT = 'OUTPUT' + LAYER_UNITS = 'LAYER_UNITS' - UNITS = ['Meters', - 'Feet', - 'Miles', - 'Kilometers', - 'Layer units' + UNITS = [QgsUnitTypes.DistanceMeters, + QgsUnitTypes.DistanceFeet, + QgsUnitTypes.DistanceMiles, + QgsUnitTypes.DistanceKilometers, + LAYER_UNITS ] def group(self): @@ -76,16 +74,16 @@ def initAlgorithm(self, config=None): self.tr('Kilometers'), self.tr('Layer units')] - self.addParameter(ParameterVector(self.POINTS, - self.tr('Source points layer'))) - self.addParameter(ParameterVector(self.HUBS, - self.tr('Destination hubs layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('Hub layer name attribute'), self.HUBS)) - self.addParameter(ParameterSelection(self.UNIT, - self.tr('Measurement unit'), self.units)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Source points layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Destination hubs layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Hub layer name attribute'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterEnum(self.UNIT, + self.tr('Measurement unit'), self.units)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub distance'), datatype=[dataobjects.TYPE_VECTOR_POINT])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub distance'), QgsProcessing.TypeVectorPoint)) def name(self): return 'distancetonearesthubpoints' @@ -94,61 +92,62 @@ def displayName(self): return self.tr('Distance to nearest hub (points)') def processAlgorithm(self, parameters, context, feedback): - layerPoints = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.POINTS), context) - layerHubs = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - fieldName = self.getParameterValue(self.FIELD) + if parameters[self.INPUT] == parameters[self.HUBS]: + raise QgsProcessingException( + self.tr('Same layer given for both hubs and spokes')) - units = self.UNITS[self.getParameterValue(self.UNIT)] + point_source = self.parameterAsSource(parameters, self.INPUT, context) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + fieldName = self.parameterAsString(parameters, self.FIELD, context) - if layerPoints.source() == layerHubs.source(): - raise GeoAlgorithmExecutionException( - self.tr('Same layer given for both hubs and spokes')) + units = self.UNITS[self.parameterAsEnum(parameters, self.UNIT, context)] - fields = layerPoints.fields() + fields = point_source.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Point, layerPoints.crs(), - context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.Point, point_source.sourceCrs()) - index = QgsProcessingUtils.createSpatialIndex(layerHubs, context) + index = QgsSpatialIndex(hub_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(point_source.sourceCrs()))) distance = QgsDistanceArea() - distance.setSourceCrs(layerPoints.crs()) + distance.setSourceCrs(point_source.sourceCrs()) distance.setEllipsoid(context.project().ellipsoid()) # Scan source points, find nearest hub, and write to output file - features = QgsProcessingUtils.getFeatures(layerPoints, context) - total = 100.0 / layerPoints.featureCount() if layerPoints.featureCount() else 0 + features = point_source.getFeatures() + total = 100.0 / point_source.featureCount() if point_source.featureCount() else 0 for current, f in enumerate(features): + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + sink.addFeature(f, QgsFeatureSink.FastInsert) + continue + src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) - ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields()))) + ft = next(hub_source.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], hub_source.fields()).setDestinationCrs(point_source.sourceCrs()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) + if units != self.LAYER_UNITS: + hub_dist_in_desired_units = distance.convertLengthMeasurement(hubDist, units) + else: + hub_dist_in_desired_units = hubDist + attributes = f.attributes() attributes.append(ft[fieldName]) - if units == 'Feet': - attributes.append(hubDist * 3.2808399) - elif units == 'Miles': - attributes.append(hubDist * 0.000621371192) - elif units == 'Kilometers': - attributes.append(hubDist / 1000.0) - elif units != 'Meters': - attributes.append(sqrt( - pow(src.x() - closest.x(), 2.0) + - pow(src.y() - closest.y(), 2.0))) - else: - attributes.append(hubDist) + attributes.append(hub_dist_in_desired_units) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPoint(src)) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 97d113597dc8..d58960a57708 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -75,6 +75,7 @@ from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade +from .HubDistancePoints import HubDistancePoints from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection @@ -141,7 +142,7 @@ # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .HubDistancePoints import HubDistancePoints + # from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines # from .GeometryConvert import GeometryConvert @@ -188,7 +189,6 @@ def getAlgs(self): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -245,6 +245,7 @@ def getAlgs(self): GridPolygon(), Heatmap(), Hillshade(), + HubDistancePoints(), ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), diff --git a/python/plugins/processing/tests/testdata/custom/hub_points.gfs b/python/plugins/processing/tests/testdata/custom/hub_points.gfs new file mode 100644 index 000000000000..35cbfd91664e --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hub_points.gfs @@ -0,0 +1,22 @@ + + + hub_points + hub_points + + 1 + EPSG:4326 + + 3 + 1.34481 + 6.29897 + -1.25947 + 2.27221 + + + name + name + String + 6 + + + diff --git a/python/plugins/processing/tests/testdata/custom/hub_points.gml b/python/plugins/processing/tests/testdata/custom/hub_points.gml new file mode 100644 index 000000000000..99d14c1d02af --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hub_points.gml @@ -0,0 +1,32 @@ + + + + + 1.344807662693645-1.259467184083282 + 6.2989682466352512.272211648033507 + + + + + + 1.34480766269365,-1.25946718408328 + point1 + + + + + 6.29896824663525,0.138489020296281 + point2 + + + + + 3.12290985247467,2.27221164803351 + point3 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs new file mode 100644 index 000000000000..43674410adb9 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs @@ -0,0 +1,37 @@ + + + hub_distance_points + hub_distance_points + + 1 + EPSG:4326 + + 9 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id + id + Integer + + + id2 + id2 + Integer + + + HubName + HubName + String + 6 + + + HubDist + HubDist + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml new file mode 100644 index 000000000000..4e767e8b117b --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml @@ -0,0 +1,95 @@ + + + + + 0-5 + 83 + + + + + + 1,1 + 1 + 2 + point1 + 254434.675423572 + + + + + 3,3 + 2 + 1 + point3 + 82164.2455422206 + + + + + 2,2 + 3 + 0 + point3 + 128622.227687308 + + + + + 5,2 + 4 + 2 + point3 + 211142.486929284 + + + + + 4,1 + 5 + 1 + point3 + 172016.876891364 + + + + + 0,-5 + 6 + 0 + point1 + 442487.532089586 + + + + + 8,-1 + 7 + 0 + point2 + 227856.24000978 + + + + + 7,-1 + 8 + 0 + point2 + 148835.564980152 + + + + + 0,-1 + 9 + 0 + point1 + 152464.26003518 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index ac3d7d2929a6..e79b77c2cd3d 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2184,6 +2184,22 @@ tests: name: expected/gridify_lines.gml type: vector + - algorithm: qgis:distancetonearesthubpoints + name: Hub distance points + params: + INPUT: + name: points.gml + type: vector + HUBS: + name: custom/hub_points.gml + type: vector + FIELD: name + UNIT: 0 + results: + OUTPUT: + name: expected/hub_distance_points.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From e0354456e3240d7b2498547727c5ffc27c7610cd Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 18:55:30 +1000 Subject: [PATCH 08/23] Port Hub Distance (lines) to new API Improvements: - handle different CRS between points and hubs - add unit test --- .../processing/algs/qgis/HubDistanceLines.py | 108 +++++++++--------- .../processing/algs/qgis/HubDistancePoints.py | 1 - .../algs/qgis/QGISAlgorithmProvider.py | 6 +- .../testdata/expected/hub_distance_lines.gfs | 37 ++++++ .../testdata/expected/hub_distance_lines.gml | 95 +++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 6 files changed, 205 insertions(+), 58 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml diff --git a/python/plugins/processing/algs/qgis/HubDistanceLines.py b/python/plugins/processing/algs/qgis/HubDistanceLines.py index bb360b5a9715..491a82a7ff9e 100644 --- a/python/plugins/processing/algs/qgis/HubDistanceLines.py +++ b/python/plugins/processing/algs/qgis/HubDistanceLines.py @@ -34,33 +34,33 @@ QgsFeatureSink, QgsFeatureRequest, QgsWkbTypes, - QgsApplication, - QgsProject, - QgsProcessingUtils) + QgsUnitTypes, + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink, + QgsProcessingException, + QgsSpatialIndex) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects from math import sqrt class HubDistanceLines(QgisAlgorithm): - POINTS = 'POINTS' + INPUT = 'INPUT' HUBS = 'HUBS' FIELD = 'FIELD' UNIT = 'UNIT' OUTPUT = 'OUTPUT' - UNITS = ['Meters', - 'Feet', - 'Miles', - 'Kilometers', - 'Layer units' + LAYER_UNITS = 'LAYER_UNITS' + + UNITS = [QgsUnitTypes.DistanceMeters, + QgsUnitTypes.DistanceFeet, + QgsUnitTypes.DistanceMiles, + QgsUnitTypes.DistanceKilometers, + LAYER_UNITS ] def group(self): @@ -76,16 +76,16 @@ def initAlgorithm(self, config=None): self.tr('Kilometers'), self.tr('Layer units')] - self.addParameter(ParameterVector(self.POINTS, - self.tr('Source points layer'))) - self.addParameter(ParameterVector(self.HUBS, - self.tr('Destination hubs layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('Hub layer name attribute'), self.HUBS)) - self.addParameter(ParameterSelection(self.UNIT, - self.tr('Measurement unit'), self.units)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Source points layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Destination hubs layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Hub layer name attribute'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterEnum(self.UNIT, + self.tr('Measurement unit'), self.units)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub distance'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub distance'), QgsProcessing.TypeVectorLine)) def name(self): return 'distancetonearesthublinetohub' @@ -94,61 +94,61 @@ def displayName(self): return self.tr('Distance to nearest hub (line to hub)') def processAlgorithm(self, parameters, context, feedback): - layerPoints = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.POINTS), context) - layerHubs = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - fieldName = self.getParameterValue(self.FIELD) + if parameters[self.INPUT] == parameters[self.HUBS]: + raise QgsProcessingException( + self.tr('Same layer given for both hubs and spokes')) - units = self.UNITS[self.getParameterValue(self.UNIT)] + point_source = self.parameterAsSource(parameters, self.INPUT, context) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + fieldName = self.parameterAsString(parameters, self.FIELD, context) - if layerPoints.source() == layerHubs.source(): - raise GeoAlgorithmExecutionException( - self.tr('Same layer given for both hubs and spokes')) + units = self.UNITS[self.parameterAsEnum(parameters, self.UNIT, context)] - fields = layerPoints.fields() + fields = point_source.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.LineString, layerPoints.crs(), - context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, point_source.sourceCrs()) - index = QgsProcessingUtils.createSpatialIndex(layerHubs, context) + index = QgsSpatialIndex(hub_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(point_source.sourceCrs()))) distance = QgsDistanceArea() - distance.setSourceCrs(layerPoints.crs()) + distance.setSourceCrs(point_source.sourceCrs()) distance.setEllipsoid(context.project().ellipsoid()) # Scan source points, find nearest hub, and write to output file - features = QgsProcessingUtils.getFeatures(layerPoints, context) - total = 100.0 / layerPoints.featureCount() if layerPoints.featureCount() else 0 + features = point_source.getFeatures() + total = 100.0 / point_source.featureCount() if point_source.featureCount() else 0 for current, f in enumerate(features): + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + sink.addFeature(f, QgsFeatureSink.FastInsert) + continue src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) - ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields()))) + ft = next(hub_source.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], hub_source.fields()).setDestinationCrs(point_source.sourceCrs()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) + if units != self.LAYER_UNITS: + hub_dist_in_desired_units = distance.convertLengthMeasurement(hubDist, units) + else: + hub_dist_in_desired_units = hubDist + attributes = f.attributes() attributes.append(ft[fieldName]) - if units == 'Feet': - attributes.append(hubDist * 3.2808399) - elif units == 'Miles': - attributes.append(hubDist * 0.000621371192) - elif units == 'Kilometers': - attributes.append(hubDist / 1000.0) - elif units != 'Meters': - attributes.append(sqrt( - pow(src.x() - closest.x(), 2.0) + - pow(src.y() - closest.y(), 2.0))) - else: - attributes.append(hubDist) + attributes.append(hub_dist_in_desired_units) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPolyline([src, closest])) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/HubDistancePoints.py b/python/plugins/processing/algs/qgis/HubDistancePoints.py index 32effadfc14a..7ec153557e6a 100644 --- a/python/plugins/processing/algs/qgis/HubDistancePoints.py +++ b/python/plugins/processing/algs/qgis/HubDistancePoints.py @@ -37,7 +37,6 @@ QgsWkbTypes, QgsUnitTypes, QgsProcessing, - QgsProcessingUtils, QgsProcessingParameterFeatureSource, QgsProcessingParameterField, QgsProcessingParameterEnum, diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index d58960a57708..01f2f57d236b 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -75,6 +75,7 @@ from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade +from .HubDistanceLines import HubDistanceLines from .HubDistancePoints import HubDistancePoints from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite @@ -142,8 +143,6 @@ # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin - -# from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines # from .GeometryConvert import GeometryConvert # from .StatisticsByCategories import StatisticsByCategories @@ -189,7 +188,7 @@ def getAlgs(self): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubDistanceLines(), HubLines(), + # HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), # FieldsPyculator(), @@ -245,6 +244,7 @@ def getAlgs(self): GridPolygon(), Heatmap(), Hillshade(), + HubDistanceLines(), HubDistancePoints(), ImportIntoPostGIS(), ImportIntoSpatialite(), diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs new file mode 100644 index 000000000000..b5b6e761f237 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs @@ -0,0 +1,37 @@ + + + hub_distance_lines + hub_distance_lines + + 2 + EPSG:4326 + + 9 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id + id + Integer + + + id2 + id2 + Integer + + + HubName + HubName + String + 6 + + + HubDist + HubDist + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml new file mode 100644 index 000000000000..275b34f7a0bb --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml @@ -0,0 +1,95 @@ + + + + + 0-5 + 83 + + + + + + 1,1 1.34480766269365,-1.25946718408328 + 1 + 2 + point1 + 254434.675423572 + + + + + 3,3 3.12290985247467,2.27221164803351 + 2 + 1 + point3 + 82164.2455422206 + + + + + 2,2 3.12290985247467,2.27221164803351 + 3 + 0 + point3 + 128622.227687308 + + + + + 5,2 3.12290985247467,2.27221164803351 + 4 + 2 + point3 + 211142.486929284 + + + + + 4,1 3.12290985247467,2.27221164803351 + 5 + 1 + point3 + 172016.876891364 + + + + + 0,-5 1.34480766269365,-1.25946718408328 + 6 + 0 + point1 + 442487.532089586 + + + + + 8,-1 6.29896824663525,0.138489020296281 + 7 + 0 + point2 + 227856.24000978 + + + + + 7,-1 6.29896824663525,0.138489020296281 + 8 + 0 + point2 + 148835.564980152 + + + + + 0,-1 1.34480766269365,-1.25946718408328 + 9 + 0 + point1 + 152464.26003518 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index e79b77c2cd3d..0b232b7d61a9 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2200,6 +2200,22 @@ tests: name: expected/hub_distance_points.gml type: vector + - algorithm: qgis:distancetonearesthublinetohub + name: Hub distance lines + params: + INPUT: + name: points.gml + type: vector + HUBS: + name: custom/hub_points.gml + type: vector + FIELD: name + UNIT: 0 + results: + OUTPUT: + name: expected/hub_distance_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From b4b39996d23cfeae09991f3234f0c221cc9af657 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 19:49:50 +1000 Subject: [PATCH 09/23] Port hub lines algorithm to new API Improvements: - transparent reprojection to match hub/spoke CRS - keep all attributes from matched hub/spoke features - don't break after matching one hub point to spoke - instead join ALL hub/spoke points with matching id values --- python/plugins/processing/algs/help/qgis.yaml | 2 + .../plugins/processing/algs/qgis/HubLines.py | 113 +++++++++++------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- .../tests/testdata/custom/spoke_points.gfs | 27 +++++ .../tests/testdata/custom/spoke_points.gml | 63 ++++++++++ .../tests/testdata/expected/hub_lines.gfs | 43 +++++++ .../tests/testdata/expected/hub_lines.gml | 84 +++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 8 files changed, 304 insertions(+), 48 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/custom/spoke_points.gfs create mode 100644 python/plugins/processing/tests/testdata/custom/spoke_points.gml create mode 100644 python/plugins/processing/tests/testdata/expected/hub_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_lines.gml diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml index f6756f0fd42d..ba5ed67e8e1a 100755 --- a/python/plugins/processing/algs/help/qgis.yaml +++ b/python/plugins/processing/algs/help/qgis.yaml @@ -242,7 +242,9 @@ qgis:generatepointspixelcentroidsinsidepolygons: qgis:hublines: + This algorithm creates hub and spoke diagrams with lines drawn from points on the Spoke Point layer to matching points in the Hub Point layer. + Determination of which hub goes with each point is based on a match between the Hub ID field on the hub points and the Spoke ID field on the spoke points. qgis:hypsometriccurves: > This algorithm computes hypsometric curves for an input Digital Elevation Model. Curves are produced as table files in an output folder specified by the user. diff --git a/python/plugins/processing/algs/qgis/HubLines.py b/python/plugins/processing/algs/qgis/HubLines.py index b42112a8304d..707780100373 100644 --- a/python/plugins/processing/algs/qgis/HubLines.py +++ b/python/plugins/processing/algs/qgis/HubLines.py @@ -31,15 +31,15 @@ QgsGeometry, QgsPointXY, QgsWkbTypes, - QgsApplication, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink, + QgsProcessingException, + QgsExpression) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects +from processing.tools import vector class HubLines(QgisAlgorithm): @@ -55,17 +55,21 @@ def group(self): def __init__(self): super().__init__() + def tags(self): + return self.tr('join,points,lines,connect,hub,spoke').split(',') + def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.HUBS, - self.tr('Hub layer'))) - self.addParameter(ParameterTableField(self.HUB_FIELD, - self.tr('Hub ID field'), self.HUBS)) - self.addParameter(ParameterVector(self.SPOKES, - self.tr('Spoke layer'))) - self.addParameter(ParameterTableField(self.SPOKE_FIELD, - self.tr('Spoke ID field'), self.SPOKES)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub lines'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Hub layer'))) + self.addParameter(QgsProcessingParameterField(self.HUB_FIELD, + self.tr('Hub ID field'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterFeatureSource(self.SPOKES, + self.tr('Spoke layer'))) + self.addParameter(QgsProcessingParameterField(self.SPOKE_FIELD, + self.tr('Spoke ID field'), parentLayerParameterName=self.SPOKES)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub lines'), QgsProcessing.TypeVectorLine)) def name(self): return 'hublines' @@ -74,44 +78,61 @@ def displayName(self): return self.tr('Hub lines') def processAlgorithm(self, parameters, context, feedback): - layerHub = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - layerSpoke = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.SPOKES), context) + if parameters[self.SPOKES] == parameters[self.HUBS]: + raise QgsProcessingException( + self.tr('Same layer given for both hubs and spokes')) - fieldHub = self.getParameterValue(self.HUB_FIELD) - fieldSpoke = self.getParameterValue(self.SPOKE_FIELD) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + spoke_source = self.parameterAsSource(parameters, self.SPOKES, context) + field_hub = self.parameterAsString(parameters, self.HUB_FIELD, context) + field_hub_index = hub_source.fields().lookupField(field_hub) + field_spoke = self.parameterAsString(parameters, self.SPOKE_FIELD, context) + field_spoke_index = hub_source.fields().lookupField(field_spoke) - if layerHub.source() == layerSpoke.source(): - raise GeoAlgorithmExecutionException( - self.tr('Same layer given for both hubs and spokes')) + fields = vector.combineFields(hub_source.fields(), spoke_source.fields()) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layerSpoke.fields(), QgsWkbTypes.LineString, - layerSpoke.crs(), context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, hub_source.sourceCrs()) - spokes = QgsProcessingUtils.getFeatures(layerSpoke, context) - hubs = QgsProcessingUtils.getFeatures(layerHub, context) - total = 100.0 / layerSpoke.featureCount() if layerSpoke.featureCount() else 0 + hubs = hub_source.getFeatures() + total = 100.0 / hub_source.featureCount() if hub_source.featureCount() else 0 - for current, spokepoint in enumerate(spokes): - p = spokepoint.geometry().boundingBox().center() - spokeX = p.x() - spokeY = p.y() - spokeId = str(spokepoint[fieldSpoke]) + matching_field_types = hub_source.fields().at(field_hub_index).type() == spoke_source.fields().at(field_spoke_index).type() - for hubpoint in hubs: - hubId = str(hubpoint[fieldHub]) - if hubId == spokeId: - p = hubpoint.geometry().boundingBox().center() - hubX = p.x() - hubY = p.y() + for current, hub_point in enumerate(hubs): + if feedback.isCanceled(): + break - f = QgsFeature() - f.setAttributes(spokepoint.attributes()) - f.setGeometry(QgsGeometry.fromPolyline( - [QgsPointXY(spokeX, spokeY), QgsPointXY(hubX, hubY)])) - writer.addFeature(f, QgsFeatureSink.FastInsert) + if not hub_point.hasGeometry(): + continue + p = hub_point.geometry().boundingBox().center() + hub_x = p.x() + hub_y = p.y() + hub_id = str(hub_point[field_hub]) + hub_attributes = hub_point.attributes() + + request = QgsFeatureRequest().setDestinationCrs(hub_source.sourceCrs()) + if matching_field_types: + request.setFilterExpression(QgsExpression.createFieldEqualityExpression(field_spoke, hub_attributes[field_hub_index])) + + spokes = spoke_source.getFeatures() + for spoke_point in spokes: + if feedback.isCanceled(): break + spoke_id = str(spoke_point[field_spoke]) + if hub_id == spoke_id: + p = spoke_point.geometry().boundingBox().center() + spoke_x = p.x() + spoke_y = p.y() + + f = QgsFeature() + f.setAttributes(hub_attributes + spoke_point.attributes()) + f.setGeometry(QgsGeometry.fromPolyline( + [QgsPointXY(hub_x, hub_y), QgsPointXY(spoke_x, spoke_y)])) + sink.addFeature(f, QgsFeatureSink.FastInsert) + feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 01f2f57d236b..e6da61cf3431 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -77,6 +77,7 @@ from .Hillshade import Hillshade from .HubDistanceLines import HubDistanceLines from .HubDistancePoints import HubDistancePoints +from .HubLines import HubLines from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection @@ -143,7 +144,6 @@ # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .HubLines import HubLines # from .GeometryConvert import GeometryConvert # from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator @@ -188,7 +188,6 @@ def getAlgs(self): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), # FieldsPyculator(), @@ -246,6 +245,7 @@ def getAlgs(self): Hillshade(), HubDistanceLines(), HubDistancePoints(), + HubLines(), ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), diff --git a/python/plugins/processing/tests/testdata/custom/spoke_points.gfs b/python/plugins/processing/tests/testdata/custom/spoke_points.gfs new file mode 100644 index 000000000000..5166cf9a29f7 --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/spoke_points.gfs @@ -0,0 +1,27 @@ + + + spoke_points + spoke_points + + 1 + EPSG:4326 + + 7 + 1.27875 + 6.82625 + -4.16750 + 3.88250 + + + id + id + Integer + + + name + name + String + 8 + + + diff --git a/python/plugins/processing/tests/testdata/custom/spoke_points.gml b/python/plugins/processing/tests/testdata/custom/spoke_points.gml new file mode 100644 index 000000000000..3896f58f81a4 --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/spoke_points.gml @@ -0,0 +1,63 @@ + + + + + 1.27875-4.1675 + 6.8262499999999993.882499999999999 + + + + + + 5.07625,-2.1725 + 1 + point 1 + + + + + 5.82,3.8825 + 2 + point 2 + + + + + 1.62,1.4675 + 3 + point 3 + + + + + 6.68625,1.23125 + 4 + point 4 + + + + + 1.27875,-3.66875 + 4 + point 4a + + + + + 3.81625,-4.1675 + 4 + point 4b + + + + + 6.82625,-2.79375 + 8 + point 8 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_lines.gfs b/python/plugins/processing/tests/testdata/expected/hub_lines.gfs new file mode 100644 index 000000000000..553c3cbf06cc --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_lines.gfs @@ -0,0 +1,43 @@ + + + hub_lines + hub_lines + + 2 + EPSG:4326 + + 7 + 1.00000 + 7.00000 + -4.16750 + 3.88250 + + + id + id + Integer + + + id2 + id2 + Integer + + + fid_2 + fid_2 + String + 14 + + + id_2 + id_2 + Integer + + + name + name + String + 8 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_lines.gml b/python/plugins/processing/tests/testdata/expected/hub_lines.gml new file mode 100644 index 000000000000..b530cce06418 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_lines.gml @@ -0,0 +1,84 @@ + + + + + 1-4.1675 + 73.8825 + + + + + + 1,1 5.07625,-2.1725 + 1 + 2 + spoke_points.0 + 1 + point 1 + + + + + 3,3 5.82,3.8825 + 2 + 1 + spoke_points.1 + 2 + point 2 + + + + + 2,2 1.62,1.4675 + 3 + 0 + spoke_points.2 + 3 + point 3 + + + + + 5,2 6.68625,1.23125 + 4 + 2 + spoke_points.3 + 4 + point 4 + + + + + 5,2 1.27875,-3.66875 + 4 + 2 + spoke_points.4 + 4 + point 4a + + + + + 5,2 3.81625,-4.1675 + 4 + 2 + spoke_points.5 + 4 + point 4b + + + + + 7,-1 6.82625,-2.79375 + 8 + 0 + spoke_points.6 + 8 + point 8 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 0b232b7d61a9..18fd8566b399 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2216,6 +2216,22 @@ tests: name: expected/hub_distance_lines.gml type: vector + - algorithm: qgis:hublines + name: Hub lines + params: + HUBS: + name: points.gml + type: vector + SPOKES: + name: custom/spoke_points.gml + type: vector + HUB_FIELD: id + SPOKE_FIELD: id + results: + OUTPUT: + name: expected/hub_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From ec4df6c019efddc05e9555c43d58a155b811d078 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 23:26:39 +1000 Subject: [PATCH 10/23] Port points to path to new API Improvements: - Maintain Z/M values - Keep original data type for group/order fields - Group field is optional - Added unit tests - Don't export text files for features by default --- python/plugins/processing/algs/help/qgis.yaml | 2 + .../processing/algs/qgis/PointsToPaths.py | 187 ++++++++++-------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../testdata/expected/points_to_path.gfs | 26 +++ .../testdata/expected/points_to_path.gml | 21 ++ .../expected/points_to_path_grouped.gfs | 31 +++ .../expected/points_to_path_grouped.gml | 38 ++++ .../tests/testdata/qgis_algorithm_tests.yaml | 25 +++ 8 files changed, 254 insertions(+), 81 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path.gml create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml index ba5ed67e8e1a..a60b0af4a272 100755 --- a/python/plugins/processing/algs/help/qgis.yaml +++ b/python/plugins/processing/algs/help/qgis.yaml @@ -370,7 +370,9 @@ qgis:pointslayerfromtable: > The attributes table of the resulting layer will be the input table. qgis:pointstopath: + Converts a point layer to a line layer, by joining points in a defined order. + Points can be grouped by a field to output individual line features per group. qgis:polarplot: > This algorithm generates a polar plot based on the value of an input vector layer. diff --git a/python/plugins/processing/algs/qgis/PointsToPaths.py b/python/plugins/processing/algs/qgis/PointsToPaths.py index e73c57a9fd56..df2105b33c0b 100644 --- a/python/plugins/processing/algs/qgis/PointsToPaths.py +++ b/python/plugins/processing/algs/qgis/PointsToPaths.py @@ -29,36 +29,34 @@ import os from datetime import datetime -from qgis.PyQt.QtCore import QVariant -from qgis.core import (QgsApplication, - QgsFeature, +from qgis.core import (QgsFeature, QgsFeatureSink, QgsFields, QgsField, QgsGeometry, QgsDistanceArea, - QgsProject, + QgsPointXY, + QgsLineString, QgsWkbTypes, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterString, + QgsProcessing, + QgsProcessingParameterFeatureSink, + QgsProcessingParameterFolderDestination) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterString -from processing.core.outputs import OutputVector -from processing.core.outputs import OutputDirectory -from processing.tools import dataobjects class PointsToPaths(QgisAlgorithm): - VECTOR = 'VECTOR' + INPUT = 'INPUT' GROUP_FIELD = 'GROUP_FIELD' ORDER_FIELD = 'ORDER_FIELD' DATE_FORMAT = 'DATE_FORMAT' - #GAP_PERIOD = 'GAP_PERIOD' - OUTPUT_LINES = 'OUTPUT_LINES' - OUTPUT_TEXT = 'OUTPUT_TEXT' + OUTPUT = 'OUTPUT' + OUTPUT_TEXT_DIR = 'OUTPUT_TEXT_DIR' def group(self): return self.tr('Vector creation tools') @@ -66,20 +64,23 @@ def group(self): def __init__(self): super().__init__() + def tags(self): + return self.tr('join,points,lines,connect').split(',') + def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.VECTOR, - self.tr('Input point layer'), [dataobjects.TYPE_VECTOR_POINT])) - self.addParameter(ParameterTableField(self.GROUP_FIELD, - self.tr('Group field'), self.VECTOR)) - self.addParameter(ParameterTableField(self.ORDER_FIELD, - self.tr('Order field'), self.VECTOR)) - self.addParameter(ParameterString(self.DATE_FORMAT, - self.tr('Date format (if order field is DateTime)'), '', optional=True)) - #self.addParameter(ParameterNumber( - # self.GAP_PERIOD, - # 'Gap period (if order field is DateTime)', 0, 60, 0)) - self.addOutput(OutputVector(self.OUTPUT_LINES, self.tr('Paths'), datatype=[dataobjects.TYPE_VECTOR_LINE])) - self.addOutput(OutputDirectory(self.OUTPUT_TEXT, self.tr('Directory'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input point layer'), [QgsProcessing.TypeVectorPoint])) + self.addParameter(QgsProcessingParameterField(self.ORDER_FIELD, + self.tr('Order field'), parentLayerParameterName=self.INPUT)) + self.addParameter(QgsProcessingParameterField(self.GROUP_FIELD, + self.tr('Group field'), parentLayerParameterName=self.INPUT, optional=True)) + self.addParameter(QgsProcessingParameterString(self.DATE_FORMAT, + self.tr('Date format (if order field is DateTime)'), optional=True)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Paths'), QgsProcessing.TypeVectorLine)) + output_dir_param = QgsProcessingParameterFolderDestination(self.OUTPUT_TEXT_DIR, self.tr('Directory for text output'), optional=True) + output_dir_param.setCreateByDefault(False) + self.addParameter(output_dir_param) def name(self): return 'pointstopath' @@ -88,29 +89,58 @@ def displayName(self): return self.tr('Points to path') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.VECTOR), context) - groupField = self.getParameterValue(self.GROUP_FIELD) - orderField = self.getParameterValue(self.ORDER_FIELD) - dateFormat = str(self.getParameterValue(self.DATE_FORMAT)) - #gap = int(self.getParameterValue(self.GAP_PERIOD)) - dirName = self.getOutputValue(self.OUTPUT_TEXT) + source = self.parameterAsSource(parameters, self.INPUT, context) + group_field_name = self.parameterAsString(parameters, self.GROUP_FIELD, context) + order_field_name = self.parameterAsString(parameters, self.ORDER_FIELD, context) + date_format = self.parameterAsString(parameters, self.DATE_FORMAT, context) + text_dir = self.parameterAsString(parameters, self.OUTPUT_TEXT_DIR, context) + + group_field_index = source.fields().lookupField(group_field_name) + order_field_index = source.fields().lookupField(order_field_name) + + if group_field_index >= 0: + group_field_def = source.fields().at(group_field_index) + else: + group_field_def = None + order_field_def = source.fields().at(order_field_index) fields = QgsFields() - fields.append(QgsField('group', QVariant.String, '', 254, 0)) - fields.append(QgsField('begin', QVariant.String, '', 254, 0)) - fields.append(QgsField('end', QVariant.String, '', 254, 0)) - writer = self.getOutputFromName(self.OUTPUT_LINES).getVectorWriter(fields, QgsWkbTypes.LineString, layer.crs(), - context) + if group_field_def is not None: + fields.append(group_field_def) + begin_field = QgsField(order_field_def) + begin_field.setName('begin') + fields.append(begin_field) + end_field = QgsField(order_field_def) + end_field.setName('end') + fields.append(end_field) + + output_wkb = QgsWkbTypes.LineString + if QgsWkbTypes.hasM(source.wkbType()): + output_wkb = QgsWkbTypes.addM(output_wkb) + if QgsWkbTypes.hasZ(source.wkbType()): + output_wkb = QgsWkbTypes.addZ(output_wkb) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, output_wkb, source.sourceCrs()) points = dict() - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([group_field_index, order_field_index])) + total = 100.0 / source.featureCount() if source.featureCount() else 0 for current, f in enumerate(features): - point = f.geometry().asPoint() - group = f[groupField] - order = f[orderField] - if dateFormat != '': - order = datetime.strptime(str(order), dateFormat) + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + continue + + point = f.geometry().geometry().clone() + if group_field_index >= 0: + group = f.attributes()[group_field_index] + else: + group = 1 + order = f.attributes()[order_field_index] + if date_format != '': + order = datetime.strptime(str(order), date_format) if group in points: points[group].append((order, point)) else: @@ -121,46 +151,45 @@ def processAlgorithm(self, parameters, context, feedback): feedback.setProgress(0) da = QgsDistanceArea() - da.setSourceCrs(layer.sourceCrs()) + da.setSourceCrs(source.sourceCrs()) da.setEllipsoid(context.project().ellipsoid()) current = 0 total = 100.0 / len(points) if points else 1 for group, vertices in list(points.items()): + if feedback.isCanceled(): + break + vertices.sort() f = QgsFeature() - f.initAttributes(len(fields)) - f.setFields(fields) - f['group'] = group - f['begin'] = vertices[0][0] - f['end'] = vertices[-1][0] - - fileName = os.path.join(dirName, '%s.txt' % group) - - with open(fileName, 'w') as fl: - fl.write('angle=Azimuth\n') - fl.write('heading=Coordinate_System\n') - fl.write('dist_units=Default\n') - - line = [] - i = 0 - for node in vertices: - line.append(node[1]) - - if i == 0: - fl.write('startAt=%f;%f;90\n' % (node[1].x(), node[1].y())) - fl.write('survey=Polygonal\n') - fl.write('[data]\n') - else: - angle = line[i - 1].azimuth(line[i]) - distance = da.measureLine(line[i - 1], line[i]) - fl.write('%f;%f;90\n' % (angle, distance)) - - i += 1 - - f.setGeometry(QgsGeometry.fromPolyline(line)) - writer.addFeature(f, QgsFeatureSink.FastInsert) + attributes = [] + if group_field_index >= 0: + attributes.append(group) + attributes.extend([vertices[0][0], vertices[-1][0]]) + f.setAttributes(attributes) + line = [node[1] for node in vertices] + + if text_dir: + fileName = os.path.join(text_dir, '%s.txt' % group) + + with open(fileName, 'w') as fl: + fl.write('angle=Azimuth\n') + fl.write('heading=Coordinate_System\n') + fl.write('dist_units=Default\n') + + for i in range(len(line)): + if i == 0: + fl.write('startAt=%f;%f;90\n' % (line[i].x(), line[i].y())) + fl.write('survey=Polygonal\n') + fl.write('[data]\n') + else: + angle = line[i - 1].azimuth(line[i]) + distance = da.measureLine(QgsPointXY(line[i - 1]), QgsPointXY(line[i])) + fl.write('%f;%f;90\n' % (angle, distance)) + + f.setGeometry(QgsGeometry(QgsLineString(line))) + sink.addFeature(f, QgsFeatureSink.FastInsert) current += 1 feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index e6da61cf3431..c9f11fabbd27 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -94,6 +94,7 @@ from .PointsAlongGeometry import PointsAlongGeometry from .PointsInPolygon import PointsInPolygon from .PointsLayerFromTable import PointsLayerFromTable +from .PointsToPaths import PointsToPaths from .PoleOfInaccessibility import PoleOfInaccessibility from .Polygonize import Polygonize from .PolygonsToLines import PolygonsToLines @@ -152,7 +153,6 @@ # from .PointsDisplacement import PointsDisplacement # from .PointsFromPolygons import PointsFromPolygons # from .PointsFromLines import PointsFromLines -# from .PointsToPaths import PointsToPaths # from .SetVectorStyle import SetVectorStyle # from .SetRasterStyle import SetRasterStyle # from .SelectByAttributeSum import SelectByAttributeSum @@ -194,7 +194,7 @@ def getAlgs(self): # StatisticsByCategories(), # RasterLayerStatistics(), PointsDisplacement(), # PointsFromPolygons(), - # PointsFromLines(), PointsToPaths(), + # PointsFromLines(), # SetVectorStyle(), SetRasterStyle(), # HypsometricCurves(), # FieldsMapper(), SelectByAttributeSum(), Datasources2Vrt(), @@ -262,6 +262,7 @@ def getAlgs(self): PointsAlongGeometry(), PointsInPolygon(), PointsLayerFromTable(), + PointsToPaths(), PoleOfInaccessibility(), Polygonize(), PolygonsToLines(), diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path.gfs new file mode 100644 index 000000000000..89a8bf0693f9 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path.gfs @@ -0,0 +1,26 @@ + + + points_to_path + points_to_path + + 2 + EPSG:4326 + + 1 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path.gml b/python/plugins/processing/tests/testdata/expected/points_to_path.gml new file mode 100644 index 000000000000..5057497c90d5 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path.gml @@ -0,0 +1,21 @@ + + + + + 0-5 + 83 + + + + + + 1,1 3,3 2,2 5,2 4,1 0,-5 8,-1 7,-1 0,-1 + 1 + 9 + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs new file mode 100644 index 000000000000..076d497b0960 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs @@ -0,0 +1,31 @@ + + + points_to_path_grouped + points_to_path_grouped + + 2 + EPSG:4326 + + 3 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id2 + id2 + Integer + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml new file mode 100644 index 000000000000..e82344ba8efb --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml @@ -0,0 +1,38 @@ + + + + + 0-5 + 83 + + + + + + 1,1 5,2 + 2 + 1 + 4 + + + + + 3,3 4,1 + 1 + 2 + 5 + + + + + 2,2 0,-5 8,-1 7,-1 0,-1 + 0 + 3 + 9 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 18fd8566b399..a0adf00e4325 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2232,6 +2232,31 @@ tests: name: expected/hub_lines.gml type: vector + - algorithm: qgis:pointstopath + name: Points to path (non grouped) + params: + INPUT: + name: points.gml + type: vector + ORDER_FIELD: id + results: + OUTPUT: + name: expected/points_to_path.gml + type: vector + + - algorithm: qgis:pointstopath + name: Points to path (grouped) + params: + INPUT: + name: points.gml + type: vector + ORDER_FIELD: id + GROUP_FIELD: id2 + results: + OUTPUT: + name: expected/points_to_path_grouped.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From 7132faa97470866baf09759bb17a2ecc005c5680 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 00:03:41 +1000 Subject: [PATCH 11/23] Port Topocolor algorithm to new API --- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../processing/algs/qgis/TopoColors.py | 73 +++++++++++-------- .../testdata/expected/topocolor_polys.gml | 4 +- .../tests/testdata/qgis_algorithm_tests.yaml | 53 +++++++------- 4 files changed, 76 insertions(+), 59 deletions(-) diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index c9f11fabbd27..e1e1cc7e47a5 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -133,6 +133,7 @@ from .SumLines import SumLines from .SymmetricalDifference import SymmetricalDifference from .TextToFloat import TextToFloat +from .TopoColors import TopoColor from .Translate import Translate from .TruncateTable import TruncateTable from .Union import Union @@ -169,7 +170,6 @@ # from .RasterCalculator import RasterCalculator # from .ExecuteSQL import ExecuteSQL # from .FindProjection import FindProjection -# from .TopoColors import TopoColor # from .EliminateSelection import EliminateSelection pluginPath = os.path.normpath(os.path.join( @@ -206,7 +206,7 @@ def getAlgs(self): # IdwInterpolation(), TinInterpolation(), # RasterCalculator(), # ExecuteSQL(), FindProjection(), - # TopoColor(), EliminateSelection() + # EliminateSelection() # ] algs = [AddTableField(), Aspect(), @@ -301,6 +301,7 @@ def getAlgs(self): SumLines(), SymmetricalDifference(), TextToFloat(), + TopoColor(), Translate(), TruncateTable(), Union(), diff --git a/python/plugins/processing/algs/qgis/TopoColors.py b/python/plugins/processing/algs/qgis/TopoColors.py index b50625b4e75e..33c078cce050 100644 --- a/python/plugins/processing/algs/qgis/TopoColors.py +++ b/python/plugins/processing/algs/qgis/TopoColors.py @@ -31,33 +31,31 @@ from collections import defaultdict -from qgis.core import (QgsApplication, - QgsField, +from qgis.core import (QgsField, QgsFeatureSink, QgsGeometry, QgsSpatialIndex, QgsPointXY, NULL, - QgsProcessingUtils) + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterNumber, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink) from qgis.PyQt.QtCore import (QVariant) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import (ParameterVector, - ParameterSelection, - ParameterNumber) -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] class TopoColor(QgisAlgorithm): - INPUT_LAYER = 'INPUT_LAYER' + INPUT = 'INPUT' MIN_COLORS = 'MIN_COLORS' MIN_DISTANCE = 'MIN_DISTANCE' BALANCE = 'BALANCE' - OUTPUT_LAYER = 'OUTPUT_LAYER' + OUTPUT = 'OUTPUT' def tags(self): return self.tr('topocolor,colors,graph,adjacent,assign').split(',') @@ -69,21 +67,23 @@ def __init__(self): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input layer'), [dataobjects.TYPE_VECTOR_POLYGON])) - self.addParameter(ParameterNumber(self.MIN_COLORS, - self.tr('Minimum number of colors'), 1, 1000, 4)) - self.addParameter(ParameterNumber(self.MIN_DISTANCE, - self.tr('Minimum distance between features'), 0.0, 999999999.0, 0.0)) + + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input layer'), [QgsProcessing.TypeVectorPolygon])) + self.addParameter(QgsProcessingParameterNumber(self.MIN_COLORS, + self.tr('Minimum number of colors'), minValue=1, maxValue=1000, defaultValue=4)) + self.addParameter(QgsProcessingParameterNumber(self.MIN_DISTANCE, + self.tr('Minimum distance between features'), type=QgsProcessingParameterNumber.Double, + minValue=0.0, maxValue=999999999.0, defaultValue=0.0)) balance_by = [self.tr('By feature count'), self.tr('By assigned area'), self.tr('By distance between colors')] - self.addParameter(ParameterSelection( + self.addParameter(QgsProcessingParameterEnum( self.BALANCE, self.tr('Balance color assignment'), - balance_by, default=0)) + options=balance_by, defaultValue=0)) - self.addOutput(OutputVector(self.OUTPUT_LAYER, self.tr('Colored'), datatype=[dataobjects.TYPE_VECTOR_POLYGON])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Colored'), QgsProcessing.TypeVectorPolygon)) def name(self): return 'topologicalcoloring' @@ -92,18 +92,18 @@ def displayName(self): return self.tr('Topological coloring') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT_LAYER), context) - min_colors = self.getParameterValue(self.MIN_COLORS) - balance_by = self.getParameterValue(self.BALANCE) - min_distance = self.getParameterValue(self.MIN_DISTANCE) + source = self.parameterAsSource(parameters, self.INPUT, context) + min_colors = self.parameterAsInt(parameters, self.MIN_COLORS, context) + balance_by = self.parameterAsEnum(parameters, self.BALANCE, context) + min_distance = self.parameterAsDouble(parameters, self.MIN_DISTANCE, context) - fields = layer.fields() + fields = source.fields() fields.append(QgsField('color_id', QVariant.Int)) - writer = self.getOutputFromName( - self.OUTPUT_LAYER).getVectorWriter(fields, layer.wkbType(), layer.crs(), context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, source.wkbType(), source.sourceCrs()) - features = {f.id(): f for f in QgsProcessingUtils.getFeatures(layer, context)} + features = {f.id(): f for f in source.getFeatures()} topology, id_graph = self.compute_graph(features, feedback, min_distance=min_distance) feature_colors = ColoringAlgorithm.balanced(features, @@ -118,6 +118,9 @@ def processAlgorithm(self, parameters, context, feedback): total = 20.0 / len(features) current = 0 for feature_id, input_feature in features.items(): + if feedback.isCanceled(): + break + output_feature = input_feature attributes = input_feature.attributes() if feature_id in feature_colors: @@ -126,11 +129,11 @@ def processAlgorithm(self, parameters, context, feedback): attributes.append(NULL) output_feature.setAttributes(attributes) - writer.addFeature(output_feature, QgsFeatureSink.FastInsert) + sink.addFeature(output_feature, QgsFeatureSink.FastInsert) current += 1 feedback.setProgress(80 + int(current * total)) - del writer + return {self.OUTPUT: dest_id} @staticmethod def compute_graph(features, feedback, create_id_graph=False, min_distance=0): @@ -148,6 +151,9 @@ def compute_graph(features, feedback, create_id_graph=False, min_distance=0): i = 0 for feature_id, f in features_with_geometry.items(): + if feedback.isCanceled(): + break + g = f.geometry() if min_distance > 0: g = g.buffer(min_distance, 5) @@ -172,6 +178,9 @@ def compute_graph(features, feedback, create_id_graph=False, min_distance=0): feedback.setProgress(int(i * total)) for feature_id, f in features_with_geometry.items(): + if feedback.isCanceled(): + break + if feature_id not in s.node_edge: s.add_edge(feature_id, None) @@ -206,6 +215,9 @@ def balanced(features, graph, feedback, balance=0, min_colors=4): i = 0 for (feature_id, n) in sorted_by_count: + if feedback.isCanceled(): + break + # first work out which already assigned colors are adjacent to this feature adjacent_colors = set() for neighbour in graph.node_edge[feature_id]: @@ -240,6 +252,9 @@ def balanced(features, graph, feedback, balance=0, min_colors=4): # loop through these, and calculate the minimum distance from this feature to the nearest # feature with each assigned color for other_feature_id, c in other_features.items(): + if feedback.isCanceled(): + break + other_geometry = features[other_feature_id].geometry() other_centroid = QgsPointXY(other_geometry.centroid().geometry()) diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml index 09ad337285b6..c16afe08ef6a 100644 --- a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml @@ -41,7 +41,7 @@ 8.23935 -3.11331 11 - 4 + 5 @@ -52,7 +52,7 @@ 8.23935 -6.11331 12 - 5 + 4 diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index a0adf00e4325..aa182b10f96b 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2598,32 +2598,33 @@ tests: name: expected/polygon_from_extent.gml type: vector -# - algorithm: qgis:topologicalcoloring -# name: Topological coloring -# params: -# INPUT_LAYER: -# name: custom/adjacent_polys.gml -# type: vector -# MIN_COLORS: 4 -# results: -# OUTPUT_LAYER: -# name: expected/topocolor_polys.gml -# type: vector -# -# - algorithm: qgis:topologicalcoloring -# name: Topological coloring w/ min distance -# params: -# BALANCE: '0' -# INPUT_LAYER: -# name: custom/adjacent_polys.gml -# type: vector -# MIN_COLORS: 4 -# MIN_DISTANCE: 4.0 -# results: -# OUTPUT_LAYER: -# name: expected/topocolor_polys_min_dist.gml -# type: vector -# + - algorithm: qgis:topologicalcoloring + name: Topological coloring + params: + BALANCE: 0 + INPUT: + name: custom/adjacent_polys.gml + type: vector + MIN_COLORS: 4 + results: + OUTPUT: + name: expected/topocolor_polys.gml + type: vector + + - algorithm: qgis:topologicalcoloring + name: Topological coloring w/ min distance + params: + BALANCE: 0 + INPUT: + name: custom/adjacent_polys.gml + type: vector + MIN_COLORS: 4 + MIN_DISTANCE: 4.0 + results: + OUTPUT: + name: expected/topocolor_polys_min_dist.gml + type: vector + - algorithm: qgis:regularpoints name: Regular point with standard extent params: From 03bae593bb3dfac77ef9f11ec8839f2d58a92ff2 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 00:37:14 +1000 Subject: [PATCH 12/23] Port Eliminate Selection to new API --- .../algs/qgis/EliminateSelection.py | 68 +++++++++++-------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/python/plugins/processing/algs/qgis/EliminateSelection.py b/python/plugins/processing/algs/qgis/EliminateSelection.py index b051d34545ff..ae54df30e0d2 100644 --- a/python/plugins/processing/algs/qgis/EliminateSelection.py +++ b/python/plugins/processing/algs/qgis/EliminateSelection.py @@ -34,15 +34,14 @@ QgsFeature, QgsFeatureSink, QgsGeometry, - QgsMessageLog, - QgsProcessingUtils) + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessing, + QgsProcessingParameterFeatureSink) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -67,16 +66,17 @@ def __init__(self): super().__init__() def initAlgorithm(self, config=None): - self.modes = [self.tr('Largest area'), + self.modes = [self.tr('Largest Area'), self.tr('Smallest Area'), - self.tr('Largest common boundary')] + self.tr('Largest Common Boundary')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'), [dataobjects.TYPE_VECTOR_POLYGON])) - self.addParameter(ParameterSelection(self.MODE, - self.tr('Merge selection with the neighbouring polygon with the'), - self.modes)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Eliminated'), datatype=[dataobjects.TYPE_VECTOR_POLYGON])) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'), [QgsProcessing.TypeVectorPolygon])) + self.addParameter(QgsProcessingParameterEnum(self.MODE, + self.tr('Merge selection with the neighbouring polygon with the'), + options=self.modes)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Eliminated'), QgsProcessing.TypeVectorPolygon)) def name(self): return 'eliminateselectedpolygons' @@ -85,29 +85,32 @@ def displayName(self): return self.tr('Eliminate selected polygons') def processAlgorithm(self, parameters, context, feedback): - inLayer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context) - boundary = self.getParameterValue(self.MODE) == self.MODE_BOUNDARY - smallestArea = self.getParameterValue(self.MODE) == self.MODE_SMALLEST_AREA + inLayer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + boundary = self.parameterAsEnum(parameters, self.MODE, context) == self.MODE_BOUNDARY + smallestArea = self.parameterAsEnum(parameters, self.MODE, context) == self.MODE_SMALLEST_AREA if inLayer.selectedFeatureCount() == 0: - QgsMessageLog.logMessage(self.tr('{0}: (No selection in input layer "{1}")').format(self.displayName(), self.getParameterValue(self.INPUT)), - self.tr('Processing'), QgsMessageLog.WARNING) + feedback.reportError(self.tr('{0}: (No selection in input layer "{1}")').format(self.displayName(), parameters[self.INPUT])) featToEliminate = [] selFeatIds = inLayer.selectedFeatureIds() - output = self.getOutputFromName(self.OUTPUT) - writer = output.getVectorWriter(inLayer.fields(), inLayer.wkbType(), inLayer.crs(), context) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + inLayer.fields(), inLayer.wkbType(), inLayer.sourceCrs()) for aFeat in inLayer.getFeatures(): + if feedback.isCanceled(): + break + if aFeat.id() in selFeatIds: # Keep references to the features to eliminate featToEliminate.append(aFeat) else: # write the others to output - writer.addFeature(aFeat, QgsFeatureSink.FastInsert) + sink.addFeature(aFeat, QgsFeatureSink.FastInsert) # Delete all features to eliminate in processLayer - processLayer = output.layer + processLayer = QgsProcessingUtils.mapLayerFromString(dest_id, context) processLayer.startEditing() # ANALYZE @@ -129,6 +132,9 @@ def processAlgorithm(self, parameters, context, feedback): # Iterate over the polygons to eliminate for i in range(len(featToEliminate)): + if feedback.isCanceled(): + break + feat = featToEliminate.pop() geom2Eliminate = feat.geometry() bbox = geom2Eliminate.boundingBox() @@ -145,6 +151,9 @@ def processAlgorithm(self, parameters, context, feedback): engine.prepareGeometry() while fit.nextFeature(selFeat): + if feedback.isCanceled(): + break + selGeom = selFeat.geometry() if engine.intersects(selGeom.geometry()): @@ -193,7 +202,7 @@ def processAlgorithm(self, parameters, context, feedback): if processLayer.changeGeometry(mergeWithFid, newGeom): madeProgress = True else: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Could not replace geometry of feature with id {0}').format(mergeWithFid)) start = start + add @@ -207,7 +216,12 @@ def processAlgorithm(self, parameters, context, feedback): # End while if not processLayer.commitChanges(): - raise GeoAlgorithmExecutionException(self.tr('Could not commit changes')) + raise QgsProcessingException(self.tr('Could not commit changes')) for feature in featNotEliminated: - writer.addFeature(feature, QgsFeatureSink.FastInsert) + if feedback.isCanceled(): + break + + sink.addFeature(feature, QgsFeatureSink.FastInsert) + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index e1e1cc7e47a5..5a04391fa90f 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -60,6 +60,7 @@ from .Difference import Difference from .DropGeometry import DropGeometry from .DropMZValues import DropMZValues +from .EliminateSelection import EliminateSelection from .EquivalentNumField import EquivalentNumField from .Explode import Explode from .ExportGeometryInfo import ExportGeometryInfo @@ -170,7 +171,6 @@ # from .RasterCalculator import RasterCalculator # from .ExecuteSQL import ExecuteSQL # from .FindProjection import FindProjection -# from .EliminateSelection import EliminateSelection pluginPath = os.path.normpath(os.path.join( os.path.split(os.path.dirname(__file__))[0], os.pardir)) @@ -206,7 +206,6 @@ def getAlgs(self): # IdwInterpolation(), TinInterpolation(), # RasterCalculator(), # ExecuteSQL(), FindProjection(), - # EliminateSelection() # ] algs = [AddTableField(), Aspect(), @@ -228,6 +227,7 @@ def getAlgs(self): Difference(), DropGeometry(), DropMZValues(), + EliminateSelection(), EquivalentNumField(), Explode(), ExportGeometryInfo(), From 5d635d190df39e4061f16b825290ec43fcf804f4 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 18:45:48 +1000 Subject: [PATCH 13/23] Allow list of acceptable raster hashes for processing algorithm tests Differences in gdal libraries mean the hash value may differ between platforms. Allow multiple acceptable hashes to be listed for expected test results --- .../plugins/processing/tests/AlgorithmsTestBase.py | 5 ++++- .../tests/testdata/qgis_algorithm_tests.yaml | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index 65c4b119687c..c4a2f437b39a 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -279,7 +279,10 @@ def check_results(self, results, context, params, expected): dataArray = nan_to_num(dataset.ReadAsArray(0)) strhash = hashlib.sha224(dataArray.data).hexdigest() - self.assertEqual(strhash, expected_result['hash']) + if not isinstance(expected_result['hash'], str): + self.assertTrue(strhash in expected_result['hash']) + else: + self.assertEqual(strhash, expected_result['hash']) elif 'file' == expected_result['type']: expected_filepath = self.filepath_from_param(expected_result) result_filepath = results[id] diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index aa182b10f96b..297f4123497b 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -1148,7 +1148,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 762865ee485a6736d188402aa10e6fd38a812a9e45a7dd2d4885a63a + hash: + - 762865ee485a6736d188402aa10e6fd38a812a9e45a7dd2d4885a63a + - f6a8e64647ae93a94f2a4945add8986526a7a07bc85849f3690d15b2 type: rasterhash - algorithm: qgis:slope @@ -1160,7 +1162,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 151ea76a21b286c16567eb6b4b692925a84145b65561a0017effb1a1 + hash: + - 151ea76a21b286c16567eb6b4b692925a84145b65561a0017effb1a1 + - 177475642c57428b395bc0a1e7e86fc1cfd4d86ffc19f31ff8bc964d type: rasterhash - algorithm: qgis:ruggednessindex @@ -1186,7 +1190,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 58365b3715b925d6286e7f082ebd9c2a20f09fa1c922176d3f238002 + hash: + - 58365b3715b925d6286e7f082ebd9c2a20f09fa1c922176d3f238002 + - 75cca4c1a870a1e21185a2d85b33b6d9958a69fc6ebb04e4d6ceb8a3 type: rasterhash # - algorithm: qgis:relief From e8d667cac33f5992c9f1a325c5bebda773cf98ea Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 19:07:03 +1000 Subject: [PATCH 14/23] Allow testing of layer equality without throwing asserts Sometimes in tests it's required to check for layer equality without aborting in case of mismatches --- python/testing/__init__.py | 79 ++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/python/testing/__init__.py b/python/testing/__init__.py index 5e2ae736cf02..a495a3356811 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -55,6 +55,21 @@ def assertLayersEqual(self, layer_expected, layer_result, **kwargs): { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } """ + self.checkLayersEqual(layer_expected, layer_result, True, **kwargs) + + def checkLayersEqual(self, layer_expected, layer_result, use_asserts=False, **kwargs): + """ + :param layer_expected: The first layer to compare + :param layer_result: The second layer to compare + :param use_asserts: If true, asserts are used to test conditions, if false, asserts + are not used and the function will only return False if the test fails + :param request: Optional, A feature request. This can be used to specify + an order by clause to make sure features are compared in + a given sequence if they don't match by default. + :keyword compare: A map of comparison options. e.g. + { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } + { fields: { __all__: cast( str ) } } + """ try: request = kwargs['request'] @@ -67,10 +82,16 @@ def assertLayersEqual(self, layer_expected, layer_result, **kwargs): compare = {} # Compare CRS - _TestCase.assertEqual(self, layer_expected.dataProvider().crs().authid(), layer_result.dataProvider().crs().authid()) + if use_asserts: + _TestCase.assertEqual(self, layer_expected.dataProvider().crs().authid(), layer_result.dataProvider().crs().authid()) + elif not layer_expected.dataProvider().crs().authid() == layer_result.dataProvider().crs().authid(): + return False # Compare features - _TestCase.assertEqual(self, layer_expected.featureCount(), layer_result.featureCount()) + if use_asserts: + _TestCase.assertEqual(self, layer_expected.featureCount(), layer_result.featureCount()) + elif layer_expected.featureCount() != layer_result.featureCount(): + return False try: precision = compare['geometry']['precision'] @@ -89,17 +110,20 @@ def assertLayersEqual(self, layer_expected, layer_result, **kwargs): geom1 = feats[1].geometry().geometry().asWkt(precision) else: geom1 = None - _TestCase.assertEqual( - self, - geom0, - geom1, - 'Features {}/{} differ in geometry: \n\n {}\n\n vs \n\n {}'.format( - feats[0].id(), - feats[1].id(), + if use_asserts: + _TestCase.assertEqual( + self, geom0, - geom1 + geom1, + 'Features {}/{} differ in geometry: \n\n {}\n\n vs \n\n {}'.format( + feats[0].id(), + feats[1].id(), + geom0, + geom1 + ) ) - ) + elif geom0 != geom1: + return False for attr_expected, field_expected in zip(feats[0].attributes(), layer_expected.fields().toList()): try: @@ -134,21 +158,26 @@ def assertLayersEqual(self, layer_expected, layer_result, **kwargs): attr_expected = round(attr_expected, cmp['precision']) attr_result = round(attr_result, cmp['precision']) - _TestCase.assertEqual( - self, - attr_expected, - attr_result, - 'Features {}/{} differ in attributes\n\n * Field1: {} ({})\n * Field2: {} ({})\n\n * {} != {}'.format( - feats[0].id(), - feats[1].id(), - field_expected.name(), - field_expected.typeName(), - field_result.name(), - field_result.typeName(), - repr(attr_expected), - repr(attr_result) + if use_asserts: + _TestCase.assertEqual( + self, + attr_expected, + attr_result, + 'Features {}/{} differ in attributes\n\n * Field1: {} ({})\n * Field2: {} ({})\n\n * {} != {}'.format( + feats[0].id(), + feats[1].id(), + field_expected.name(), + field_expected.typeName(), + field_result.name(), + field_result.typeName(), + repr(attr_expected), + repr(attr_result) + ) ) - ) + elif attr_expected != attr_result: + return False + + return True def assertFilesEqual(self, filepath_expected, filepath_result): with open(filepath_expected, 'r') as file_expected: From 9968962ab925474da97218eede9162106eb84d41 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:39:40 +1000 Subject: [PATCH 15/23] Allow specifying multiple possible vector layer results for processing tests Some algorithms are non-deterministic and the results may vary from run to run. In this case we allow specifying multiple possible valid results, and the test will pass if the result layer matches any of these. --- .../processing/tests/AlgorithmsTestBase.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index c4a2f437b39a..2ae126ccbf67 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -43,6 +43,7 @@ from osgeo.gdalconst import GA_ReadOnly from numpy import nan_to_num +from copy import deepcopy import processing @@ -186,7 +187,10 @@ def load_result_param(self, param): if param['type'] in ['vector', 'file', 'table', 'regex']: outdir = tempfile.mkdtemp() self.cleanup_paths.append(outdir) - basename = os.path.basename(param['name']) + if isinstance(param['name'], str): + basename = os.path.basename(param['name']) + else: + basename = os.path.basename(param['name'][0]) filepath = os.path.join(outdir, basename) return filepath elif param['type'] == 'rasterhash': @@ -198,6 +202,19 @@ def load_result_param(self, param): raise KeyError("Unknown type '{}' specified for parameter".format(param['type'])) + def load_layers(self, id, param): + layers = [] + if param['type'] in ('vector', 'table') and isinstance(param['name'], str): + layers.append(self.load_layer(id, param)) + elif param['type'] in ('vector', 'table'): + for n in param['name']: + layer_param = deepcopy(param) + layer_param['name'] = n + layers.append(self.load_layer(id, layer_param)) + else: + layers.append(self.load_layer(id, param)) + return layers + def load_layer(self, id, param): """ Loads a layer which was specified as parameter. @@ -253,7 +270,7 @@ def check_results(self, results, context, params, expected): self.assertTrue(result_lyr.isValid()) continue - expected_lyr = self.load_layer(id, expected_result) + expected_lyrs = self.load_layers(id, expected_result) if 'in_place_result' in expected_result: result_lyr = QgsProcessingUtils.mapLayerFromString(self.in_place_layers[id], context) self.assertTrue(result_lyr.isValid(), self.in_place_layers[id]) @@ -271,7 +288,15 @@ def check_results(self, results, context, params, expected): compare = expected_result.get('compare', {}) - self.assertLayersEqual(expected_lyr, result_lyr, compare=compare) + if len(expected_lyrs) == 1: + self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare) + else: + res = False + for l in expected_lyrs: + if self.checkLayersEqual(l, result_lyr, compare=compare): + res = True + break + self.assertTrue(res, 'Could not find matching layer in expected results') elif 'rasterhash' == expected_result['type']: print("id:{} result:{}".format(id, results[id])) From c2559d827317fa201d1e1211b41bef00150ff6cd Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:40:49 +1000 Subject: [PATCH 16/23] Add second reference layer for topocolor algorithm --- .../testdata/expected/topocolor_polys.gml | 4 +- .../testdata/expected/topocolor_polys2.gfs | 46 ++++++ .../testdata/expected/topocolor_polys2.gml | 133 ++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 4 +- 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml index c16afe08ef6a..09ad337285b6 100644 --- a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml @@ -41,7 +41,7 @@ 8.23935 -3.11331 11 - 5 + 4 @@ -52,7 +52,7 @@ 8.23935 -6.11331 12 - 4 + 5 diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs new file mode 100644 index 000000000000..eb07beba7512 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs @@ -0,0 +1,46 @@ + + + topocolor_polys + topocolor_polys + + 3 + EPSG:4326 + + 11 + -0.76065 + 14.23935 + -6.11331 + 5.88669 + + + left + left + Real + + + top + top + Real + + + right + right + Real + + + bottom + bottom + Real + + + id + id + Integer + + + color_id + color_id + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml new file mode 100644 index 000000000000..c16afe08ef6a --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml @@ -0,0 +1,133 @@ + + + + + -0.760650357995228-6.11330548926014 + 14.23934964200485.88669451073986 + + + + + + -0.760650357995228,-0.113305489260142 2.23934964200477,-0.113305489260142 2.23934964200477,-3.11330548926014 -0.760650357995228,-3.11330548926014 -0.760650357995228,-0.113305489260142 + -0.76065 + -0.11331 + 2.23935 + -3.11331 + 3 + 1 + + + + + -0.760650357995228,-3.11330548926014 2.23934964200477,-3.11330548926014 2.23934964200477,-6.11330548926014 -0.760650357995228,-6.11330548926014 -0.760650357995228,-3.11330548926014 + -0.76065 + -3.11331 + 2.23935 + -6.11331 + 4 + 3 + + + + + 5.23934964200477,-0.113305489260142 8.23934964200477,-0.113305489260142 8.23934964200477,-3.11330548926014 5.23934964200477,-3.11330548926014 5.23934964200477,-0.113305489260142 + 5.23935 + -0.11331 + 8.23935 + -3.11331 + 11 + 5 + + + + + 5.23934964200477,-3.11330548926014 8.23934964200477,-3.11330548926014 8.23934964200477,-6.11330548926014 5.23934964200477,-6.11330548926014 5.23934964200477,-3.11330548926014 + 5.23935 + -3.11331 + 8.23935 + -6.11331 + 12 + 4 + + + + + 8.23934964200477,-3.11330548926014 11.2393496420048,-3.11330548926014 11.2393496420048,-6.11330548926014 8.23934964200477,-6.11330548926014 8.23934964200477,-3.11330548926014 + 8.23935 + -3.11331 + 11.23935 + -6.11331 + 16 + 3 + + + + + 11.2393496420048,-0.113305489260142 14.2393496420048,-0.113305489260142 14.2393496420048,-3.11330548926014 11.2393496420048,-3.11330548926014 11.2393496420048,-0.113305489260142 + 11.23935 + -0.11331 + 14.23935 + -3.11331 + 19 + 2 + + + + + 11.2393496420048,-3.11330548926014 14.2393496420048,-3.11330548926014 14.2393496420048,-6.11330548926014 11.2393496420048,-6.11330548926014 11.2393496420048,-3.11330548926014 + 11.23935 + -3.11331 + 14.23935 + -6.11331 + 20 + 5 + + + + + 2.23934964200477,5.88669451073986 5.23934964200477,5.88669451073986 5.23934964200477,2.88669451073986 2.23934964200477,2.88669451073986 2.23934964200477,-0.113305489260142 -0.760650357995228,-0.113305489260142 -0.760650357995228,2.88669451073986 -0.760650357995228,5.88669451073986 2.23934964200477,5.88669451073986 + 2.23935 + 5.88669 + 5.23935 + 2.88669 + 5 + 3 + + + + + 5.23934964200477,2.88669451073986 8.23934964200477,2.88669451073986 11.2393496420048,2.88669451073986 11.2393496420048,-0.113305489260142 11.2393496420048,-3.11330548926014 8.23934964200477,-3.11330548926014 8.23934964200477,-0.113305489260142 5.23934964200477,-0.113305489260142 5.23934964200477,2.88669451073986 + 5.23935 + 2.88669 + 8.23935 + 10 + 1 + + + + + 2.23934964200477,2.88669451073986 5.23934964200477,2.88669451073986 5.23934964200477,-0.113305489260142 5.23934964200477,-3.11330548926014 5.23934964200477,-6.11330548926014 2.23934964200477,-6.11330548926014 2.23934964200477,-3.11330548926014 2.23934964200477,-0.113305489260142 2.23934964200477,2.88669451073986 + 2.23935 + 2.88669 + 5.23935 + 6 + 2 + + + + + 5.23934964200477,5.88669451073986 8.23934964200477,5.88669451073986 11.2393496420048,5.88669451073986 14.2393496420048,5.88669451073986 14.2393496420048,2.88669451073986 14.2393496420048,-0.113305489260142 11.2393496420048,-0.113305489260142 11.2393496420048,2.88669451073986 8.23934964200477,2.88669451073986 5.23934964200477,2.88669451073986 5.23934964200477,5.88669451073986 + 5.23935 + 5.88669 + 8.23935 + 2.88669 + 9 + 4 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 297f4123497b..846d2c65b1f9 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2614,8 +2614,10 @@ tests: MIN_COLORS: 4 results: OUTPUT: - name: expected/topocolor_polys.gml type: vector + name: + - expected/topocolor_polys.gml + - expected/topocolor_polys2.gml - algorithm: qgis:topologicalcoloring name: Topological coloring w/ min distance From a1f487d679ed180a34a6949b24c9eb2a208ef462 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:45:16 +1000 Subject: [PATCH 17/23] Add alternate reference layer for points to path algorithm --- .../expected/points_to_path_grouped2.gfs | 31 +++++++++++++++ .../expected/points_to_path_grouped2.gml | 39 +++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 4 +- 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs new file mode 100644 index 000000000000..076d497b0960 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs @@ -0,0 +1,31 @@ + + + points_to_path_grouped + points_to_path_grouped + + 2 + EPSG:4326 + + 3 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id2 + id2 + Integer + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml new file mode 100644 index 000000000000..68ec6408c300 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml @@ -0,0 +1,39 @@ + + + + + 0-5 + 83 + + + + + + 2,2 0,-5 8,-1 7,-1 0,-1 + 0 + 3 + 9 + + + + + 3,3 4,1 + 1 + 2 + 5 + + + + + 1,1 5,2 + 2 + 1 + 4 + + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 846d2c65b1f9..83b6a0b3dd4b 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2260,7 +2260,9 @@ tests: GROUP_FIELD: id2 results: OUTPUT: - name: expected/points_to_path_grouped.gml + name: + - expected/points_to_path_grouped.gml + - expected/points_to_path_grouped2.gml type: vector # - algorithm: qgis:joinattributestable From 572dadab0157e60ea1d75d2cc854ee7ba85cb436 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 05:06:46 +1000 Subject: [PATCH 18/23] Remember window geometry in multi input dialogs --- python/plugins/processing/gui/MultipleFileInputDialog.py | 8 ++++++++ python/plugins/processing/gui/MultipleInputDialog.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/python/plugins/processing/gui/MultipleFileInputDialog.py b/python/plugins/processing/gui/MultipleFileInputDialog.py index 43762ac8708b..2b448056f555 100644 --- a/python/plugins/processing/gui/MultipleFileInputDialog.py +++ b/python/plugins/processing/gui/MultipleFileInputDialog.py @@ -32,6 +32,7 @@ from qgis.core import QgsSettings from qgis.PyQt import uic +from qgis.PyQt.QtCore import QByteArray from qgis.PyQt.QtWidgets import QDialog, QAbstractItemView, QPushButton, QDialogButtonBox, QFileDialog from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem @@ -65,7 +66,14 @@ def __init__(self, options): self.btnRemove.clicked.connect(lambda: self.removeRows()) self.btnRemoveAll.clicked.connect(lambda: self.removeRows(True)) + self.settings = QgsSettings() + self.restoreGeometry(self.settings.value("/Processing/multipleFileInputDialogGeometry", QByteArray())) + self.populateList() + self.finished.connect(self.saveWindowGeometry) + + def saveWindowGeometry(self): + self.settings.setValue("/Processing/multipleInputDialogGeometry", self.saveGeometry()) def populateList(self): model = QStandardItemModel() diff --git a/python/plugins/processing/gui/MultipleInputDialog.py b/python/plugins/processing/gui/MultipleInputDialog.py index e609af1959f3..a8fd860f02dd 100644 --- a/python/plugins/processing/gui/MultipleInputDialog.py +++ b/python/plugins/processing/gui/MultipleInputDialog.py @@ -29,8 +29,10 @@ import os +from qgis.core import QgsSettings from qgis.PyQt import uic from qgis.PyQt.QtCore import Qt +from qgis.PyQt.QtCore import QByteArray from qgis.PyQt.QtWidgets import QDialog, QAbstractItemView, QPushButton, QDialogButtonBox from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem @@ -71,7 +73,14 @@ def __init__(self, options, selectedoptions=None): self.btnClearSelection.clicked.connect(lambda: self.selectAll(False)) self.btnToggleSelection.clicked.connect(self.toggleSelection) + self.settings = QgsSettings() + self.restoreGeometry(self.settings.value("/Processing/multipleInputDialogGeometry", QByteArray())) + self.populateList() + self.finished.connect(self.saveWindowGeometry) + + def saveWindowGeometry(self): + self.settings.setValue("/Processing/multipleInputDialogGeometry", self.saveGeometry()) def populateList(self): model = QStandardItemModel() From adda744576be7722377a0fc81dc18658b600c9e7 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 05:41:15 +1000 Subject: [PATCH 19/23] Port Join Attributes alg to new API Improvements: - don't fetch unused geometry for joined table --- .../processing/algs/qgis/JoinAttributes.py | 81 ++++++++++--------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- .../tests/testdata/qgis_algorithm_tests.yaml | 30 +++---- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/python/plugins/processing/algs/qgis/JoinAttributes.py b/python/plugins/processing/algs/qgis/JoinAttributes.py index 1354d5a7a798..5898bd891866 100644 --- a/python/plugins/processing/algs/qgis/JoinAttributes.py +++ b/python/plugins/processing/algs/qgis/JoinAttributes.py @@ -30,14 +30,13 @@ from qgis.core import (QgsFeature, QgsFeatureSink, - QgsApplication, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessingParameterFeatureSource, + QgsProcessingUtils, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTable -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector from processing.tools import vector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -45,11 +44,11 @@ class JoinAttributes(QgisAlgorithm): - OUTPUT_LAYER = 'OUTPUT_LAYER' - INPUT_LAYER = 'INPUT_LAYER' - INPUT_LAYER_2 = 'INPUT_LAYER_2' - TABLE_FIELD = 'TABLE_FIELD' - TABLE_FIELD_2 = 'TABLE_FIELD_2' + OUTPUT = 'OUTPUT' + INPUT = 'INPUT' + INPUT_2 = 'INPUT_2' + FIELD = 'FIELD' + FIELD_2 = 'FIELD_2' def group(self): return self.tr('Vector general tools') @@ -58,16 +57,15 @@ def __init__(self): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input layer'))) - self.addParameter(ParameterTable(self.INPUT_LAYER_2, - self.tr('Input layer 2'), False)) - self.addParameter(ParameterTableField(self.TABLE_FIELD, - self.tr('Table field'), self.INPUT_LAYER)) - self.addParameter(ParameterTableField(self.TABLE_FIELD_2, - self.tr('Table field 2'), self.INPUT_LAYER_2)) - self.addOutput(OutputVector(self.OUTPUT_LAYER, - self.tr('Joined layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT_2, + self.tr('Input layer 2'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Table field'), parentLayerParameterName=self.INPUT)) + self.addParameter(QgsProcessingParameterField(self.FIELD_2, + self.tr('Table field 2'), parentLayerParameterName=self.INPUT_2)) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Joined layer'))) def name(self): return 'joinattributestable' @@ -76,26 +74,27 @@ def displayName(self): return self.tr('Join attributes table') def processAlgorithm(self, parameters, context, feedback): - input = self.getParameterValue(self.INPUT_LAYER) - input2 = self.getParameterValue(self.INPUT_LAYER_2) - output = self.getOutputFromName(self.OUTPUT_LAYER) - field = self.getParameterValue(self.TABLE_FIELD) - field2 = self.getParameterValue(self.TABLE_FIELD_2) + input = self.parameterAsSource(parameters, self.INPUT, context) + input2 = self.parameterAsSource(parameters, self.INPUT_2, context) + field = self.parameterAsString(parameters, self.FIELD, context) + field2 = self.parameterAsString(parameters, self.FIELD_2, context) - layer = QgsProcessingUtils.mapLayerFromString(input, context) - joinField1Index = layer.fields().lookupField(field) + joinField1Index = input.fields().lookupField(field) + joinField2Index = input2.fields().lookupField(field2) - layer2 = QgsProcessingUtils.mapLayerFromString(input2, context) - joinField2Index = layer2.fields().lookupField(field2) + outFields = vector.combineFields(input.fields(), input2.fields()) - outFields = vector.combineVectorFields(layer, layer2) - writer = output.getVectorWriter(outFields, layer.wkbType(), layer.crs(), context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + outFields, input.wkbType(), input.sourceCrs()) - # Cache attributes of Layer 2 + # Cache attributes of input2 cache = {} - features = QgsProcessingUtils.getFeatures(layer2, context) - total = 100.0 / layer2.featureCount() if layer2.featureCount() else 0 + features = input2.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)) + total = 100.0 / input2.featureCount() if input2.featureCount() else 0 for current, feat in enumerate(features): + if feedback.isCanceled(): + break + attrs = feat.attributes() joinValue2 = str(attrs[joinField2Index]) if joinValue2 not in cache: @@ -104,14 +103,18 @@ def processAlgorithm(self, parameters, context, feedback): # Create output vector layer with additional attribute outFeat = QgsFeature() - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = input.getFeatures() + total = 100.0 / input.featureCount() if input.featureCount() else 0 for current, feat in enumerate(features): + if feedback.isCanceled(): + break + outFeat.setGeometry(feat.geometry()) attrs = feat.attributes() joinValue1 = str(attrs[joinField1Index]) attrs.extend(cache.get(joinValue1, [])) outFeat.setAttributes(attrs) - writer.addFeature(outFeat, QgsFeatureSink.FastInsert) + sink.addFeature(outFeat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 5a04391fa90f..3773bc80779a 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -82,6 +82,7 @@ from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection +from .JoinAttributes import JoinAttributes from .LinesIntersection import LinesIntersection from .LinesToPolygons import LinesToPolygons from .MeanCoords import MeanCoords @@ -151,7 +152,6 @@ # from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator # from .FieldPyculator import FieldsPyculator -# from .JoinAttributes import JoinAttributes # from .PointsDisplacement import PointsDisplacement # from .PointsFromPolygons import PointsFromPolygons # from .PointsFromLines import PointsFromLines @@ -189,7 +189,6 @@ def getAlgs(self): # ExtractByLocation(), # SpatialJoin(), # GeometryConvert(), FieldsCalculator(), - # JoinAttributes(), # FieldsPyculator(), # StatisticsByCategories(), # RasterLayerStatistics(), PointsDisplacement(), @@ -249,6 +248,7 @@ def getAlgs(self): ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), + JoinAttributes(), LinesIntersection(), LinesToPolygons(), MeanCoords(), diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 83b6a0b3dd4b..64d8729b470b 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2265,21 +2265,21 @@ tests: - expected/points_to_path_grouped2.gml type: vector -# - algorithm: qgis:joinattributestable -# name: join the attribute table by common field -# params: -# INPUT_LAYER: -# name: points.gml -# type: vector -# INPUT_LAYER_2: -# name: table.dbf -# type: table -# TABLE_FIELD: id -# TABLE_FIELD_2: ID -# results: -# OUTPUT_LAYER: -# name: expected/join_attribute_table.gml -# type: vector + - algorithm: qgis:joinattributestable + name: join the attribute table by common field + params: + INPUT: + name: points.gml + type: vector + INPUT_2: + name: table.dbf + type: table + FIELD: id + FIELD_2: ID + results: + OUTPUT: + name: expected/join_attribute_table.gml + type: vector - algorithm: qgis:convexhull name: Simple convex hull From b93be39c2469edb776ad898a4693c92a9800ecbe Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 07:01:06 +1000 Subject: [PATCH 20/23] Port Stats by Category to new API Improvements: - keep original field type and name for category field - add unit test --- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../algs/qgis/StatisticsByCategories.py | 80 ++++++++++++------- .../testdata/expected/stats_by_category.gfs | 45 +++++++++++ .../testdata/expected/stats_by_category.gml | 42 ++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 13 +++ 5 files changed, 153 insertions(+), 32 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/stats_by_category.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/stats_by_category.gml diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 3773bc80779a..c93138350e52 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -132,6 +132,7 @@ from .SpatialiteExecuteSQL import SpatialiteExecuteSQL from .SpatialIndex import SpatialIndex from .SplitWithLines import SplitWithLines +from .StatisticsByCategories import StatisticsByCategories from .SumLines import SumLines from .SymmetricalDifference import SymmetricalDifference from .TextToFloat import TextToFloat @@ -149,7 +150,6 @@ # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin # from .GeometryConvert import GeometryConvert -# from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator # from .FieldPyculator import FieldsPyculator # from .PointsDisplacement import PointsDisplacement @@ -190,7 +190,7 @@ def getAlgs(self): # SpatialJoin(), # GeometryConvert(), FieldsCalculator(), # FieldsPyculator(), - # StatisticsByCategories(), + # # RasterLayerStatistics(), PointsDisplacement(), # PointsFromPolygons(), # PointsFromLines(), @@ -298,6 +298,7 @@ def getAlgs(self): SpatialiteExecuteSQL(), SpatialIndex(), SplitWithLines(), + StatisticsByCategories(), SumLines(), SymmetricalDifference(), TextToFloat(), diff --git a/python/plugins/processing/algs/qgis/StatisticsByCategories.py b/python/plugins/processing/algs/qgis/StatisticsByCategories.py index 1b6e9a5a277a..8257d67c53c6 100644 --- a/python/plugins/processing/algs/qgis/StatisticsByCategories.py +++ b/python/plugins/processing/algs/qgis/StatisticsByCategories.py @@ -26,19 +26,24 @@ __revision__ = '$Format:%H$' -from qgis.core import (QgsApplication, - QgsFeatureSink, +from qgis.core import (QgsProcessingParameterFeatureSource, QgsStatisticalSummary, - QgsProcessingUtils) -from processing.core.outputs import OutputTable + QgsFeatureRequest, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink, + QgsFields, + QgsField, + QgsWkbTypes, + QgsCoordinateReferenceSystem, + QgsFeature, + QgsFeatureSink) +from qgis.PyQt.QtCore import QVariant from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField class StatisticsByCategories(QgisAlgorithm): - INPUT_LAYER = 'INPUT_LAYER' + INPUT = 'INPUT' VALUES_FIELD_NAME = 'VALUES_FIELD_NAME' CATEGORIES_FIELD_NAME = 'CATEGORIES_FIELD_NAME' OUTPUT = 'OUTPUT' @@ -50,16 +55,16 @@ def __init__(self): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input vector layer'))) - self.addParameter(ParameterTableField(self.VALUES_FIELD_NAME, - self.tr('Field to calculate statistics on'), - self.INPUT_LAYER, ParameterTableField.DATA_TYPE_NUMBER)) - self.addParameter(ParameterTableField(self.CATEGORIES_FIELD_NAME, - self.tr('Field with categories'), - self.INPUT_LAYER, ParameterTableField.DATA_TYPE_ANY)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input vector layer'))) + self.addParameter(QgsProcessingParameterField(self.VALUES_FIELD_NAME, + self.tr('Field to calculate statistics on'), + parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Numeric)) + self.addParameter(QgsProcessingParameterField(self.CATEGORIES_FIELD_NAME, + self.tr('Field with categories'), + parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any)) - self.addOutput(OutputTable(self.OUTPUT, self.tr('Statistics by category'))) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Statistics by category'))) def name(self): return 'statisticsbycategories' @@ -68,36 +73,51 @@ def displayName(self): return self.tr('Statistics by categories') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT_LAYER), context) - valuesFieldName = self.getParameterValue(self.VALUES_FIELD_NAME) - categoriesFieldName = self.getParameterValue(self.CATEGORIES_FIELD_NAME) + source = self.parameterAsSource(parameters, self.INPUT, context) + value_field_name = self.parameterAsString(parameters, self.VALUES_FIELD_NAME, context) + category_field_name = self.parameterAsString(parameters, self.CATEGORIES_FIELD_NAME, context) - output = self.getOutputFromName(self.OUTPUT) - valuesField = layer.fields().lookupField(valuesFieldName) - categoriesField = layer.fields().lookupField(categoriesFieldName) + value_field_index = source.fields().lookupField(value_field_name) + category_field_index = source.fields().lookupField(category_field_name) - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = source.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)) + total = 100.0 / source.featureCount() if source.featureCount() else 0 values = {} for current, feat in enumerate(features): + if feedback.isCanceled(): + break + feedback.setProgress(int(current * total)) attrs = feat.attributes() try: - value = float(attrs[valuesField]) - cat = str(attrs[categoriesField]) + value = float(attrs[value_field_index]) + cat = attrs[category_field_index] if cat not in values: values[cat] = [] values[cat].append(value) except: pass - fields = ['category', 'min', 'max', 'mean', 'stddev', 'sum', 'count'] - writer = output.getTableWriter(fields) + fields = QgsFields() + fields.append(source.fields().at(category_field_index)) + fields.append(QgsField('min', QVariant.Double)) + fields.append(QgsField('max', QVariant.Double)) + fields.append(QgsField('mean', QVariant.Double)) + fields.append(QgsField('stddev', QVariant.Double)) + fields.append(QgsField('sum', QVariant.Double)) + fields.append(QgsField('count', QVariant.Int)) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem()) + stat = QgsStatisticalSummary(QgsStatisticalSummary.Min | QgsStatisticalSummary.Max | QgsStatisticalSummary.Mean | QgsStatisticalSummary.StDevSample | QgsStatisticalSummary.Sum | QgsStatisticalSummary.Count) for (cat, v) in list(values.items()): stat.calculate(v) - record = [cat, stat.min(), stat.max(), stat.mean(), stat.sampleStDev(), stat.sum(), stat.count()] - writer.addRecord(record) + f = QgsFeature() + f.setAttributes([cat, stat.min(), stat.max(), stat.mean(), stat.sampleStDev(), stat.sum(), stat.count()]) + sink.addFeature(f, QgsFeatureSink.FastInsert) + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs b/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs new file mode 100644 index 000000000000..ba663caa0c04 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs @@ -0,0 +1,45 @@ + + + stats_by_category + stats_by_category + 100 + + 3 + + + id2 + id2 + Integer + + + min + min + Integer + + + max + max + Integer + + + mean + mean + Real + + + stddev + stddev + Real + + + sum + sum + Integer + + + count + count + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/stats_by_category.gml b/python/plugins/processing/tests/testdata/expected/stats_by_category.gml new file mode 100644 index 000000000000..4647a986f759 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/stats_by_category.gml @@ -0,0 +1,42 @@ + + + missing + + + + 2 + 1 + 4 + 2.5 + 2.12132034355964 + 5 + 2 + + + + + 1 + 2 + 5 + 3.5 + 2.12132034355964 + 7 + 2 + + + + + 0 + 3 + 9 + 6.6 + 2.30217288664427 + 33 + 5 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 64d8729b470b..215160793c68 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2489,6 +2489,19 @@ tests: name: expected/single_to_multi.gml type: vector + - algorithm: qgis:statisticsbycategories + name: stats by category + params: + VALUES_FIELD_NAME: id + CATEGORIES_FIELD_NAME: id2 + INPUT: + name: points.gml + type: vector + results: + OUTPUT: + name: expected/stats_by_category.gml + type: vector + # - algorithm: qgis:zonalstatistics # name: simple zonal statistics # params: From 6aa672d9e0ee4a4833b5ee41cd122ae9a7cd14c3 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 17:51:34 +1000 Subject: [PATCH 21/23] Fix typo in gridify alg --- python/plugins/processing/algs/qgis/Gridify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/plugins/processing/algs/qgis/Gridify.py b/python/plugins/processing/algs/qgis/Gridify.py index 2605ccd9cd0c..06f99983bf8e 100644 --- a/python/plugins/processing/algs/qgis/Gridify.py +++ b/python/plugins/processing/algs/qgis/Gridify.py @@ -83,7 +83,7 @@ def processFeature(self, feature, feedback): points = self._gridify([geom.asPoint()], self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromPoint(points[0]) elif geomType == QgsWkbTypes.MultiPoint: - points = self._gridify(geom.aMultiPoint(), self.h_spacing, self.v_spacing) + points = self._gridify(geom.asMultiPoint(), self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromMultiPoint(points) elif geomType == QgsWkbTypes.LineString: points = self._gridify(geom.asPolyline(), self.h_spacing, self.v_spacing) From d4ad063f45b32191d320620d2bf1c4b54c1a1359 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 22:10:18 +1000 Subject: [PATCH 22/23] Allow specifying a 'primary key' field when comparing layers for processing tests Some algorithms will return results in different orders, e.g. due to the use of dicts or other methods which do not guarantee a fixed return order. Using a primary key to do the feature match allows us to flexibly handle these situations and provide tests for these algorithms. --- .../processing/tests/AlgorithmsTestBase.py | 5 +++-- .../tests/testdata/qgis_algorithm_tests.yaml | 4 ++++ python/testing/__init__.py | 18 ++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index 2ae126ccbf67..2e19146a7b76 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -287,13 +287,14 @@ def check_results(self, results, context, params, expected): self.assertTrue(result_lyr, results[id]) compare = expected_result.get('compare', {}) + pk = expected_result.get('pk', None) if len(expected_lyrs) == 1: - self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare) + self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare, pk=pk) else: res = False for l in expected_lyrs: - if self.checkLayersEqual(l, result_lyr, compare=compare): + if self.checkLayersEqual(l, result_lyr, compare=compare, pk=pk): res = True break self.assertTrue(res, 'Could not find matching layer in expected results') diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 215160793c68..79378a18d392 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2501,6 +2501,10 @@ tests: OUTPUT: name: expected/stats_by_category.gml type: vector + pk: id2 + compare: + fields: + fid: skip # - algorithm: qgis:zonalstatistics # name: simple zonal statistics diff --git a/python/testing/__init__.py b/python/testing/__init__.py index a495a3356811..fc3eafe1fa02 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -54,6 +54,10 @@ def assertLayersEqual(self, layer_expected, layer_result, **kwargs): :keyword compare: A map of comparison options. e.g. { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } + :keyword pk: "Primary key" type field - used to match features + from the expected table to their corresponding features in the result table. If not specified + features are compared by their order in the layer (e.g. first feature compared with first feature, + etc) """ self.checkLayersEqual(layer_expected, layer_result, True, **kwargs) @@ -69,6 +73,10 @@ def checkLayersEqual(self, layer_expected, layer_result, use_asserts=False, **kw :keyword compare: A map of comparison options. e.g. { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } + :keyword pk: "Primary key" type field - used to match features + from the expected table to their corresponding features in the result table. If not specified + features are compared by their order in the layer (e.g. first feature compared with first feature, + etc) """ try: @@ -98,8 +106,14 @@ def checkLayersEqual(self, layer_expected, layer_result, use_asserts=False, **kw except KeyError: precision = 14 - expected_features = sorted(layer_expected.getFeatures(request), key=lambda f: f.id()) - result_features = sorted(layer_result.getFeatures(request), key=lambda f: f.id()) + def sort_by_pk_or_fid(f): + if 'pk' in kwargs and kwargs['pk'] is not None: + return f[kwargs['pk']] + else: + return f.id() + + expected_features = sorted(layer_expected.getFeatures(request), key=sort_by_pk_or_fid) + result_features = sorted(layer_result.getFeatures(request), key=sort_by_pk_or_fid) for feats in zip(expected_features, result_features): if feats[0].hasGeometry(): From 470afbebbe7063828cb0e2222c7e6ba4c16ae692 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 23:19:03 +1000 Subject: [PATCH 23/23] Use correct file filters for processing vector/raster input selectors --- .../processing/gui/ParameterGuiUtils.py | 21 ++++++++++++------- python/plugins/processing/gui/wrappers.py | 3 +-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/python/plugins/processing/gui/ParameterGuiUtils.py b/python/plugins/processing/gui/ParameterGuiUtils.py index 3694daf8e8ac..2931926e7e96 100644 --- a/python/plugins/processing/gui/ParameterGuiUtils.py +++ b/python/plugins/processing/gui/ParameterGuiUtils.py @@ -27,7 +27,7 @@ __revision__ = '$Format:%H$' from qgis.core import (QgsProcessing, - QgsProcessingParameterDefinition, + QgsProviderRegistry, QgsProcessingFeatureSourceDefinition, QgsVectorFileWriter) from qgis.PyQt.QtCore import QCoreApplication @@ -55,22 +55,29 @@ def getFileFilter(param): exts = QgsVectorFileWriter.supportedFormatExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'QgsProcessingParameterMultipleLayers').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) - elif param.type() in ('raster', 'rasterDestination'): + return tr('All files (*.*)') + ';;' + ';;'.join(exts) + elif param.type() == 'raster': + return QgsProviderRegistry.instance().fileRasterFilters() + elif param.type() == 'rasterDestination': exts = dataobjects.getSupportedOutputRasterLayerExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'QgsProcessingParameterRasterDestination').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) elif param.type() == 'table': exts = ['csv', 'dbf'] for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'ParameterTable').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) elif param.type() == 'sink': exts = QgsVectorFileWriter.supportedFormatExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'ParameterVector').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) + elif param.type() == 'source': + return QgsProviderRegistry.instance().fileVectorFilters() + elif param.type() == 'vector': + return QgsProviderRegistry.instance().fileVectorFilters() elif param.type() == 'fileOut': - return param.fileFilter() + return tr('All files (*.*)') + ';;' + param.fileFilter() + return '' diff --git a/python/plugins/processing/gui/wrappers.py b/python/plugins/processing/gui/wrappers.py index 2908bae1a3d8..7635ecc300d0 100644 --- a/python/plugins/processing/gui/wrappers.py +++ b/python/plugins/processing/gui/wrappers.py @@ -218,8 +218,7 @@ def getFileName(self, initial_value=''): path = '' filename, selected_filter = QFileDialog.getOpenFileName(self.widget, self.tr('Select file'), - path, self.tr( - 'All files (*.*);;') + getFileFilter(self.param)) + path, getFileFilter(self.param)) if filename: settings.setValue('/Processing/LastInputPath', os.path.dirname(str(filename)))