Skip to content

Commit 4b5d81b

Browse files
arnaud-morvanarnaud-morvan
committed
[processing] Add Aggregate algorithm
1 parent b26e681 commit 4b5d81b

16 files changed

+1031
-4
lines changed

‎python/plugins/processing/algs/help/qgis.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,17 @@ qgis:adduniquevalueindexfield: >
2020
qgis:advancedpythonfieldcalculator: >
2121
This algorithm adds a new attribute to a vector layer, with values resulting from applying an expression to each feature. The expression is defined as a Python function.
2222

23+
qgis:aggregate: >
24+
This algorithm take a vector or table layer and aggregate features based on a group by expression. Features for which group by expression return the same value are grouped together.
25+
26+
It is possible to group all source features together using constant value in group by parameter, example: NULL.
27+
28+
It is also possible to group features using multiple fields using Array function, example: Array("Field1", "Field2").
29+
30+
Geometries (if present) are combined into one multipart geometry for each group.
31+
32+
Output attributes are computed depending on each given aggregate definition.
33+
2334
qgis:barplot:
2435

2536
qgis:basicstatisticsforfields: >

‎python/plugins/processing/algs/qgis/Aggregate.py

+269
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
# -*- coding: utf-8 -*-
2+
3+
"""
4+
***************************************************************************
5+
Aggregate.py
6+
---------------------
7+
Date : February 2017
8+
Copyright : (C) 2017 by Arnaud Morvan
9+
Email : arnaud dot morvan at camptocamp dot com
10+
***************************************************************************
11+
* *
12+
* This program is free software; you can redistribute it and/or modify *
13+
* it under the terms of the GNU General Public License as published by *
14+
* the Free Software Foundation; either version 2 of the License, or *
15+
* (at your option) any later version. *
16+
* *
17+
***************************************************************************
18+
"""
19+
20+
__author__ = 'Arnaud Morvan'
21+
__date__ = 'February 2017'
22+
__copyright__ = '(C) 2017, Arnaud Morvan'
23+
24+
# This will get replaced with a git SHA1 when you do a git archive
25+
26+
__revision__ = '$Format:%H$'
27+
28+
from qgis.core import (
29+
QgsDistanceArea,
30+
QgsExpression,
31+
QgsExpressionContextUtils,
32+
QgsFeature,
33+
QgsFeatureSink,
34+
QgsField,
35+
QgsFields,
36+
QgsGeometry,
37+
QgsProcessingParameterDefinition,
38+
QgsProcessingParameterExpression,
39+
QgsProcessingParameterFeatureSink,
40+
QgsProcessingParameterFeatureSource,
41+
QgsProcessingException,
42+
QgsProcessingUtils,
43+
QgsWkbTypes,
44+
)
45+
46+
from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm
47+
48+
49+
class Aggregate(QgisAlgorithm):
50+
51+
INPUT = 'INPUT'
52+
GROUP_BY = 'GROUP_BY'
53+
AGGREGATES = 'AGGREGATES'
54+
DISSOLVE = 'DISSOLVE'
55+
OUTPUT = 'OUTPUT'
56+
57+
def group(self):
58+
return self.tr('Vector geometry tools')
59+
60+
def name(self):
61+
return 'aggregate'
62+
63+
def displayName(self):
64+
return self.tr('Aggregate')
65+
66+
def initAlgorithm(self, config=None):
67+
self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT,
68+
self.tr('Input layer')))
69+
self.addParameter(QgsProcessingParameterExpression(self.GROUP_BY,
70+
self.tr('Group by expression (NULL to group all features)'),
71+
defaultValue='NULL',
72+
optional=False,
73+
parentLayerParameterName=self.INPUT))
74+
75+
class ParameterAggregates(QgsProcessingParameterDefinition):
76+
77+
def __init__(self, name, description, parentLayerParameterName='INPUT'):
78+
super().__init__(name, description)
79+
self._parentLayerParameter = parentLayerParameterName
80+
81+
def type(self):
82+
return 'aggregates'
83+
84+
def checkValueIsAcceptable(self, value, context=None):
85+
if not isinstance(value, list):
86+
return False
87+
for field_def in value:
88+
if not isinstance(field_def, dict):
89+
return False
90+
if not field_def.get('input', False):
91+
return False
92+
if not field_def.get('aggregate', False):
93+
return False
94+
if not field_def.get('name', False):
95+
return False
96+
if not field_def.get('type', False):
97+
return False
98+
return True
99+
100+
def valueAsPythonString(self, value, context):
101+
return str(value)
102+
103+
def asScriptCode(self):
104+
raise NotImplementedError()
105+
106+
@classmethod
107+
def fromScriptCode(cls, name, description, isOptional, definition):
108+
raise NotImplementedError()
109+
110+
def parentLayerParameter(self):
111+
return self._parentLayerParameter
112+
113+
self.addParameter(ParameterAggregates(self.AGGREGATES,
114+
description=self.tr('Aggregates')))
115+
self.parameterDefinition(self.AGGREGATES).setMetadata({
116+
'widget_wrapper': 'processing.algs.qgis.ui.AggregatesPanel.AggregatesWidgetWrapper'
117+
})
118+
119+
self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT,
120+
self.tr('Aggregated')))
121+
122+
def parameterAsAggregates(self, parameters, name, context):
123+
return parameters[name]
124+
125+
def prepareAlgorithm(self, parameters, context, feedback):
126+
source = self.parameterAsSource(parameters, self.INPUT, context)
127+
group_by = self.parameterAsExpression(parameters, self.GROUP_BY, context)
128+
aggregates = self.parameterAsAggregates(parameters, self.AGGREGATES, context)
129+
130+
da = QgsDistanceArea()
131+
da.setSourceCrs(source.sourceCrs())
132+
da.setEllipsoid(context.project().ellipsoid())
133+
134+
self.source = source
135+
self.group_by = group_by
136+
self.group_by_expr = self.createExpression(group_by, da, context)
137+
self.geometry_expr = self.createExpression('collect($geometry, {})'.format(group_by), da, context)
138+
139+
self.fields = QgsFields()
140+
self.fields_expr = []
141+
for field_def in aggregates:
142+
self.fields.append(QgsField(name=field_def['name'],
143+
type=field_def['type'],
144+
typeName="",
145+
len=field_def['length'],
146+
prec=field_def['precision']))
147+
aggregate = field_def['aggregate']
148+
if aggregate == 'first_value':
149+
expression = field_def['input']
150+
elif aggregate == 'concatenate':
151+
expression = ('{}({}, {}, {}, \'{}\')'
152+
.format(field_def['aggregate'],
153+
field_def['input'],
154+
group_by,
155+
'TRUE',
156+
field_def['delimiter']))
157+
else:
158+
expression = '{}({}, {})'.format(field_def['aggregate'],
159+
field_def['input'],
160+
group_by)
161+
expr = self.createExpression(expression, da, context)
162+
self.fields_expr.append(expr)
163+
return True
164+
165+
def processAlgorithm(self, parameters, context, feedback):
166+
expr_context = self.createExpressionContext(parameters, context)
167+
self.group_by_expr.prepare(expr_context)
168+
169+
# Group features in memory layers
170+
source = self.source
171+
count = self.source.featureCount()
172+
if count:
173+
progress_step = 50.0 / count
174+
current = 0
175+
groups = {}
176+
keys = [] # We need deterministic order for the tests
177+
feature = QgsFeature()
178+
for feature in self.source.getFeatures():
179+
expr_context.setFeature(feature)
180+
group_by_value = self.evaluateExpression(self.group_by_expr, expr_context)
181+
182+
# Get an hashable key for the dict
183+
key = group_by_value
184+
if isinstance(key, list):
185+
key = tuple(key)
186+
187+
group = groups.get(key, None)
188+
if group is None:
189+
sink, id = QgsProcessingUtils.createFeatureSink(
190+
'memory:',
191+
context,
192+
source.fields(),
193+
source.wkbType(),
194+
source.sourceCrs())
195+
layer = QgsProcessingUtils.mapLayerFromString(id, context)
196+
group = {
197+
'sink': sink,
198+
'layer': layer,
199+
'feature': feature
200+
}
201+
groups[key] = group
202+
keys.append(key)
203+
204+
group['sink'].addFeature(feature, QgsFeatureSink.FastInsert)
205+
206+
current += 1
207+
feedback.setProgress(int(current * progress_step))
208+
if feedback.isCanceled():
209+
return
210+
211+
(sink, dest_id) = self.parameterAsSink(parameters,
212+
self.OUTPUT,
213+
context,
214+
self.fields,
215+
QgsWkbTypes.multiType(source.wkbType()),
216+
source.sourceCrs())
217+
218+
# Calculate aggregates on memory layers
219+
if len(keys):
220+
progress_step = 50.0 / len(keys)
221+
for current, key in enumerate(keys):
222+
group = groups[key]
223+
expr_context = self.createExpressionContext(parameters, context)
224+
expr_context.appendScope(QgsExpressionContextUtils.layerScope(group['layer']))
225+
expr_context.setFeature(group['feature'])
226+
227+
geometry = self.evaluateExpression(self.geometry_expr, expr_context)
228+
if geometry is not None and not geometry.isEmpty():
229+
geometry = QgsGeometry.unaryUnion(geometry.asGeometryCollection())
230+
if geometry.isEmpty():
231+
raise QgsProcessingException(
232+
'Impossible to combine geometries for {} = {}'
233+
.format(self.group_by, group_by_value))
234+
235+
attrs = []
236+
for fields_expr in self.fields_expr:
237+
attrs.append(self.evaluateExpression(fields_expr, expr_context))
238+
239+
# Write output feature
240+
outFeat = QgsFeature()
241+
if geometry is not None:
242+
outFeat.setGeometry(geometry)
243+
outFeat.setAttributes(attrs)
244+
sink.addFeature(outFeat, QgsFeatureSink.FastInsert)
245+
246+
feedback.setProgress(50 + int(current * progress_step))
247+
if feedback.isCanceled():
248+
return
249+
250+
return {self.OUTPUT: dest_id}
251+
252+
def createExpression(self, text, da, context):
253+
expr = QgsExpression(text)
254+
expr.setGeomCalculator(da)
255+
expr.setDistanceUnits(context.project().distanceUnits())
256+
expr.setAreaUnits(context.project().areaUnits())
257+
if expr.hasParserError():
258+
raise QgsProcessingException(
259+
self.tr(u'Parser error in expression "{}": {}')
260+
.format(text, expr.parserErrorString()))
261+
return expr
262+
263+
def evaluateExpression(self, expr, context):
264+
value = expr.evaluate(context)
265+
if expr.hasEvalError():
266+
raise QgsProcessingException(
267+
self.tr(u'Evaluation error in expression "{}": {}')
268+
.format(expr.expression(), expr.evalErrorString()))
269+
return value

‎python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py

+2
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from .QgisAlgorithm import QgisAlgorithm
4242

4343
from .AddTableField import AddTableField
44+
from .Aggregate import Aggregate
4445
from .Aspect import Aspect
4546
from .AutoincrementalField import AutoincrementalField
4647
from .BasicStatistics import BasicStatisticsForField
@@ -204,6 +205,7 @@ def getAlgs(self):
204205
# ExecuteSQL(), FindProjection(),
205206
# ]
206207
algs = [AddTableField(),
208+
Aggregate(),
207209
Aspect(),
208210
AutoincrementalField(),
209211
BasicStatisticsForField(),

0 commit comments

Comments
 (0)