Skip to content

Commit

Permalink
[processing] Optimise feature requests within qgis algs
Browse files Browse the repository at this point in the history
- don't use setFilterFid() within loops to fetch features one
at time (as it's extremely slow), instead use setFilterFids()
outside the loop
- don't fetch unused attributes/geometry when it can be
avoided
  • Loading branch information
nyalldawson committed Oct 17, 2016
1 parent 55f2071 commit 86368f3
Show file tree
Hide file tree
Showing 22 changed files with 43 additions and 54 deletions.
6 changes: 3 additions & 3 deletions python/plugins/processing/algs/qgis/Difference.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,9 @@ def processAlgorithm(self, progress):
diff_geom = QgsGeometry(geom)
attrs = inFeatA.attributes()
intersections = index.intersects(geom.boundingBox())
for i in intersections:
request = QgsFeatureRequest().setFilterFid(i)
inFeatB = next(layerB.getFeatures(request))

request = QgsFeatureRequest().setFilterFids(intersections).setSubsetOfAttributes([])
for inFeatB in layerB.getFeatures(request):
tmpGeom = inFeatB.geometry()
if diff_geom.intersects(tmpGeom):
diff_geom = QgsGeometry(diff_geom.difference(tmpGeom))
Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/qgis/Eliminate.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ def processAlgorithm(self, progress):
geom2Eliminate = feat.geometry()
bbox = geom2Eliminate.boundingBox()
fit = processLayer.getFeatures(
QgsFeatureRequest().setFilterRect(bbox))
QgsFeatureRequest().setFilterRect(bbox).setSubsetOfAttributes([]))
mergeWithFid = None
mergeWithGeom = None
max = 0
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/ExtractByLocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,8 @@ def processAlgorithm(self, progress):
geom = vector.snapToPrecision(f.geometry(), precision)
bbox = vector.bufferedBoundingBox(geom.boundingBox(), 0.51 * precision)
intersects = index.intersects(bbox)
for i in intersects:
request = QgsFeatureRequest().setFilterFid(i)
feat = next(layer.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
for feat in layer.getFeatures(request):
tmpGeom = vector.snapToPrecision(feat.geometry(), precision)
res = False
for predicate in predicates:
Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/qgis/HubDistanceLines.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def processAlgorithm(self, progress):
src = f.geometry().boundingBox().center()

neighbors = index.nearestNeighbor(src, 1)
ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0])))
ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields())))
closest = ft.geometry().boundingBox().center()
hubDist = distance.measureLine(src, closest)

Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/qgis/HubDistancePoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def processAlgorithm(self, progress):
src = f.geometry().boundingBox().center()

neighbors = index.nearestNeighbor(src, 1)
ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0])))
ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields())))
closest = ft.geometry().boundingBox().center()
hubDist = distance.measureLine(src, closest)

Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/Intersection.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,8 @@ def processAlgorithm(self, progress):
geom = inFeatA.geometry()
atMapA = inFeatA.attributes()
intersects = index.intersects(geom.boundingBox())
for i in intersects:
request = QgsFeatureRequest().setFilterFid(i)
inFeatB = next(vlayerB.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(intersects)
for inFeatB in vlayerB.getFeatures(request):
tmpGeom = inFeatB.geometry()
if geom.intersects(tmpGeom):
atMapB = inFeatB.attributes()
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/LinesIntersection.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,8 @@ def processAlgorithm(self, progress):
hasIntersections = True

if hasIntersections:
for i in lines:
request = QgsFeatureRequest().setFilterFid(i)
inFeatB = next(layerB.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(lines)
for inFeatB in layerB.getFeatures(request):
tmpGeom = inFeatB.geometry()

points = []
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def processAlgorithm(self, progress):
for current, feat in enumerate(features):
neighbourID = spatialIndex.nearestNeighbor(
feat.geometry().asPoint(), 2)[1]
request = QgsFeatureRequest().setFilterFid(neighbourID)
request = QgsFeatureRequest().setFilterFid(neighbourID).setSubsetOfAttributes([])
neighbour = next(layer.getFeatures(request))
sumDist += distance.measureLine(neighbour.geometry().asPoint(),
feat.geometry().asPoint())
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/PointDistance.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,8 @@ def linearMatrix(self, inLayer, inField, targetLayer, targetField,
featList = index.nearestNeighbor(inGeom.asPoint(), nPoints)
distList = []
vari = 0.0
for i in featList:
request = QgsFeatureRequest().setFilterFid(i)
outFeat = next(targetLayer.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(featList).setSubsetOfAttributes([outIdx])
for outFeat in targetLayer.getFeatures(request):
outID = outFeat.attributes()[outIdx]
outGeom = outFeat.geometry()
dist = distArea.measureLine(inGeom.asPoint(),
Expand Down
9 changes: 4 additions & 5 deletions python/plugins/processing/algs/qgis/PointsDisplacement.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,10 @@ def processAlgorithm(self, progress):

fullPerimeter = 2 * math.pi

request = QgsFeatureRequest()
for (geom, fids) in list(duplicates.items()):
count = len(fids)
if count == 1:
f = next(layer.getFeatures(request.setFilterFid(fids[0])))
f = next(layer.getFeatures(QgsFeatureRequest().setFilterFid(fids[0])))
writer.addFeature(f)
else:
angleStep = fullPerimeter / count
Expand All @@ -100,14 +99,14 @@ def processAlgorithm(self, progress):
currentAngle = 0

old_point = QgsGeometry.fromWkt(geom).asPoint()
for fid in fids:

request = QgsFeatureRequest().setFilterFids(fids).setFlags(QgsFeatureRequest.NoGeometry)
for f in layer.getFeatures(request):
sinusCurrentAngle = math.sin(currentAngle)
cosinusCurrentAngle = math.cos(currentAngle)
dx = radius * sinusCurrentAngle
dy = radius * cosinusCurrentAngle

f = next(layer.getFeatures(request.setFilterFid(fid)))

new_point = QgsPoint(old_point.x() + dx, old_point.y()
+ dy)
out_feature = QgsFeature()
Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/qgis/PointsInPolygon.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def processAlgorithm(self, progress):
count = 0
points = spatialIndex.intersects(geom.boundingBox())
if len(points) > 0:
request = QgsFeatureRequest().setFilterFids(points)
request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([])
fit = pointLayer.getFeatures(request)
ftPoint = QgsFeature()
while fit.nextFeature(ftPoint):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def processAlgorithm(self, progress):
classes = set()
points = spatialIndex.intersects(geom.boundingBox())
if len(points) > 0:
request = QgsFeatureRequest().setFilterFids(points)
request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([classFieldIndex])
fit = pointLayer.getFeatures(request)
ftPoint = QgsFeature()
while fit.nextFeature(ftPoint):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def processAlgorithm(self, progress):
points = spatialIndex.intersects(geom.boundingBox())
if len(points) > 0:
progress.setText(str(len(points)))
request = QgsFeatureRequest().setFilterFids(points)
request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([fieldIdx])
fit = pointLayer.getFeatures(request)
ftPoint = QgsFeature()
while fit.nextFeature(ftPoint):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def processAlgorithm(self, progress):
while nIterations < maxIterations and nPoints < pointCount:
# pick random feature
fid = random.randint(0, featureCount - 1)
f = next(layer.getFeatures(request.setFilterFid(fid)))
f = next(layer.getFeatures(request.setFilterFid(fid).setSubsetOfAttributes([])))
fGeom = f.geometry()

if fGeom.isMultipart():
Expand Down
6 changes: 2 additions & 4 deletions python/plugins/processing/algs/qgis/RandomPointsLayer.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,6 @@ def processAlgorithm(self, progress):
index = QgsSpatialIndex()
points = dict()

request = QgsFeatureRequest()

random.seed()

while nIterations < maxIterations and nPoints < pointCount:
Expand All @@ -101,8 +99,8 @@ def processAlgorithm(self, progress):
ids = idxLayer.intersects(geom.buffer(5, 5).boundingBox())
if len(ids) > 0 and \
vector.checkMinDistance(pnt, index, minDistance, points):
for i in ids:
f = next(layer.getFeatures(request.setFilterFid(i)))
request = QgsFeatureRequest().setFilterFids(ids).setSubsetOfAttributes([])
for f in layer.getFeatures(request):
tmpGeom = f.geometry()
if geom.within(tmpGeom):
f = QgsFeature(nPoints)
Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/qgis/SelectByAttribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def processAlgorithm(self, progress):

qExp = QgsExpression(expr)
if not qExp.hasParserError():
qReq = QgsFeatureRequest(qExp)
qReq = QgsFeatureRequest(qExp).setSubsetOfAttributes([])
else:
raise GeoAlgorithmExecutionException(qExp.parserErrorString())
selected = [f.id() for f in layer.getFeatures(qReq)]
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/qgis/SelectByAttributeSum.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ def processAlgorithm(self, progress):
progress.setInfo(self.tr('No adjacent features found.'))
break

for i in intersected:
ft = next(layer.getFeatures(req.setFilterFid(i)))
req = QgsFeatureRequest().setFilterFids(intersected).setSubsetOfAttributes([fieldName], layer.fields())
for ft in layer.getFeatures(req):
tmpGeom = ft.geometry()
if tmpGeom.touches(geom):
geom = tmpGeom.combine(geom)
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/SelectByLocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,8 @@ def processAlgorithm(self, progress):
bbox = vector.bufferedBoundingBox(geom.boundingBox(), 0.51 * precision)
intersects = index.intersects(bbox)

for i in intersects:
request = QgsFeatureRequest().setFilterFid(i)
feat = next(inputLayer.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
for feat in inputLayer.getFeatures(request):
tmpGeom = vector.snapToPrecision(feat.geometry(), precision)

res = False
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/SplitLinesWithLines.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,8 @@ def processAlgorithm(self, progress):
if len(lines) > 0: # hasIntersections
splittingLines = []

for i in lines:
request = QgsFeatureRequest().setFilterFid(i)
inFeatB = next(layerB.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(lines).setSubsetOfAttributes([])
for inFeatB in layerB.getFeatures(request):
# check if trying to self-intersect
if sameLayer:
if inFeatA.id() == inFeatB.id():
Expand Down
5 changes: 2 additions & 3 deletions python/plugins/processing/algs/qgis/SumLines.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,8 @@ def processAlgorithm(self, progress):
hasIntersections = True

if hasIntersections:
for i in lines:
request = QgsFeatureRequest().setFilterFid(i)
ftLine = next(lineLayer.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(lines).setSubsetOfAttributes([])
for ftLine in lineLayer.getFeatures(request):
tmpGeom = ftLine.geometry()
if inGeom.intersects(tmpGeom):
outGeom = inGeom.intersection(tmpGeom)
Expand Down
8 changes: 4 additions & 4 deletions python/plugins/processing/algs/qgis/SymmetricalDifference.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ def processAlgorithm(self, progress):
diffGeom = QgsGeometry(geom)
attrs = featA.attributes()
intersects = indexA.intersects(geom.boundingBox())
for i in intersects:
layerB.getFeatures(QgsFeatureRequest().setFilterFid(i)).nextFeature(featB)
request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
for featB in layerB.getFeatures(request):
tmpGeom = featB.geometry()
if diffGeom.intersects(tmpGeom):
diffGeom = QgsGeometry(diffGeom.difference(tmpGeom))
Expand Down Expand Up @@ -123,8 +123,8 @@ def processAlgorithm(self, progress):
attrs = featA.attributes()
attrs = [NULL] * length + attrs
intersects = indexB.intersects(geom.boundingBox())
for i in intersects:
layerA.getFeatures(QgsFeatureRequest().setFilterFid(i)).nextFeature(featB)
request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
for featB in layerA.getFeatures(request):
tmpGeom = featB.geometry()
if diffGeom.intersects(tmpGeom):
diffGeom = QgsGeometry(diffGeom.difference(tmpGeom))
Expand Down
11 changes: 5 additions & 6 deletions python/plugins/processing/algs/qgis/Union.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,10 @@ def processAlgorithm(self, progress):
ProcessingLog.addToLog(ProcessingLog.LOG_INFO,
self.tr('Feature geometry error: One or more output features ignored due to invalid geometry.'))
else:
for id in intersects:
request = QgsFeatureRequest().setFilterFids(intersects)
for inFeatB in vlayerB.getFeatures(request):
count += 1
request = QgsFeatureRequest().setFilterFid(id)
inFeatB = next(vlayerB.getFeatures(request))

atMapB = inFeatB.attributes()
tmpGeom = inFeatB.geometry()

Expand Down Expand Up @@ -197,9 +197,8 @@ def processAlgorithm(self, progress):
ProcessingLog.addToLog(ProcessingLog.LOG_INFO,
self.tr('Feature geometry error: One or more output features ignored due to invalid geometry.'))
else:
for id in intersects:
request = QgsFeatureRequest().setFilterFid(id)
inFeatB = next(vlayerA.getFeatures(request))
request = QgsFeatureRequest().setFilterFids(intersects)
for inFeatB in vlayerA.getFeatures(request):
atMapB = inFeatB.attributes()
tmpGeom = inFeatB.geometry()

Expand Down

1 comment on commit 86368f3

@m-kuhn
Copy link
Member

@m-kuhn m-kuhn commented on 86368f3 Oct 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good job! 👍

Please sign in to comment.