[FEATURE] Optimise processing clip algorithm

Before the algorithm was written to optimise clipping a few
features against thousands of mask features. The revised algorithm
is optimised for clipping thousands of input features against
a few mask features.

Given that this second operation is much more likely, it makes
sense to optimise for this use case.

I've also applied some other optimisations like taking advantage
of spatial indexes on the providers, using prepared geometries
and also only applying an intersection operation if the geometry
isn't wholly contained by the mask geometry.

Benchmarks:

clipping roads layer with 1 million lines against 2 polygons

before: 5 mins 30 seconds
after: 10 seconds

clipping address layer with 5 million points against 2 polygons

before: 50 minutes
after: 30 seconds

python/plugins/processing/algs/qgis/Clip.py

@@ -60,76 +60,69 @@ def defineCharacteristics(self):
         self.addOutput(OutputVector(Clip.OUTPUT, self.tr('Clipped')))
 
     def processAlgorithm(self, progress):
-        layerA = dataobjects.getObjectFromUri(
+        sourceLayer = dataobjects.getObjectFromUri(
             self.getParameterValue(Clip.INPUT))
-        layerB = dataobjects.getObjectFromUri(
+        maskLayer = dataobjects.getObjectFromUri(
             self.getParameterValue(Clip.OVERLAY))
 
         writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
-            layerA.pendingFields(),
-            layerA.dataProvider().geometryType(),
-            layerA.dataProvider().crs())
+            sourceLayer.pendingFields(),
+            sourceLayer.dataProvider().geometryType(),
+            sourceLayer.dataProvider().crs())
 
         inFeatA = QgsFeature()
         inFeatB = QgsFeature()
         outFeat = QgsFeature()
 
-        index = vector.spatialindex(layerB)
-
-        selectionA = vector.features(layerA)
-
-        total = 100.0 / len(selectionA)
-
-        for current, inFeatA in enumerate(selectionA):
-            geom = inFeatA.geometry()
-            attrs = inFeatA.attributes()
-            intersects = index.intersects(geom.boundingBox())
-            first = True
-            found = False
-            if len(intersects) > 0:
-                for i in intersects:
-                    layerB.getFeatures(
-                        QgsFeatureRequest().setFilterFid(i)).nextFeature(
-                            inFeatB)
-                    tmpGeom = inFeatB.geometry()
-                    if tmpGeom.intersects(geom):
-                        found = True
-                        if first:
-                            outFeat.setGeometry(QgsGeometry(tmpGeom))
-                            first = False
-                        else:
-                            cur_geom = outFeat.geometry()
-                            new_geom = QgsGeometry(cur_geom.combine(tmpGeom))
-                            if new_geom.isEmpty() or new_geom.isGeosEmpty() or not new_geom.isGeosValid():
-                                ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
-                                                       self.tr('GEOS geoprocessing error: One or '
-                                                               'more input features have invalid '
-                                                               'geometry.'))
-                                break
-
-                            outFeat.setGeometry(QgsGeometry(new_geom))
-                if found:
-                    cur_geom = outFeat.geometry()
-                    new_geom = QgsGeometry(geom.intersection(cur_geom))
-                    if new_geom.wkbType() == Qgis.WKBUnknown or QgsWKBTypes.flatType(new_geom.geometry().wkbType()) == QgsWKBTypes.GeometryCollection:
-                        int_com = QgsGeometry(geom.combine(cur_geom))
-                        int_sym = QgsGeometry(geom.symDifference(cur_geom))
-                        new_geom = QgsGeometry(int_com.difference(int_sym))
-                        if new_geom.isGeosEmpty() or not new_geom.isGeosValid():
-                            ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
-                                                   self.tr('GEOS geoprocessing error: One or more '
-                                                           'input features have invalid geometry.'))
-                            continue
-                    try:
-                        outFeat.setGeometry(new_geom)
-                        outFeat.setAttributes(attrs)
-                        writer.addFeature(outFeat)
-                    except:
+        # first build up a list of clip geometries
+        clip_geoms = []
+        for maskFeat in vector.features(maskLayer, QgsFeatureRequest().setSubsetOfAttributes([])):
+            clip_geoms.append(maskFeat.geometry())
+
+        if len(clip_geoms) > 1:
+            combined_clip_geom = QgsGeometry.unaryUnion(clip_geoms)
+        else:
+            combined_clip_geom = clip_geoms[0]
+
+        # use prepared geometries for faster insection tests
+        engine = QgsGeometry.createGeometryEngine(combined_clip_geom.geometry())
+        engine.prepareGeometry()
+
+        input_features = [f for f in vector.features(sourceLayer, QgsFeatureRequest().setFilterRect(combined_clip_geom.boundingBox()))]
+        total = 100.0 / len(input_features)
+        for current, in_feat in enumerate(input_features):
+            if not in_feat.geometry():
+                continue
+
+            if not engine.intersects(in_feat.geometry().geometry()):
+                continue
+
+            if not engine.contains(in_feat.geometry().geometry()):
+                cur_geom = in_feat.geometry()
+                new_geom = combined_clip_geom.intersection(cur_geom)
+                if new_geom.wkbType() == Qgis.WKBUnknown or QgsWKBTypes.flatType(new_geom.geometry().wkbType()) == QgsWKBTypes.GeometryCollection:
+                    int_com = in_feat.geometry().combine(new_geom)
+                    int_sym = in_feat.geometry().symDifference(new_geom)
+                    new_geom = int_com.difference(int_sym)
+                    if new_geom.isGeosEmpty() or not new_geom.isGeosValid():
                         ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
-                                               self.tr('Feature geometry error: One or more '
-                                                       'output features ignored due to '
-                                                       'invalid geometry.'))
-                        continue
+                                               self.tr('GEOS geoprocessing error: One or more '
+                                                       'input features have invalid geometry.'))
+            else:
+                # clip geometry totally contains feature geometry, so no need to perform intersection
+                new_geom = in_feat.geometry()
+
+            try:
+                outFeat = QgsFeature()
+                outFeat.setGeometry(new_geom)
+                outFeat.setAttributes(in_feat.attributes())
+                writer.addFeature(outFeat)
+            except:
+                ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
+                                       self.tr('Feature geometry error: One or more '
+                                               'output features ignored due to '
+                                               'invalid geometry.'))
+                continue
 
             progress.setPercentage(int(current * total))
 

python/plugins/processing/tests/testdata/expected/clip_lines_by_multipolygon.gml

@@ -23,7 +23,7 @@
   </gml:featureMember>
   <gml:featureMember>
     <ogr:clip_lines_by_multipolygon fid="lines.3">
-      <ogr:geometryProperty><gml:LineString srsName="EPSG:4326"><gml:coordinates>3,1 4,1</gml:coordinates></gml:LineString></ogr:geometryProperty>
+      <ogr:geometryProperty><gml:LineString srsName="EPSG:4326"><gml:coordinates>4,1 3,1</gml:coordinates></gml:LineString></ogr:geometryProperty>
     </ogr:clip_lines_by_multipolygon>
   </gml:featureMember>
   <gml:featureMember>

python/plugins/processing/tests/testdata/expected/clip_multipolygons_by_polygons.gml

@@ -13,15 +13,15 @@
 
   <gml:featureMember>
     <ogr:clip_multipolygons_by_polygons fid="multipolys.0">
-      <ogr:geometryProperty><gml:MultiPolygon srsName="EPSG:4326"><gml:polygonMember><gml:Polygon><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>4.0,1.666666666666667 4,1 2,1 2,2 3,2 4.0,1.666666666666667</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></gml:polygonMember></gml:MultiPolygon></ogr:geometryProperty>
+      <ogr:geometryProperty><gml:MultiPolygon srsName="EPSG:4326"><gml:polygonMember><gml:Polygon><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2 1, 2 2, 3 2, 4 1.66666666666667, 4 1, 2 1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></gml:polygonMember></gml:MultiPolygon></ogr:geometryProperty>
       <ogr:Bname>Test</ogr:Bname>
       <ogr:Bintval>1</ogr:Bintval>
       <ogr:Bfloatval>0.123</ogr:Bfloatval>
     </ogr:clip_multipolygons_by_polygons>
   </gml:featureMember>
   <gml:featureMember>
     <ogr:clip_multipolygons_by_polygons fid="multipolys.1">
-      <ogr:geometryProperty><gml:MultiPolygon srsName="EPSG:4326"><gml:polygonMember><gml:Polygon><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>8,1 8,0 7,0 7,1 8,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></gml:polygonMember></gml:MultiPolygon></ogr:geometryProperty>
+      <ogr:geometryProperty><gml:MultiPolygon srsName="EPSG:4326"><gml:polygonMember><gml:Polygon><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>7 0, 7 1, 8 1, 8 0, 7 0</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></gml:polygonMember></gml:MultiPolygon></ogr:geometryProperty>
     </ogr:clip_multipolygons_by_polygons>
   </gml:featureMember>
   <gml:featureMember>

python/plugins/processing/tests/testdata/expected/clip_polys_by_multipolygon.gml

@@ -21,14 +21,14 @@
   </gml:featureMember>
   <gml:featureMember>
     <ogr:clip_polys_by_multipolygon fid="polys.3">
-      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>8,0 7,0 7,1 8,1 8,0</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>7 1, 8 1, 8 0, 7 0, 7 1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
       <ogr:name>ASDF</ogr:name>
       <ogr:intval>0</ogr:intval>
     </ogr:clip_polys_by_multipolygon>
   </gml:featureMember>
   <gml:featureMember>
     <ogr:clip_polys_by_multipolygon fid="polys.5">
-      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 4.0,1.666666666666667 4,1 2,1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2 1, 2 2, 3 2, 4 1.66666666666667, 4 1, 2 1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
       <ogr:name>elim</ogr:name>
       <ogr:intval>2</ogr:intval>
       <ogr:floatval>3.33</ogr:floatval>