Refactor processing writer to use QgsFeatureSink, and to store

temporary output layers in the provided processing context Should fix loss of intermediate memory layers when running a multi-step model
qgis · Apr 26, 2017 · fd2c18e
1 parent 946f0fa
commit fd2c18e
Show file tree

Hide file tree

Showing 123 changed files with 327 additions and 478 deletions.
diff --git a/python/core/processing/qgsprocessingcontext.sip b/python/core/processing/qgsprocessingcontext.sip
@@ -75,8 +75,12 @@ class QgsProcessingContext
  Sets the expression ``context``.
 %End
 
-
-
+    QgsProject &temporaryLayerStore();
+%Docstring
+ Returns a reference to the project used for storing temporary layers during
+ algorithm execution.
+ :rtype: QgsProject
+%End
 
     QgsFeatureRequest::InvalidGeometryCheck invalidGeometryCheck() const;
 %Docstring

diff --git a/python/plugins/processing/algs/qgis/AddTableField.py b/python/plugins/processing/algs/qgis/AddTableField.py
@@ -97,8 +97,7 @@ def processAlgorithm(self, context, feedback):
         fields = layer.fields()
         fields.append(QgsField(fieldName, self.TYPES[fieldType], '',
                                fieldLength, fieldPrecision))
-        writer = output.getVectorWriter(fields, layer.wkbType(),
-                                        layer.crs())
+        writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
         outFeat = QgsFeature()
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/AutoincrementalField.py b/python/plugins/processing/algs/qgis/AutoincrementalField.py
@@ -67,8 +67,7 @@ def processAlgorithm(self, context, feedback):
             dataobjects.getLayerFromString(self.getParameterValue(self.INPUT))
         fields = vlayer.fields()
         fields.append(QgsField('AUTO', QVariant.Int))
-        writer = output.getVectorWriter(fields, vlayer.wkbType(),
-                                        vlayer.crs())
+        writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
         outFeat = QgsFeature()
         features = QgsProcessingUtils.getFeatures(vlayer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(vlayer, context)

diff --git a/python/plugins/processing/algs/qgis/Boundary.py b/python/plugins/processing/algs/qgis/Boundary.py
@@ -78,10 +78,7 @@ def processAlgorithm(self, context, feedback):
             output_wkb = QgsWkbTypes.addM(output_wkb)
 
         writer = self.getOutputFromName(
-            self.OUTPUT_LAYER).getVectorWriter(
-                layer.fields(),
-                output_wkb,
-                layer.crs())
+            self.OUTPUT_LAYER).getVectorWriter(layer.fields(), output_wkb, layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/BoundingBox.py b/python/plugins/processing/algs/qgis/BoundingBox.py
@@ -67,10 +67,7 @@ def processAlgorithm(self, context, feedback):
             self.getParameterValue(self.INPUT_LAYER))
 
         writer = self.getOutputFromName(
-            self.OUTPUT_LAYER).getVectorWriter(
-                layer.fields().toList(),
-                QgsWkbTypes.Polygon,
-                layer.crs())
+            self.OUTPUT_LAYER).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/Centroids.py b/python/plugins/processing/algs/qgis/Centroids.py
@@ -69,10 +69,7 @@ def processAlgorithm(self, context, feedback):
             self.getParameterValue(self.INPUT_LAYER))
 
         writer = self.getOutputFromName(
-            self.OUTPUT_LAYER).getVectorWriter(
-                layer.fields(),
-                QgsWkbTypes.Point,
-                layer.crs())
+            self.OUTPUT_LAYER).getVectorWriter(layer.fields(), QgsWkbTypes.Point, layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/CheckValidity.py b/python/plugins/processing/algs/qgis/CheckValidity.py
@@ -108,32 +108,23 @@ def doCheck(self, context, feedback):
 
         valid_output = self.getOutputFromName(self.VALID_OUTPUT)
         valid_fields = layer.fields()
-        valid_writer = valid_output.getVectorWriter(
-            valid_fields,
-            layer.wkbType(),
-            layer.crs())
+        valid_writer = valid_output.getVectorWriter(valid_fields, layer.wkbType(), layer.crs(), context)
         valid_count = 0
 
         invalid_output = self.getOutputFromName(self.INVALID_OUTPUT)
         invalid_fields = layer.fields().toList() + [
             QgsField(name='_errors',
                      type=QVariant.String,
                      len=255)]
-        invalid_writer = invalid_output.getVectorWriter(
-            invalid_fields,
-            layer.wkbType(),
-            layer.crs())
+        invalid_writer = invalid_output.getVectorWriter(invalid_fields, layer.wkbType(), layer.crs(), context)
         invalid_count = 0
 
         error_output = self.getOutputFromName(self.ERROR_OUTPUT)
         error_fields = [
             QgsField(name='message',
                      type=QVariant.String,
                      len=255)]
-        error_writer = error_output.getVectorWriter(
-            error_fields,
-            QgsWkbTypes.Point,
-            layer.crs())
+        error_writer = error_output.getVectorWriter(error_fields, QgsWkbTypes.Point, layer.crs(), context)
         error_count = 0
 
         features = QgsProcessingUtils.getFeatures(layer, context)

diff --git a/python/plugins/processing/algs/qgis/Clip.py b/python/plugins/processing/algs/qgis/Clip.py
@@ -75,10 +75,9 @@ def processAlgorithm(self, context, feedback):
         mask_layer = dataobjects.getLayerFromString(
             self.getParameterValue(Clip.OVERLAY))
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
-            source_layer.fields(),
-            QgsWkbTypes.multiType(source_layer.wkbType()),
-            source_layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(source_layer.fields(),
+                                                                     QgsWkbTypes.multiType(source_layer.wkbType()),
+                                                                     source_layer.crs(), context)
 
         # first build up a list of clip geometries
         clip_geoms = []

diff --git a/python/plugins/processing/algs/qgis/ConcaveHull.py b/python/plugins/processing/algs/qgis/ConcaveHull.py
@@ -133,8 +133,8 @@ def processAlgorithm(self, context, feedback):
         feedback.setProgressText(self.tr('Saving data...'))
         feat = QgsFeature()
         dissolved_layer.getFeatures(QgsFeatureRequest().setFilterFid(0)).nextFeature(feat)
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
-            layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon,
+                                                                     layer.crs(), context)
         geom = feat.geometry()
         if no_multigeom and geom.isMultipart():
             # Only singlepart geometries are allowed

diff --git a/python/plugins/processing/algs/qgis/ConvexHull.py b/python/plugins/processing/algs/qgis/ConvexHull.py
@@ -103,8 +103,7 @@ def processAlgorithm(self, context, feedback):
                   QgsField('perim', QVariant.Double, '', 20, 6)
                   ]
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
-            fields, QgsWkbTypes.Polygon, layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)
 
         outFeat = QgsFeature()
         inGeom = QgsGeometry()

diff --git a/python/plugins/processing/algs/qgis/Delaunay.py b/python/plugins/processing/algs/qgis/Delaunay.py
@@ -77,8 +77,7 @@ def processAlgorithm(self, context, feedback):
                   QgsField('POINTB', QVariant.Double, '', 24, 15),
                   QgsField('POINTC', QVariant.Double, '', 24, 15)]
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
-                                                                     QgsWkbTypes.Polygon, layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)
 
         pts = []
         ptDict = {}

diff --git a/python/plugins/processing/algs/qgis/DeleteColumn.py b/python/plugins/processing/algs/qgis/DeleteColumn.py
@@ -83,8 +83,7 @@ def processAlgorithm(self, context, feedback):
         for index in field_indices:
             fields.remove(index)
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
-                                                                     layer.wkbType(), layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/DeleteDuplicateGeometries.py b/python/plugins/processing/algs/qgis/DeleteDuplicateGeometries.py
@@ -65,8 +65,7 @@ def processAlgorithm(self, context, feedback):
 
         fields = layer.fields()
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
-                                                                     layer.wkbType(), layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
 

diff --git a/python/plugins/processing/algs/qgis/DeleteHoles.py b/python/plugins/processing/algs/qgis/DeleteHoles.py
@@ -77,10 +77,8 @@ def processAlgorithm(self, context, feedback):
         if min_area == 0.0:
             min_area = -1.0
 
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
-            layer.fields(),
-            layer.wkbType(),
-            layer.crs())
+        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
+                                                                     context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/DensifyGeometries.py b/python/plugins/processing/algs/qgis/DensifyGeometries.py
@@ -85,8 +85,7 @@ def processAlgorithm(self, context, feedback):
         isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry
 
         writer = self.getOutputFromName(
-            self.OUTPUT).getVectorWriter(layer.fields().toList(),
-                                         layer.wkbType(), layer.crs())
+            self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/DensifyGeometriesInterval.py b/python/plugins/processing/algs/qgis/DensifyGeometriesInterval.py
@@ -79,8 +79,7 @@ def processAlgorithm(self, context, feedback):
         isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry
 
         writer = self.getOutputFromName(
-            self.OUTPUT).getVectorWriter(layer.fields().toList(),
-                                         layer.wkbType(), layer.crs())
+            self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(layer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(layer, context)

diff --git a/python/plugins/processing/algs/qgis/Difference.py b/python/plugins/processing/algs/qgis/Difference.py
@@ -77,9 +77,7 @@ def processAlgorithm(self, context, feedback):
 
         geomType = QgsWkbTypes.multiType(layerA.wkbType())
         writer = self.getOutputFromName(
-            Difference.OUTPUT).getVectorWriter(layerA.fields(),
-                                               geomType,
-                                               layerA.crs())
+            Difference.OUTPUT).getVectorWriter(layerA.fields(), geomType, layerA.crs(), context)
 
         outFeat = QgsFeature()
         index = vector.spatialindex(layerB)

diff --git a/python/plugins/processing/algs/qgis/Dissolve.py b/python/plugins/processing/algs/qgis/Dissolve.py
@@ -82,10 +82,7 @@ def processAlgorithm(self, context, feedback):
             self.getParameterValue(Dissolve.INPUT))
 
         writer = self.getOutputFromName(
-            Dissolve.OUTPUT).getVectorWriter(
-                vlayerA.fields().toList(),
-                vlayerA.wkbType(),
-                vlayerA.crs())
+            Dissolve.OUTPUT).getVectorWriter(vlayerA.fields().toList(), vlayerA.wkbType(), vlayerA.crs(), context)
 
         outFeat = QgsFeature()
         features = QgsProcessingUtils.getFeatures(vlayerA, context)

diff --git a/python/plugins/processing/algs/qgis/DropGeometry.py b/python/plugins/processing/algs/qgis/DropGeometry.py
@@ -70,10 +70,8 @@ def processAlgorithm(self, context, feedback):
         layer = dataobjects.getLayerFromString(
             self.getParameterValue(self.INPUT_LAYER))
         writer = self.getOutputFromName(
-            self.OUTPUT_TABLE).getVectorWriter(
-                layer.fields(),
-                QgsWkbTypes.NoGeometry,
-                QgsCoordinateReferenceSystem())
+            self.OUTPUT_TABLE).getVectorWriter(layer.fields(), QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem(),
+                                               context)
 
         request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)
         features = QgsProcessingUtils.getFeatures(layer, context, request)

diff --git a/python/plugins/processing/algs/qgis/EliminateSelection.py b/python/plugins/processing/algs/qgis/EliminateSelection.py
@@ -92,8 +92,7 @@ def processAlgorithm(self, context, feedback):
         featToEliminate = []
         selFeatIds = inLayer.selectedFeatureIds()
         output = self.getOutputFromName(self.OUTPUT)
-        writer = output.getVectorWriter(inLayer.fields(),
-                                        inLayer.wkbType(), inLayer.crs())
+        writer = output.getVectorWriter(inLayer.fields(), inLayer.wkbType(), inLayer.crs(), context)
 
         for aFeat in inLayer.getFeatures():
             if aFeat.id() in selFeatIds:

diff --git a/python/plugins/processing/algs/qgis/EquivalentNumField.py b/python/plugins/processing/algs/qgis/EquivalentNumField.py
@@ -73,8 +73,7 @@ def processAlgorithm(self, context, feedback):
         fieldindex = vlayer.fields().lookupField(fieldname)
         fields = vlayer.fields()
         fields.append(QgsField('NUM_FIELD', QVariant.Int))
-        writer = output.getVectorWriter(fields, vlayer.wkbType(),
-                                        vlayer.crs())
+        writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
         outFeat = QgsFeature()
         classes = {}
 

diff --git a/python/plugins/processing/algs/qgis/ExecuteSQL.py b/python/plugins/processing/algs/qgis/ExecuteSQL.py
@@ -146,11 +146,9 @@ def processAlgorithm(self, context, feedback):
         if not vLayer.isValid():
             raise GeoAlgorithmExecutionException(vLayer.dataProvider().error().message())
 
-        writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
-            vLayer.fields().toList(),
-            # Create a point layer (without any points) if 'no geometry' is chosen
-            vLayer.wkbType() if geometry_type != 1 else 1,
-            vLayer.crs())
+        writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(vLayer.fields().toList(),
+                                                                           vLayer.wkbType() if geometry_type != 1 else 1,
+                                                                           vLayer.crs(), context)
 
         features = QgsProcessingUtils.getFeatures(vLayer, context)
         total = 100.0 / QgsProcessingUtils.featureCount(vLayer, context)