Skip to content

Commit

Permalink
Refactor processing writer to use QgsFeatureSink, and to store
Browse files Browse the repository at this point in the history
temporary output layers in the provided processing context

Should fix loss of intermediate memory layers when running
a multi-step model
  • Loading branch information
nyalldawson committed Apr 26, 2017
1 parent 946f0fa commit fd2c18e
Show file tree
Hide file tree
Showing 123 changed files with 327 additions and 478 deletions.
8 changes: 6 additions & 2 deletions python/core/processing/qgsprocessingcontext.sip
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,12 @@ class QgsProcessingContext
Sets the expression ``context``.
%End



QgsProject &temporaryLayerStore();
%Docstring
Returns a reference to the project used for storing temporary layers during
algorithm execution.
:rtype: QgsProject
%End

QgsFeatureRequest::InvalidGeometryCheck invalidGeometryCheck() const;
%Docstring
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/AddTableField.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,7 @@ def processAlgorithm(self, context, feedback):
fields = layer.fields()
fields.append(QgsField(fieldName, self.TYPES[fieldType], '',
fieldLength, fieldPrecision))
writer = output.getVectorWriter(fields, layer.wkbType(),
layer.crs())
writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/AutoincrementalField.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ def processAlgorithm(self, context, feedback):
dataobjects.getLayerFromString(self.getParameterValue(self.INPUT))
fields = vlayer.fields()
fields.append(QgsField('AUTO', QVariant.Int))
writer = output.getVectorWriter(fields, vlayer.wkbType(),
vlayer.crs())
writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(vlayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vlayer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Boundary.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,7 @@ def processAlgorithm(self, context, feedback):
output_wkb = QgsWkbTypes.addM(output_wkb)

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields(),
output_wkb,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields(), output_wkb, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/BoundingBox.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT_LAYER))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields().toList(),
QgsWkbTypes.Polygon,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Centroids.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT_LAYER))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields(),
QgsWkbTypes.Point,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields(), QgsWkbTypes.Point, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
15 changes: 3 additions & 12 deletions python/plugins/processing/algs/qgis/CheckValidity.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,32 +108,23 @@ def doCheck(self, context, feedback):

valid_output = self.getOutputFromName(self.VALID_OUTPUT)
valid_fields = layer.fields()
valid_writer = valid_output.getVectorWriter(
valid_fields,
layer.wkbType(),
layer.crs())
valid_writer = valid_output.getVectorWriter(valid_fields, layer.wkbType(), layer.crs(), context)
valid_count = 0

invalid_output = self.getOutputFromName(self.INVALID_OUTPUT)
invalid_fields = layer.fields().toList() + [
QgsField(name='_errors',
type=QVariant.String,
len=255)]
invalid_writer = invalid_output.getVectorWriter(
invalid_fields,
layer.wkbType(),
layer.crs())
invalid_writer = invalid_output.getVectorWriter(invalid_fields, layer.wkbType(), layer.crs(), context)
invalid_count = 0

error_output = self.getOutputFromName(self.ERROR_OUTPUT)
error_fields = [
QgsField(name='message',
type=QVariant.String,
len=255)]
error_writer = error_output.getVectorWriter(
error_fields,
QgsWkbTypes.Point,
layer.crs())
error_writer = error_output.getVectorWriter(error_fields, QgsWkbTypes.Point, layer.crs(), context)
error_count = 0

features = QgsProcessingUtils.getFeatures(layer, context)
Expand Down
7 changes: 3 additions & 4 deletions python/plugins/processing/algs/qgis/Clip.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,9 @@ def processAlgorithm(self, context, feedback):
mask_layer = dataobjects.getLayerFromString(
self.getParameterValue(Clip.OVERLAY))

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
source_layer.fields(),
QgsWkbTypes.multiType(source_layer.wkbType()),
source_layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(source_layer.fields(),
QgsWkbTypes.multiType(source_layer.wkbType()),
source_layer.crs(), context)

# first build up a list of clip geometries
clip_geoms = []
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/qgis/ConcaveHull.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@ def processAlgorithm(self, context, feedback):
feedback.setProgressText(self.tr('Saving data...'))
feat = QgsFeature()
dissolved_layer.getFeatures(QgsFeatureRequest().setFilterFid(0)).nextFeature(feat)
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon,
layer.crs(), context)
geom = feat.geometry()
if no_multigeom and geom.isMultipart():
# Only singlepart geometries are allowed
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ConvexHull.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,7 @@ def processAlgorithm(self, context, feedback):
QgsField('perim', QVariant.Double, '', 20, 6)
]

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
fields, QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)

outFeat = QgsFeature()
inGeom = QgsGeometry()
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/Delaunay.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ def processAlgorithm(self, context, feedback):
QgsField('POINTB', QVariant.Double, '', 24, 15),
QgsField('POINTC', QVariant.Double, '', 24, 15)]

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)

pts = []
ptDict = {}
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DeleteColumn.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,7 @@ def processAlgorithm(self, context, feedback):
for index in field_indices:
fields.remove(index)

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ def processAlgorithm(self, context, feedback):

fields = layer.fields()

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)

Expand Down
6 changes: 2 additions & 4 deletions python/plugins/processing/algs/qgis/DeleteHoles.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,8 @@ def processAlgorithm(self, context, feedback):
if min_area == 0.0:
min_area = -1.0

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
layer.fields(),
layer.wkbType(),
layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DensifyGeometries.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ def processAlgorithm(self, context, feedback):
isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry

writer = self.getOutputFromName(
self.OUTPUT).getVectorWriter(layer.fields().toList(),
layer.wkbType(), layer.crs())
self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,7 @@ def processAlgorithm(self, context, feedback):
isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry

writer = self.getOutputFromName(
self.OUTPUT).getVectorWriter(layer.fields().toList(),
layer.wkbType(), layer.crs())
self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
4 changes: 1 addition & 3 deletions python/plugins/processing/algs/qgis/Difference.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,7 @@ def processAlgorithm(self, context, feedback):

geomType = QgsWkbTypes.multiType(layerA.wkbType())
writer = self.getOutputFromName(
Difference.OUTPUT).getVectorWriter(layerA.fields(),
geomType,
layerA.crs())
Difference.OUTPUT).getVectorWriter(layerA.fields(), geomType, layerA.crs(), context)

outFeat = QgsFeature()
index = vector.spatialindex(layerB)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Dissolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(Dissolve.INPUT))

writer = self.getOutputFromName(
Dissolve.OUTPUT).getVectorWriter(
vlayerA.fields().toList(),
vlayerA.wkbType(),
vlayerA.crs())
Dissolve.OUTPUT).getVectorWriter(vlayerA.fields().toList(), vlayerA.wkbType(), vlayerA.crs(), context)

outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(vlayerA, context)
Expand Down
6 changes: 2 additions & 4 deletions python/plugins/processing/algs/qgis/DropGeometry.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,8 @@ def processAlgorithm(self, context, feedback):
layer = dataobjects.getLayerFromString(
self.getParameterValue(self.INPUT_LAYER))
writer = self.getOutputFromName(
self.OUTPUT_TABLE).getVectorWriter(
layer.fields(),
QgsWkbTypes.NoGeometry,
QgsCoordinateReferenceSystem())
self.OUTPUT_TABLE).getVectorWriter(layer.fields(), QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem(),
context)

request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)
features = QgsProcessingUtils.getFeatures(layer, context, request)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/EliminateSelection.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,7 @@ def processAlgorithm(self, context, feedback):
featToEliminate = []
selFeatIds = inLayer.selectedFeatureIds()
output = self.getOutputFromName(self.OUTPUT)
writer = output.getVectorWriter(inLayer.fields(),
inLayer.wkbType(), inLayer.crs())
writer = output.getVectorWriter(inLayer.fields(), inLayer.wkbType(), inLayer.crs(), context)

for aFeat in inLayer.getFeatures():
if aFeat.id() in selFeatIds:
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/EquivalentNumField.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ def processAlgorithm(self, context, feedback):
fieldindex = vlayer.fields().lookupField(fieldname)
fields = vlayer.fields()
fields.append(QgsField('NUM_FIELD', QVariant.Int))
writer = output.getVectorWriter(fields, vlayer.wkbType(),
vlayer.crs())
writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
outFeat = QgsFeature()
classes = {}

Expand Down
8 changes: 3 additions & 5 deletions python/plugins/processing/algs/qgis/ExecuteSQL.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,9 @@ def processAlgorithm(self, context, feedback):
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(vLayer.dataProvider().error().message())

writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
vLayer.fields().toList(),
# Create a point layer (without any points) if 'no geometry' is chosen
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(vLayer.fields().toList(),
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs(), context)

features = QgsProcessingUtils.getFeatures(vLayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vLayer, context)
Expand Down