Merge pull request #4495 from nyalldawson/writer

[processing] Port vector.createVectorWriter to c++

python/core/processing/qgsprocessingcontext.sip

@@ -118,9 +118,23 @@ class QgsProcessingContext
 %End
 
 
+    QString defaultEncoding() const;
+%Docstring
+ Returns the default encoding to use for newly created files.
+.. seealso:: setDefaultEncoding()
+ :rtype: str
+%End
+
+    void setDefaultEncoding( const QString &encoding );
+%Docstring
+ Sets the default ``encoding`` to use for newly created files.
+.. seealso:: defaultEncoding()
+%End
+
   private:
     QgsProcessingContext( const QgsProcessingContext &other );
 };
+
 QFlags<QgsProcessingContext::Flag> operator|(QgsProcessingContext::Flag f1, QFlags<QgsProcessingContext::Flag> f2);
 
 

python/core/processing/qgsprocessingutils.sip

@@ -122,6 +122,38 @@ class QgsProcessingUtils
  :rtype: list of QVariant
 %End
 
+
+    static void createFeatureSinkPython(
+      QgsFeatureSink **sink /Out,TransferBack/,
+      QString &destination /In,Out/,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer **outputLayer /Out/ ) /PyName=createFeatureSink/;
+%Docstring
+ Creates a feature sink ready for adding features. The ``destination`` specifies a destination
+ URI for the resultant layer. It may be updated in place to reflect the actual destination
+ for the layer.
+
+ Sink parameters such as desired ``encoding``, ``fields``, ``geometryType`` and ``crs`` must be specified.
+
+ If the ``encoding`` is not specified, the default encoding from the ``context`` will be used.
+
+ If a layer is created for the feature sink, the layer will automatically be added to the ``context``'s
+ temporary layer store, and the ``outputLayer`` argument updated to point at this newly created layer.
+
+.. note::
+
+   this version of the createFeatureSink() function has an API designed around use from the
+ SIP bindings. c++ code should call the other createFeatureSink() version.
+.. note::
+
+   available in Python bindings as createFeatureSink()
+%End
+
+
 };
 
 

python/plugins/processing/algs/qgis/VectorSplit.py

@@ -86,7 +86,7 @@ def processAlgorithm(self, context, feedback):
         for current, i in enumerate(uniqueValues):
             fName = u'{0}_{1}.shp'.format(baseName, str(i).strip())
 
-            writer, dest, _layer = vector.createVectorWriter(fName, None, fields, geomType, crs, context)
+            writer, dest, _layer = QgsProcessingUtils.createFeatureSink(fName, None, fields, geomType, crs, context)
             for f in QgsProcessingUtils.getFeatures(layer, context):
                 if f[fieldName] == i:
                     writer.addFeature(f)

python/plugins/processing/algs/qgis/scripts/Number_of_unique_values_in_classes.py

@@ -6,13 +6,12 @@
 
 from qgis.PyQt.QtCore import QVariant
 from qgis.core import QgsFeature, QgsField, QgsProcessingUtils
-from processing.tools.vector import createVectorWriter
 
 layer = QgsProcessingUtils.mapLayerFromString(input, context)
 fields = layer.fields()
 fields.append(QgsField('UNIQ_COUNT', QVariant.Int))
-writer, writer_dest, writer_layer = createVectorWriter(N_unique_values, None, fields, layer.wkbType(), layer.crs(),
-                                                       context)
+writer, writer_dest, writer_layer = QgsProcessingUtils.createFeatureSink(N_unique_values, None, fields, layer.wkbType(), layer.crs(),
+                                                                         context)
 
 class_field_index = layer.fields().lookupField(class_field)
 value_field_index = layer.fields().lookupField(value_field)

python/plugins/processing/core/outputs.py

@@ -35,7 +35,7 @@
 
 from processing.core.ProcessingConfig import ProcessingConfig
 from processing.tools.system import isWindows, getTempFilenameInTempFolder, getTempDirInTempFolder
-from processing.tools.vector import createVectorWriter, TableWriter, NOGEOMETRY_EXTENSIONS
+from processing.tools.vector import TableWriter, NOGEOMETRY_EXTENSIONS
 from processing.tools import dataobjects
 
 from qgis.core import (QgsExpressionContext,
@@ -44,7 +44,8 @@
                        QgsExpressionContextScope,
                        QgsProject,
                        QgsSettings,
-                       QgsVectorFileWriter)
+                       QgsVectorFileWriter,
+                       QgsProcessingUtils)
 
 
 def _expressionContext(alg):
@@ -384,7 +385,7 @@ def getVectorWriter(self, fields, geomType, crs, context):
             settings = QgsSettings()
             self.encoding = settings.value('/Processing/encoding', 'System', str)
 
-        w, w_dest, w_layer = createVectorWriter(self.value, self.encoding, fields, geomType, crs, context)
+        w, w_dest, w_layer = QgsProcessingUtils.createFeatureSink(self.value, self.encoding, fields, geomType, crs, context)
         self.layer = w_layer
         self.value = w_dest
         return w

python/plugins/processing/tests/testdata/scripts/centroids.py

@@ -3,15 +3,13 @@
 ##INPUT_LAYER=vector
 ##OUTPUT_LAYER=output vector
 
-from qgis.core import QgsWkbTypes, QgsGeometry, QgsProcessingUtils
-
-from processing.tools.vector import createVectorWriter
+from qgis.core import QgsWkbTypes, QgsProcessingUtils
 
 layer = QgsProcessingUtils.mapLayerFromString(INPUT_LAYER, context)
 fields = layer.fields()
 
-writer, writer_dest, writer_layer = createVectorWriter(OUTPUT_LAYER, 'utf-8', fields, QgsWkbTypes.Point, layer.crs(),
-                                                       context)
+writer, writer_dest, writer_layer = QgsProcessingUtils.createFeatureSink(OUTPUT_LAYER, 'utf-8', fields, QgsWkbTypes.Point, layer.crs(),
+                                                                         context)
 
 features = QgsProcessingUtils.getFeatures(layer, context)
 count = QgsProcessingUtils.featureCount(layer, context)

python/plugins/processing/tools/dataobjects.py

@@ -85,6 +85,9 @@ def raise_error(f):
 
     context.setInvalidGeometryCallback(raise_error)
 
+    settings = QgsSettings()
+    context.setDefaultEncoding(settings.value("/Processing/encoding", "System"))
+
     return context
 
 

python/plugins/processing/tools/vector.py

@@ -55,39 +55,7 @@
                        QgsProcessingContext,
                        QgsProcessingUtils)
 
-from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
-from processing.tools import dataobjects, spatialite, postgis
-
-
-TYPE_MAP = {
-    str: QVariant.String,
-    float: QVariant.Double,
-    int: QVariant.Int,
-    bool: QVariant.Bool
-}
-
-TYPE_MAP_MEMORY_LAYER = {
-    QVariant.String: "string",
-    QVariant.Double: "double",
-    QVariant.Int: "integer",
-    QVariant.Date: "date",
-    QVariant.DateTime: "datetime",
-    QVariant.Time: "time"
-}
-
-TYPE_MAP_POSTGIS_LAYER = {
-    QVariant.String: "VARCHAR",
-    QVariant.Double: "REAL",
-    QVariant.Int: "INTEGER",
-    QVariant.Bool: "BOOLEAN"
-}
-
-TYPE_MAP_SPATIALITE_LAYER = {
-    QVariant.String: "VARCHAR",
-    QVariant.Double: "REAL",
-    QVariant.Int: "INTEGER",
-    QVariant.Bool: "INTEGER"
-}
+from processing.tools import dataobjects
 
 
 def resolveFieldIndex(layer, attr):
@@ -305,12 +273,6 @@ def checkMinDistance(point, index, distance, points):
     return True
 
 
-def _toQgsField(f):
-    if isinstance(f, QgsField):
-        return f
-    return QgsField(f[0], TYPE_MAP.get(f[1], QVariant.String))
-
-
 def snapToPrecision(geom, precision):
     snapped = QgsGeometry(geom)
     if precision == 0.0:
@@ -447,10 +409,6 @@ def ogrLayerName(uri):
     return name
 
 
-MEMORY_LAYER_PREFIX = 'memory:'
-POSTGIS_LAYER_PREFIX = 'postgis:'
-SPATIALITE_LAYER_PREFIX = 'spatialite:'
-
 NOGEOMETRY_EXTENSIONS = [
     u'csv',
     u'dbf',
@@ -459,134 +417,6 @@ def ogrLayerName(uri):
 ]
 
 
-def createVectorWriter(destination, encoding, fields, geometryType, crs, context):
-    layer = None
-    sink = None
-
-    if encoding is None:
-        settings = QgsSettings()
-        encoding = settings.value('/Processing/encoding', 'System', str)
-
-    if destination.startswith(MEMORY_LAYER_PREFIX):
-        uri = QgsWkbTypes.displayString(geometryType) + "?uuid=" + str(uuid.uuid4())
-        if crs.isValid():
-            uri += '&crs=' + crs.authid()
-        fieldsdesc = []
-        for f in fields:
-            qgsfield = _toQgsField(f)
-            fieldsdesc.append('field=%s:%s' % (qgsfield.name(),
-                                               TYPE_MAP_MEMORY_LAYER.get(qgsfield.type(), "string")))
-        if fieldsdesc:
-            uri += '&' + '&'.join(fieldsdesc)
-
-        layer = QgsVectorLayer(uri, destination, 'memory')
-        sink = layer.dataProvider()
-        context.temporaryLayerStore().addMapLayer(layer)
-        destination = layer.id()
-    elif destination.startswith(POSTGIS_LAYER_PREFIX):
-        uri = QgsDataSourceUri(destination[len(POSTGIS_LAYER_PREFIX):])
-        connInfo = uri.connectionInfo()
-        (success, user, passwd) = QgsCredentials.instance().get(connInfo, None, None)
-        if success:
-            QgsCredentials.instance().put(connInfo, user, passwd)
-        else:
-            raise GeoAlgorithmExecutionException("Couldn't connect to database")
-        try:
-            db = postgis.GeoDB(host=uri.host(), port=int(uri.port()),
-                               dbname=uri.database(), user=user, passwd=passwd)
-        except postgis.DbError as e:
-            raise GeoAlgorithmExecutionException(
-                "Couldn't connect to database:\n%s" % e.message)
-
-        def _runSQL(sql):
-            try:
-                db._exec_sql_and_commit(str(sql))
-            except postgis.DbError as e:
-                raise GeoAlgorithmExecutionException(
-                    'Error creating output PostGIS table:\n%s' % e.message)
-
-        fields = [_toQgsField(f) for f in fields]
-        fieldsdesc = ",".join('%s %s' % (f.name(),
-                                         TYPE_MAP_POSTGIS_LAYER.get(f.type(), "VARCHAR"))
-                              for f in fields)
-
-        _runSQL("CREATE TABLE %s.%s (%s)" % (uri.schema(), uri.table().lower(), fieldsdesc))
-        if geometryType != QgsWkbTypes.NullGeometry:
-            _runSQL("SELECT AddGeometryColumn('{schema}', '{table}', 'the_geom', {srid}, '{typmod}', 2)".format(
-                table=uri.table().lower(), schema=uri.schema(), srid=crs.authid().split(":")[-1],
-                typmod=QgsWkbTypes.displayString(geometryType).upper()))
-
-        layer = QgsVectorLayer(uri.uri(), uri.table(), "postgres")
-        sink = layer.dataProvider()
-        context.temporaryLayerStore().addMapLayer(layer)
-    elif destination.startswith(SPATIALITE_LAYER_PREFIX):
-        uri = QgsDataSourceUri(destination[len(SPATIALITE_LAYER_PREFIX):])
-        try:
-            db = spatialite.GeoDB(uri=uri)
-        except spatialite.DbError as e:
-            raise GeoAlgorithmExecutionException(
-                "Couldn't connect to database:\n%s" % e.message)
-
-        def _runSQL(sql):
-            try:
-                db._exec_sql_and_commit(str(sql))
-            except spatialite.DbError as e:
-                raise GeoAlgorithmExecutionException(
-                    'Error creating output Spatialite table:\n%s' % str(e))
-
-        fields = [_toQgsField(f) for f in fields]
-        fieldsdesc = ",".join('%s %s' % (f.name(),
-                                         TYPE_MAP_SPATIALITE_LAYER.get(f.type(), "VARCHAR"))
-                              for f in fields)
-
-        _runSQL("DROP TABLE IF EXISTS %s" % uri.table().lower())
-        _runSQL("CREATE TABLE %s (%s)" % (uri.table().lower(), fieldsdesc))
-        if geometryType != QgsWkbTypes.NullGeometry:
-            _runSQL("SELECT AddGeometryColumn('{table}', 'the_geom', {srid}, '{typmod}', 2)".format(
-                table=uri.table().lower(), srid=crs.authid().split(":")[-1],
-                typmod=QgsWkbTypes.displayString(geometryType).upper()))
-
-        layer = QgsVectorLayer(uri.uri(), uri.table(), "spatialite")
-        sink = layer.dataProvider()
-        context.temporaryLayerStore().addMapLayer(layer)
-    else:
-        formats = QgsVectorFileWriter.supportedFiltersAndFormats()
-        OGRCodes = {}
-        for (key, value) in list(formats.items()):
-            extension = str(key)
-            extension = extension[extension.find('*.') + 2:]
-            extension = extension[:extension.find(' ')]
-            OGRCodes[extension] = value
-        OGRCodes['dbf'] = "DBF file"
-
-        extension = destination[destination.rfind('.') + 1:]
-
-        if extension not in OGRCodes:
-            extension = 'shp'
-            destination = destination + '.shp'
-
-        if geometryType == QgsWkbTypes.NoGeometry:
-            if extension == 'shp':
-                extension = 'dbf'
-                destination = destination[:destination.rfind('.')] + '.dbf'
-            if extension not in NOGEOMETRY_EXTENSIONS:
-                raise GeoAlgorithmExecutionException(
-                    "Unsupported format for tables with no geometry")
-
-        qgsfields = QgsFields()
-        for field in fields:
-            qgsfields.append(_toQgsField(field))
-
-        # use default dataset/layer options
-        dataset_options = QgsVectorFileWriter.defaultDatasetOptions(OGRCodes[extension])
-        layer_options = QgsVectorFileWriter.defaultLayerOptions(OGRCodes[extension])
-
-        sink = QgsVectorFileWriter(destination, encoding,
-                                   qgsfields, geometryType, crs, OGRCodes[extension],
-                                   dataset_options, layer_options)
-    return sink, destination, layer
-
-
 class TableWriter(object):
 
     def __init__(self, fileName, encoding, fields):

src/core/processing/qgsprocessingcontext.h

@@ -142,6 +142,18 @@ class CORE_EXPORT QgsProcessingContext
      */
     SIP_SKIP std::function< void( const QgsFeature & ) > invalidGeometryCallback() const { return mInvalidGeometryCallback; }
 
+    /**
+     * Returns the default encoding to use for newly created files.
+     * \see setDefaultEncoding()
+     */
+    QString defaultEncoding() const { return mDefaultEncoding; }
+
+    /**
+     * Sets the default \a encoding to use for newly created files.
+     * \see defaultEncoding()
+     */
+    void setDefaultEncoding( const QString &encoding ) { mDefaultEncoding = encoding; }
+
   private:
 
     QgsProcessingContext::Flags mFlags = 0;
@@ -151,11 +163,13 @@ class CORE_EXPORT QgsProcessingContext
     QgsExpressionContext mExpressionContext;
     QgsFeatureRequest::InvalidGeometryCheck mInvalidGeometryCheck = QgsFeatureRequest::GeometryNoCheck;
     std::function< void( const QgsFeature & ) > mInvalidGeometryCallback;
+    QString mDefaultEncoding;
 
 #ifdef SIP_RUN
     QgsProcessingContext( const QgsProcessingContext &other );
 #endif
 };
+
 Q_DECLARE_OPERATORS_FOR_FLAGS( QgsProcessingContext::Flags )
 
 #endif // QGSPROCESSINGPARAMETERS_H