[processing] Port vector.createVectorWriter to c++

This implements an improved version of vector.createVectorWriter
in QgsProcessingUtils. The improved version relies on the
core class QgsVectorLayerImport to create empty layers,
which:
- reduces duplicate code and reuses the mature QgsVectorLayerImport
routines
- avoids manual conversion of field types to destination provider
field types
- potentially allows any writable provider to be used as a feature
sink for algorithms (e.g. output direct to MSSQL/Oracle/db2). This
should work now - it just needs exposing via UI.

Author: nyalldawson

python/core/processing/qgsprocessingcontext.sip

@@ -118,9 +118,23 @@ class QgsProcessingContext
 %End
 
 
+    QString defaultEncoding() const;
+%Docstring
+ Returns the default encoding to use for newly created files.
+.. seealso:: setDefaultEncoding()
+ :rtype: str
+%End
+
+    void setDefaultEncoding( const QString &encoding );
+%Docstring
+ Sets the default ``encoding`` to use for newly created files.
+.. seealso:: defaultEncoding()
+%End
+
   private:
     QgsProcessingContext( const QgsProcessingContext &other );
 };
+
 QFlags<QgsProcessingContext::Flag> operator|(QgsProcessingContext::Flag f1, QFlags<QgsProcessingContext::Flag> f2);
 
 

python/core/processing/qgsprocessingutils.sip

@@ -122,6 +122,38 @@ class QgsProcessingUtils
  :rtype: list of QVariant
 %End
 
+
+    static void createFeatureSinkPython(
+      QgsFeatureSink **sink /Out,TransferBack/,
+      QString &destination /In,Out/,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer **outputLayer /Out/ ) /PyName=createFeatureSink/;
+%Docstring
+ Creates a feature sink ready for adding features. The ``destination`` specifies a destination
+ URI for the resultant layer. It may be updated in place to reflect the actual destination
+ for the layer.
+
+ Sink parameters such as desired ``encoding``, ``fields``, ``geometryType`` and ``crs`` must be specified.
+
+ If the ``encoding`` is not specified, the default encoding from the ``context`` will be used.
+
+ If a layer is created for the feature sink, the layer will automatically be added to the ``context``'s
+ temporary layer store, and the ``outputLayer`` argument updated to point at this newly created layer.
+
+.. note::
+
+   this version of the createFeatureSink() function has an API designed around use from the
+ SIP bindings. c++ code should call the other createFeatureSink() version.
+.. note::
+
+   available in Python bindings as createFeatureSink()
+%End
+
+
 };
 
 

python/plugins/processing/tools/dataobjects.py

@@ -85,6 +85,9 @@ def raise_error(f):
 
     context.setInvalidGeometryCallback(raise_error)
 
+    settings = QgsSettings()
+    context.setDefaultEncoding(settings.value("/Processing/encoding", "System"))
+
     return context
 
 

python/plugins/processing/tools/vector.py

@@ -460,6 +460,7 @@ def ogrLayerName(uri):
 
 
 def createVectorWriter(destination, encoding, fields, geometryType, crs, context):
+    return QgsProcessingUtils.createFeatureSink(destination, encoding, fields, geometryType, crs, context)
     layer = None
     sink = None
 

src/core/processing/qgsprocessingcontext.h

@@ -142,6 +142,18 @@ class CORE_EXPORT QgsProcessingContext
      */
     SIP_SKIP std::function< void( const QgsFeature & ) > invalidGeometryCallback() const { return mInvalidGeometryCallback; }
 
+    /**
+     * Returns the default encoding to use for newly created files.
+     * \see setDefaultEncoding()
+     */
+    QString defaultEncoding() const { return mDefaultEncoding; }
+
+    /**
+     * Sets the default \a encoding to use for newly created files.
+     * \see defaultEncoding()
+     */
+    void setDefaultEncoding( const QString &encoding ) { mDefaultEncoding = encoding; }
+
   private:
 
     QgsProcessingContext::Flags mFlags = 0;
@@ -151,11 +163,13 @@ class CORE_EXPORT QgsProcessingContext
     QgsExpressionContext mExpressionContext;
     QgsFeatureRequest::InvalidGeometryCheck mInvalidGeometryCheck = QgsFeatureRequest::GeometryNoCheck;
     std::function< void( const QgsFeature & ) > mInvalidGeometryCallback;
+    QString mDefaultEncoding;
 
 #ifdef SIP_RUN
     QgsProcessingContext( const QgsProcessingContext &other );
 #endif
 };
+
 Q_DECLARE_OPERATORS_FOR_FLAGS( QgsProcessingContext::Flags )
 
 #endif // QGSPROCESSINGPARAMETERS_H

src/core/processing/qgsprocessingutils.cpp

@@ -19,6 +19,9 @@
 #include "qgsproject.h"
 #include "qgssettings.h"
 #include "qgsprocessingcontext.h"
+#include "qgsvectorlayerimport.h"
+#include "qgsvectorfilewriter.h"
+#include "qgsmemoryproviderutils.h"
 
 QList<QgsRasterLayer *> QgsProcessingUtils::compatibleRasterLayers( QgsProject *project, bool sort )
 {
@@ -289,4 +292,116 @@ QList<QVariant> QgsProcessingUtils::uniqueValues( QgsVectorLayer *layer, int fie
   }
 }
 
+void parseDestinationString( QString &destination, QString &providerKey, QString &uri, QString &format, QMap<QString, QVariant> &options )
+{
+  QRegularExpression splitRx( "^(.*?):(.*)$" );
+  QRegularExpressionMatch match = splitRx.match( destination );
+  if ( match.hasMatch() )
+  {
+    providerKey = match.captured( 1 );
+    if ( providerKey == QStringLiteral( "postgis" ) ) // older processing used "postgis" instead of "postgres"
+    {
+      providerKey = QStringLiteral( "postgres" );
+    }
+    uri = match.captured( 2 );
+  }
+  else
+  {
+    providerKey = QStringLiteral( "ogr" );
+    QRegularExpression splitRx( "^(.*)\\.(.*?)$" );
+    QRegularExpressionMatch match = splitRx.match( destination );
+    QString extension;
+    if ( match.hasMatch() )
+    {
+      extension = match.captured( 2 );
+      format = QgsVectorFileWriter::driverForExtension( extension );
+    }
+
+    if ( format.isEmpty() )
+    {
+      format = QStringLiteral( "ESRI Shapefile" );
+      destination = destination + QStringLiteral( ".shp" );
+    }
+
+    options.insert( QStringLiteral( "driverName" ), format );
+    uri = destination;
+  }
+}
+
+QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer *&outputLayer )
+{
+  outputLayer = nullptr;
+  QgsVectorLayer *layer = nullptr;
+
+  QString destEncoding = encoding;
+  if ( destEncoding.isEmpty() )
+  {
+    // no destination encoding specified, use default
+    destEncoding = context.defaultEncoding().isEmpty() ? QStringLiteral( "system" ) : context.defaultEncoding();
+  }
+
+  if ( destination.isEmpty() || destination.startsWith( QStringLiteral( "memory:" ) ) )
+  {
+    // memory provider cannot be used with QgsVectorLayerImport - so create layer manually
+    layer = QgsMemoryProviderUtils::createMemoryLayer( destination, fields, geometryType, crs );
+    if ( layer && layer->isValid() )
+      destination = layer->id();
+  }
+  else
+  {
+    QMap<QString, QVariant> options;
+    options.insert( QStringLiteral( "fileEncoding" ), destEncoding );
+
+    QString providerKey;
+    QString uri;
+    QString format;
+    parseDestinationString( destination, providerKey, uri, format, options );
+
+    if ( providerKey == "ogr" )
+    {
+      // use QgsVectorFileWriter for OGR destinations instead of QgsVectorLayerImport, as that allows
+      // us to use any OGR format which supports feature addition
+      QString finalFileName;
+      QgsVectorFileWriter *writer = new QgsVectorFileWriter( destination, destEncoding, fields, geometryType, crs, format, QgsVectorFileWriter::defaultDatasetOptions( format ),
+          QgsVectorFileWriter::defaultLayerOptions( format ), &finalFileName );
+      destination = finalFileName;
+      return writer;
+    }
+    else
+    {
+      //create empty layer
+      {
+        QgsVectorLayerImport import( uri, providerKey, fields, geometryType, crs, false, &options );
+        if ( import.hasError() )
+          return nullptr;
+      }
+
+      layer = new QgsVectorLayer( uri, destination, providerKey );
+    }
+  }
+
+  if ( !layer )
+    return nullptr;
+
+  if ( !layer->isValid() )
+  {
+    delete layer;
+    return nullptr;
+  }
+
+  context.temporaryLayerStore()->addMapLayer( layer );
+
+  outputLayer = layer;
+  // this is a factory, so we need to return a proxy
+  return new QgsProxyFeatureSink( layer->dataProvider() );
+}
+
+void QgsProcessingUtils::createFeatureSinkPython( QgsFeatureSink **sink, QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer **outputLayer )
+{
+  QgsVectorLayer *layer = nullptr;
+  *sink = createFeatureSink( destination, encoding, fields, geometryType, crs, context, layer );
+  if ( outputLayer )
+    *outputLayer = layer;
+}
+
 

src/core/processing/qgsprocessingutils.h

@@ -131,6 +131,58 @@ class CORE_EXPORT QgsProcessingUtils
      */
     static QList< QVariant > uniqueValues( QgsVectorLayer *layer, int fieldIndex, const QgsProcessingContext &context );
 
+    /**
+     * Creates a feature sink ready for adding features. The \a destination specifies a destination
+     * URI for the resultant layer. It may be updated in place to reflect the actual destination
+     * for the layer.
+     *
+     * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified.
+     *
+     * If the \a encoding is not specified, the default encoding from the \a context will be used.
+     *
+     * If a layer is created for the feature sink, the layer will automatically be added to the \a context's
+     * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer.
+     *
+     * The caller takes responsibility for deleting the returned sink.
+     */
+#ifndef SIP_RUN
+    static QgsFeatureSink *createFeatureSink(
+      QString &destination,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer *&outputLayer ) SIP_FACTORY;
+#endif
+
+    /**
+     * Creates a feature sink ready for adding features. The \a destination specifies a destination
+     * URI for the resultant layer. It may be updated in place to reflect the actual destination
+     * for the layer.
+     *
+     * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified.
+     *
+     * If the \a encoding is not specified, the default encoding from the \a context will be used.
+     *
+     * If a layer is created for the feature sink, the layer will automatically be added to the \a context's
+     * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer.
+     *
+     * \note this version of the createFeatureSink() function has an API designed around use from the
+     * SIP bindings. c++ code should call the other createFeatureSink() version.
+     * \note available in Python bindings as createFeatureSink()
+     */
+    static void createFeatureSinkPython(
+      QgsFeatureSink **sink SIP_OUT SIP_TRANSFERBACK,
+      QString &destination SIP_INOUT,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer **outputLayer SIP_OUT ) SIP_PYNAME( createFeatureSink );
+
+
   private:
 
     static bool canUseLayer( const QgsRasterLayer *layer );