Cache unique values when creating features

elpaso · elpaso · commit ba3d9ed066fd · 2019-02-19T10:02:17.000+01:00
Fixes #21305 - pasting features is very slow Aggressively optimize createFeature for speed and introduces createFeatures for bulk creation.
diff --git a/python/core/auto_generated/qgsvectorlayerutils.sip.in b/python/core/auto_generated/qgsvectorlayerutils.sip.in
@@ -57,6 +57,40 @@ Returns the duplicated features in the given layer
 
     };
 
+    class QgsFeaturesData
+{
+%Docstring
+Encapsulate geometry and attributes for new features, to be passed to createFeatures
+
+.. seealso:: :py:func:`createFeatures`
+
+.. versionadded:: 3.6
+%End
+
+%TypeHeaderCode
+#include "qgsvectorlayerutils.h"
+%End
+      public:
+
+        QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );
+%Docstring
+Constructs a new QgsFeaturesData with given ``geometry`` and ``attributes``
+%End
+
+        QgsGeometry geometry() const;
+%Docstring
+Returns geometry
+%End
+
+        QgsAttributeMap attributes() const;
+%Docstring
+Returns attributes
+%End
+
+    };
+
+    typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;
+
     static QgsFeatureIterator getValuesIterator( const QgsVectorLayer *layer, const QString &fieldOrExpression, bool &ok, bool selectedOnly );
 %Docstring
 Create a feature iterator for a specified field name or expression.
@@ -143,6 +177,17 @@ Creates a new feature ready for insertion into a layer. Default values and const
 passed for the new feature to copy as many attribute values as possible from the map,
 assuming that they respect the layer's constraints. Note that the created feature is not
 automatically inserted into the layer.
+%End
+
+    static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
+                                          const QgsFeaturesDataList &featuresData,
+                                          QgsExpressionContext *context = 0 );
+%Docstring
+Creates a set of new features ready for insertion into a layer. Default values and constraints
+(e.g., unique constraints) will automatically be handled. An optional attribute map can be
+passed for the new feature to copy as many attribute values as possible from the map,
+assuming that they respect the layer's constraints. Note that the created features are not
+automatically inserted into the layer.
 %End
 
     static QgsFeature duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext /Out/ );
diff --git a/src/app/qgisapp.cpp b/src/app/qgisapp.cpp
@@ -8996,11 +8996,12 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
   QgsExpressionContext context = pasteVectorLayer->createExpressionContext();
 
   QgsFeatureList compatibleFeatures( QgsVectorLayerUtils::makeFeaturesCompatible( features, pasteVectorLayer ) );
-  QgsFeatureList newFeatures;
+  QgsVectorLayerUtils::QgsFeaturesDataList newFeaturesDataList;
+  newFeaturesDataList.reserve( compatibleFeatures.size() );
+
   // Count collapsed geometries
   int invalidGeometriesCount = 0;
 
-  newFeatures.reserve( compatibleFeatures.size() );
   for ( const auto &feature : qgis::as_const( compatibleFeatures ) )
   {
 
@@ -9022,8 +9023,10 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
     }
     // now create new feature using pasted feature as a template. This automatically handles default
     // values and field constraints
-    newFeatures << QgsVectorLayerUtils::createFeature( pasteVectorLayer, geom, attrMap, &context );
+    newFeaturesDataList << QgsVectorLayerUtils::QgsFeaturesData( geom, attrMap );
   }
+
+  QgsFeatureList newFeatures {QgsVectorLayerUtils::createFeatures( pasteVectorLayer, newFeaturesDataList, &context )};
   pasteVectorLayer->addFeatures( newFeatures );
   QgsFeatureIds newIds;
   newIds.reserve( newFeatures.size() );
@@ -9032,7 +9035,6 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
     newIds << f.id();
   }
 
-
   pasteVectorLayer->selectByIds( newIds );
   pasteVectorLayer->endEditCommand();
   pasteVectorLayer->updateExtents();
diff --git a/src/core/qgsvectorlayerutils.cpp b/src/core/qgsvectorlayerutils.cpp
@@ -358,11 +358,17 @@ bool QgsVectorLayerUtils::validateAttribute( const QgsVectorLayer *layer, const
 
 QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, const QgsGeometry &geometry,
     const QgsAttributeMap &attributes, QgsExpressionContext *context )
+{
+  return createFeatures( layer, QgsFeaturesDataList() << QgsFeaturesData( geometry, attributes ), context ).first();
+}
+
+QgsFeatureList QgsVectorLayerUtils::createFeatures( const QgsVectorLayer *layer, const QgsFeaturesDataList &featuresData, QgsExpressionContext *context )
 {
   if ( !layer )
-  {
-    return QgsFeature();
-  }
+    return QgsFeatureList();
+
+  QgsFeatureList result;
+  result.reserve( featuresData.length() );
 
   QgsExpressionContext *evalContext = context;
   std::unique_ptr< QgsExpressionContext > tempContext;
@@ -375,94 +381,104 @@ QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, cons
 
   QgsFields fields = layer->fields();
 
-  QgsFeature newFeature( fields );
-  newFeature.setValid( true );
-  newFeature.setGeometry( geometry );
+  // Cache unique values
+  QMap<int, QSet<QVariant>> uniqueValueCaches;
 
-  // initialize attributes
-  newFeature.initAttributes( fields.count() );
-  for ( int idx = 0; idx < fields.count(); ++idx )
+  for ( const auto &fd : qgis::as_const( featuresData ) )
   {
-    QVariant v;
-    bool checkUnique = true;
 
-    // in order of priority:
-    // 1. passed attribute value and if field does not have a unique constraint like primary key
-    if ( attributes.contains( idx ) )
+    QgsFeature newFeature( fields );
+    newFeature.setValid( true );
+    newFeature.setGeometry( fd.geometry() );
+
+    // initialize attributes
+    newFeature.initAttributes( fields.count() );
+    for ( int idx = 0; idx < fields.count(); ++idx )
     {
-      v = attributes.value( idx );
-    }
+      QVariant v;
+      bool checkUnique = true;
+      const bool hasUniqueConstraint { static_cast<bool>( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique ) };
 
-    // Cache unique values
-    QSet<QVariant> uniqueValues { layer->uniqueValues( idx ) };
+      // in order of priority:
+      // 1. passed attribute value and if field does not have a unique constraint like primary key
+      if ( fd.attributes().contains( idx ) )
+      {
+        v = fd.attributes().value( idx );
+      }
 
-    // 2. client side default expression
-    // note - deliberately not using else if!
-    if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && layer->defaultValueDefinition( idx ).isValid() )
-    {
-      // client side default expression set - takes precedence over all. Why? Well, this is the only default
-      // which QGIS users have control over, so we assume that they're deliberately overriding any
-      // provider defaults for some good reason and we should respect that
-      v = layer->defaultValue( idx, newFeature, evalContext );
-    }
+      // Cache unique values
+      if ( hasUniqueConstraint && ! uniqueValueCaches.contains( idx ) )
+        uniqueValueCaches[ idx ] = layer->uniqueValues( idx );
 
-    // 3. provider side default value clause
-    // note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
-    if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
-    {
-      int providerIndex = fields.fieldOriginIndex( idx );
-      QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
-      if ( !providerDefault.isEmpty() )
+      // 2. client side default expression
+      // note - deliberately not using else if!
+      if ( ( !v.isValid() || ( hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && layer->defaultValueDefinition( idx ).isValid() )
       {
-        v = providerDefault;
-        checkUnique = false;
+        // client side default expression set - takes precedence over all. Why? Well, this is the only default
+        // which QGIS users have control over, so we assume that they're deliberately overriding any
+        // provider defaults for some good reason and we should respect that
+        v = layer->defaultValue( idx, newFeature, evalContext );
       }
-    }
 
-    // 4. provider side default literal
-    // note - deliberately not using else if!
-    if ( ( !v.isValid() || ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
-    {
-      int providerIndex = fields.fieldOriginIndex( idx );
-      v = layer->dataProvider()->defaultValue( providerIndex );
-      if ( v.isValid() )
+      // 3. provider side default value clause
+      // note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
+      if ( ( !v.isValid() || ( hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
       {
-        //trust that the provider default has been sensibly set not to violate any constraints
-        checkUnique = false;
+        int providerIndex = fields.fieldOriginIndex( idx );
+        QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
+        if ( !providerDefault.isEmpty() )
+        {
+          v = providerDefault;
+          checkUnique = false;
+        }
       }
-    }
 
-    // 5. passed attribute value
-    // note - deliberately not using else if!
-    if ( !v.isValid() && attributes.contains( idx ) )
-    {
-      v = attributes.value( idx );
-    }
+      // 4. provider side default literal
+      // note - deliberately not using else if!
+      if ( ( !v.isValid() || ( checkUnique && hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
+      {
+        int providerIndex = fields.fieldOriginIndex( idx );
+        v = layer->dataProvider()->defaultValue( providerIndex );
+        if ( v.isValid() )
+        {
+          //trust that the provider default has been sensibly set not to violate any constraints
+          checkUnique = false;
+        }
+      }
 
-    // last of all... check that unique constraints are respected
-    // we can't handle not null or expression constraints here, since there's no way to pick a sensible
-    // value if the constraint is violated
-    if ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique )
-    {
-      if ( uniqueValues.contains( v ) )
+      // 5. passed attribute value
+      // note - deliberately not using else if!
+      if ( !v.isValid() && fd.attributes().contains( idx ) )
       {
-        // unique constraint violated
-        QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
-        if ( uniqueValue.isValid() )
-          v = uniqueValue;
+        v = fd.attributes().value( idx );
       }
-    }
 
-    newFeature.setAttribute( idx, v );
+      // last of all... check that unique constraints are respected
+      // we can't handle not null or expression constraints here, since there's no way to pick a sensible
+      // value if the constraint is violated
+      if ( checkUnique && hasUniqueConstraint )
+      {
+        if ( uniqueValueCaches[ idx ].contains( v ) )
+        {
+          // unique constraint violated
+          QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
+          if ( uniqueValue.isValid() )
+            v = uniqueValue;
+        }
+      }
+      if ( hasUniqueConstraint )
+        uniqueValueCaches[ idx ].insert( v );
+      newFeature.setAttribute( idx, v );
+    }
+    result.append( newFeature );
   }
-
-  return newFeature;
+  return result;
 }
 
 QgsFeature QgsVectorLayerUtils::duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext )
@@ -772,3 +788,18 @@ QMap<QgsVectorLayer *, QgsFeatureIds>  QgsVectorLayerUtils::QgsDuplicateFeatureC
   return mDuplicatedFeatures;
 }
 */
+
+QgsVectorLayerUtils::QgsFeaturesData::QgsFeaturesData( const QgsGeometry &geometry, const QgsAttributeMap &attributes ):
+  mGeometry( geometry ),
+  mAttributes( attributes )
+{}
+
+QgsGeometry QgsVectorLayerUtils::QgsFeaturesData::geometry() const
+{
+  return mGeometry;
+}
+
+QgsAttributeMap QgsVectorLayerUtils::QgsFeaturesData::attributes() const
+{
+  return mAttributes;
+}
diff --git a/src/core/qgsvectorlayerutils.h b/src/core/qgsvectorlayerutils.h
@@ -70,6 +70,37 @@ class CORE_EXPORT QgsVectorLayerUtils
         void setDuplicatedFeatures( QgsVectorLayer *layer, const QgsFeatureIds &ids );
     };
 
+    /**
+     * \ingroup core
+     * \class QgsFeatureSetData
+     * \brief Encapsulate geometry and attributes for new features, to be passed to createFeatures
+     * \see createFeatures()
+     * \since QGIS 3.6
+     */
+    class CORE_EXPORT QgsFeaturesData
+    {
+      public:
+
+        /**
+         * Constructs a new QgsFeaturesData with given \a geometry and \a attributes
+         */
+        QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );
+
+        //! Returns geometry
+        QgsGeometry geometry() const;
+
+        //! Returns attributes
+        QgsAttributeMap attributes() const;
+
+      private:
+        QgsGeometry mGeometry;
+        QgsAttributeMap mAttributes;
+    };
+
+    // SIP does not lile "using", use legacy typedef
+    //! Alias for list of QgsFeaturesData
+    typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;
+
     /**
      * Create a feature iterator for a specified field name or expression.
      * \param layer vector layer to retrieve values from
@@ -145,6 +176,17 @@ class CORE_EXPORT QgsVectorLayerUtils
                                      const QgsAttributeMap &attributes = QgsAttributeMap(),
                                      QgsExpressionContext *context = nullptr );
 
+    /**
+     * Creates a set of new features ready for insertion into a layer. Default values and constraints
+     * (e.g., unique constraints) will automatically be handled. An optional attribute map can be
+     * passed for the new feature to copy as many attribute values as possible from the map,
+     * assuming that they respect the layer's constraints. Note that the created features are not
+     * automatically inserted into the layer.
+     */
+    static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
+                                          const QgsFeaturesDataList &featuresData,
+                                          QgsExpressionContext *context = nullptr );
+
     /**
      * Duplicates a feature and it's children (one level deep). It calls CreateFeature, so
      * default values and constraints (e.g., unique constraints) will automatically be handled.
diff --git a/tests/src/python/test_provider_postgres.py b/tests/src/python/test_provider_postgres.py
