Merge pull request #53168 from elpaso/attribute-table-defer-virtual-f…

…ields

Attribute table optimization: do not load hidden fields until required

python/core/auto_generated/vector/qgsvectorlayercache.sip.in

@@ -65,9 +65,26 @@ Returns ``True`` if the cache will fetch and cache feature geometries.
 
     void setCacheSubsetOfAttributes( const QgsAttributeList &attributes );
 %Docstring
-Set the subset of attributes to be cached
+Set the list (possibly a subset) of attributes to be cached.
+
+.. note::
+
+   By default the cache will store all layer's attributes.
 
 :param attributes: The attributes to be cached
+%End
+
+    QgsAttributeList cacheSubsetOfAttributes( ) const;
+%Docstring
+Returns the list (possibly a subset) of cached attributes.
+
+.. note::
+
+   By default the cache will store all layer's attributes.
+
+.. seealso:: :py:func:`setCacheSubsetOfAttributes`
+
+.. versionadded:: 3.32
 %End
 
     void setCacheAddedAttributes( bool cacheAddedAttributes );
@@ -179,6 +196,25 @@ Gets the feature at the given feature id. Considers the changed, added, deleted
 :param skipCache: Will query the layer regardless if the feature is in the cache already
 
 :return: ``True`` in case of success
+%End
+
+    bool featureAtIdWithAllAttributes( QgsFeatureId featureId, QgsFeature &feature, bool skipCache = false );
+%Docstring
+Gets the feature at the given feature id with all attributes, if the cached feature
+already contains all attributes, calling this function has the same effect as calling
+:py:func:`~QgsVectorLayerCache.featureAtId`.
+
+Considers the changed, added, deleted and permanent features
+
+:param featureId: The id of the feature to query
+:param feature: The result of the operation will be written to this feature
+:param skipCache: Will query the layer regardless if the feature is in the cache already
+
+:return: ``True`` in case of success
+
+.. seealso:: :py:func:`featureAtId`
+
+.. versionadded:: 3.32
 %End
 
     bool removeCachedFeature( QgsFeatureId fid );

python/gui/auto_generated/attributetable/qgsdualview.sip.in

@@ -196,6 +196,14 @@ Gets the expression used for sorting the table and feature list.
 The config used for the attribute table.
 
 :return: The config used for the attribute table.
+%End
+
+    static QgsAttributeList requiredAttributes( const QgsVectorLayer *layer );
+%Docstring
+Returns the list of required attributes according to the attribute table configuration of the ``layer``,
+only visible attributes and virtual fields referenced fields are returned.
+
+.. versionadded:: 3.32
 %End
 
   public slots:

src/app/qgsattributetabledialog.cpp

@@ -198,39 +198,39 @@ QgsAttributeTableDialog::QgsAttributeTableDialog( QgsVectorLayer *layer, QgsAttr
   QgsAttributeEditorContext editorContext = QgisApp::instance()->createAttributeEditorContext();
   editorContext.setDistanceArea( da );
 
-  QgsFeatureRequest r;
+  QgsFeatureRequest request;
   bool needsGeom = false;
   if ( mLayer && mLayer->geometryType() != Qgis::GeometryType::Null &&
        initialMode == QgsAttributeTableFilterModel::ShowVisible )
   {
     QgsMapCanvas *mc = QgisApp::instance()->mapCanvas();
     QgsRectangle extent( mc->mapSettings().mapToLayerCoordinates( layer, mc->extent() ) );
-    r.setFilterRect( extent );
+    request.setFilterRect( extent );
     needsGeom = true;
   }
   else if ( initialMode == QgsAttributeTableFilterModel::ShowSelected )
   {
-    r.setFilterFids( layer->selectedFeatureIds() );
+    request.setFilterFids( layer->selectedFeatureIds() );
   }
   else if ( initialMode == QgsAttributeTableFilterModel::ShowEdited )
   {
-    r.setFilterFids( layer->editBuffer() ? layer->editBuffer()->allAddedOrEditedFeatures() : QgsFeatureIds() );
+    request.setFilterFids( layer->editBuffer() ? layer->editBuffer()->allAddedOrEditedFeatures() : QgsFeatureIds() );
   }
   else if ( !filterExpression.isEmpty() )
   {
-    r.setFilterExpression( filterExpression );
+    request.setFilterExpression( filterExpression );
   }
   if ( !needsGeom )
-    r.setFlags( QgsFeatureRequest::NoGeometry );
+    request.setFlags( QgsFeatureRequest::NoGeometry );
+
 
   // Initialize dual view
   if ( mLayer )
   {
-    mMainView->init( mLayer, QgisApp::instance()->mapCanvas(), r, editorContext, false );
-
+    request.setSubsetOfAttributes( mMainView->requiredAttributes( mLayer ) );
+    mMainView->init( mLayer, QgisApp::instance()->mapCanvas(), request, editorContext, false );
     QgsAttributeTableConfig config = mLayer->attributeTableConfig();
     mMainView->setAttributeTableConfig( config );
-
     mFeatureFilterWidget->init( mLayer, editorContext, mMainView, QgisApp::instance()->messageBar(), QgsMessageBar::defaultMessageTimeout() );
   }
 

src/core/qgscachedfeatureiterator.cpp

@@ -182,7 +182,7 @@ bool QgsCachedFeatureWriterIterator::fetchFeature( QgsFeature &f )
   if ( mFeatIt.nextFeature( f ) )
   {
     // As long as features can be fetched from the provider: Write them to cache
-    mVectorLayerCache->cacheFeature( f );
+    mVectorLayerCache->cacheFeature( f, ! mRequest.flags().testFlag( QgsFeatureRequest::Flag::SubsetOfAttributes ) );
     mFids.insert( f.id() );
     geometryToDestinationCrs( f, mTransform );
     return true;

src/core/vector/qgsvectorlayercache.cpp

@@ -83,9 +83,17 @@ void QgsVectorLayerCache::setCacheGeometry( bool cacheGeometry )
 
 void QgsVectorLayerCache::setCacheSubsetOfAttributes( const QgsAttributeList &attributes )
 {
+  if ( ! mCache.isEmpty() && ! QSet<int>( mCachedAttributes.cbegin(), mCachedAttributes.cend() )
+       .contains( QSet<int>( attributes.cbegin(), attributes.cend( ) ) ) )
+    invalidate();
   mCachedAttributes = attributes;
 }
 
+QgsAttributeList QgsVectorLayerCache::cacheSubsetOfAttributes( ) const
+{
+  return mCachedAttributes;
+}
+
 void QgsVectorLayerCache::setFullCache( bool fullCache )
 {
   mFullCache = fullCache;
@@ -160,13 +168,51 @@ bool QgsVectorLayerCache::featureAtId( QgsFeatureId featureId, QgsFeature &featu
     feature = QgsFeature( *cachedFeature->feature() );
     featureFound = true;
   }
+  else
+  {
+    QgsFeatureRequest request { featureId };
+    const bool allAttrsFetched { mCachedAttributes.count( ) == mLayer->fields().count() };
+    if ( ! allAttrsFetched )
+    {
+      request.setSubsetOfAttributes( mCachedAttributes );
+    }
+    if ( !mCacheGeometry )
+    {
+      request.setFlags( request.flags().setFlag( QgsFeatureRequest::NoGeometry ) );
+    }
+    if ( mLayer->getFeatures( request ).nextFeature( feature ) )
+    {
+      cacheFeature( feature, allAttrsFetched );
+      featureFound = true;
+    }
+  }
+
+  return featureFound;
+}
+
+bool QgsVectorLayerCache::featureAtIdWithAllAttributes( QgsFeatureId featureId, QgsFeature &feature, bool skipCache )
+{
+
+  bool featureFound = false;
+
+  QgsCachedFeature *cachedFeature = nullptr;
+
+  if ( !skipCache )
+  {
+    cachedFeature = mCache[ featureId ];
+  }
+
+  if ( cachedFeature && cachedFeature->allAttributesFetched() )
+  {
+    feature = QgsFeature( *cachedFeature->feature() );
+    featureFound = true;
+  }
   else if ( mLayer->getFeatures( QgsFeatureRequest()
                                  .setFilterFid( featureId )
-                                 .setSubsetOfAttributes( mCachedAttributes )
                                  .setFlags( !mCacheGeometry ? QgsFeatureRequest::NoGeometry : QgsFeatureRequest::Flags() ) )
             .nextFeature( feature ) )
   {
-    cacheFeature( feature );
+    cacheFeature( feature, true );
     featureFound = true;
   }
 
@@ -342,11 +388,14 @@ void QgsVectorLayerCache::layerDeleted()
 
 void QgsVectorLayerCache::invalidate()
 {
-  mCache.clear();
-  mCacheOrderedKeys.clear();
-  mCacheUnorderedKeys.clear();
-  mFullCache = false;
-  emit invalidated();
+  if ( ! mCache.isEmpty() )
+  {
+    mCache.clear();
+    mCacheOrderedKeys.clear();
+    mCacheUnorderedKeys.clear();
+    mFullCache = false;
+    emit invalidated();
+  }
 }
 
 bool QgsVectorLayerCache::canUseCacheForRequest( const QgsFeatureRequest &featureRequest, QgsFeatureIterator &it )
@@ -433,12 +482,23 @@ QgsFeatureIterator QgsVectorLayerCache::getFeatures( const QgsFeatureRequest &fe
     if ( mCacheGeometry && mLayer->isSpatial() )
       myRequest.setFlags( featureRequest.flags() & ~QgsFeatureRequest::NoGeometry );
 
-    // Make sure, all the cached attributes are requested as well
-    const QgsAttributeList requestSubset = featureRequest.subsetOfAttributes();
-    QSet<int> attrs( requestSubset.begin(), requestSubset.end() );
-    for ( int attr : std::as_const( mCachedAttributes ) )
-      attrs.insert( attr );
-    myRequest.setSubsetOfAttributes( QgsAttributeList( attrs.begin(), attrs.end() ) );
+    // Make sure all the cached attributes are requested as well if requesting a subset
+    if ( myRequest.flags().testFlag( QgsFeatureRequest::SubsetOfAttributes ) )
+    {
+      if ( mCachedAttributes.count( ) != mLayer->fields().count() )
+      {
+        const QgsAttributeList requestSubset = featureRequest.subsetOfAttributes();
+        QSet<int> attrs( requestSubset.begin(), requestSubset.end() );
+        for ( int attr : std::as_const( mCachedAttributes ) )
+          attrs.insert( attr );
+        myRequest.setSubsetOfAttributes( attrs.values() );
+      }
+      else // we are already caching all attributes
+      {
+        myRequest.setSubsetOfAttributes( QgsAttributeList() );
+        myRequest.setFlags( myRequest.flags().setFlag( QgsFeatureRequest::Flag::SubsetOfAttributes, false ) );
+      }
+    }
 
     it = QgsFeatureIterator( new QgsCachedFeatureWriterIterator( this, myRequest ) );
   }
@@ -495,3 +555,9 @@ void QgsVectorLayerCache::connectJoinedLayers() const
       connect( vl, &QgsVectorLayer::attributeValueChanged, this, &QgsVectorLayerCache::onJoinAttributeValueChanged );
   }
 }
+
+bool QgsVectorLayerCache::QgsCachedFeature::allAttributesFetched() const
+{
+  return mAllAttributesFetched;
+}
+

src/core/vector/qgsvectorlayercache.h

@@ -54,7 +54,7 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
      * will inform the cache, when it has been deleted, so indexes can be
      * updated that the wrapped feature needs to be fetched again if needed.
      */
-    class QgsCachedFeature
+    class CORE_EXPORT QgsCachedFeature
     {
       public:
 
@@ -63,9 +63,11 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
          *
          * \param feat     The feature to cache. A copy will be made.
          * \param vlCache  The cache to inform when the feature has been removed from the cache.
+         * \param allAttributesFetched TRUE if the feature was fetched with all attributes (and not a subset)
          */
-        QgsCachedFeature( const QgsFeature &feat, QgsVectorLayerCache *vlCache )
+        QgsCachedFeature( const QgsFeature &feat, QgsVectorLayerCache *vlCache, bool allAttributesFetched )
           : mCache( vlCache )
+          , mAllAttributesFetched( allAttributesFetched )
         {
           mFeature = new QgsFeature( feat );
         }
@@ -80,9 +82,12 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
 
         inline const QgsFeature *feature() { return mFeature; }
 
+        bool allAttributesFetched() const;
+
       private:
         QgsFeature *mFeature = nullptr;
         QgsVectorLayerCache *mCache = nullptr;
+        bool mAllAttributesFetched = true;
 
         friend class QgsVectorLayerCache;
         Q_DISABLE_COPY( QgsCachedFeature )
@@ -125,12 +130,22 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
     bool cacheGeometry() const { return mCacheGeometry; }
 
     /**
-     * Set the subset of attributes to be cached
+     * Set the list (possibly a subset) of attributes to be cached.
      *
+     * \note By default the cache will store all layer's attributes.
      * \param attributes   The attributes to be cached
      */
     void setCacheSubsetOfAttributes( const QgsAttributeList &attributes );
 
+    /**
+     * Returns the list (possibly a subset) of cached attributes.
+     *
+     * \note By default the cache will store all layer's attributes.
+     * \see setCacheSubsetOfAttributes()
+     * \since QGIS 3.32
+     */
+    QgsAttributeList cacheSubsetOfAttributes( ) const;
+
     /**
      * If this is enabled, the subset of cached attributes will automatically be extended
      * to also include newly added attributes.
@@ -243,6 +258,21 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
      */
     bool featureAtId( QgsFeatureId featureId, QgsFeature &feature, bool skipCache = false );
 
+    /**
+     * Gets the feature at the given feature id with all attributes, if the cached feature
+     * already contains all attributes, calling this function has the same effect as calling
+     * featureAtId().
+     *
+     * Considers the changed, added, deleted and permanent features
+     * \param featureId The id of the feature to query
+     * \param feature   The result of the operation will be written to this feature
+     * \param skipCache Will query the layer regardless if the feature is in the cache already
+     * \returns TRUE in case of success
+     * \see featureAtId()
+     * \since QGIS 3.32
+     */
+    bool featureAtIdWithAllAttributes( QgsFeatureId featureId, QgsFeature &feature, bool skipCache = false );
+
     /**
      * Removes the feature identified by fid from the cache if present.
      * \param fid The id of the feature to delete
@@ -390,9 +420,9 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
 
     void connectJoinedLayers() const;
 
-    inline void cacheFeature( QgsFeature &feat )
+    inline void cacheFeature( QgsFeature &feat, bool allAttributesFetched )
     {
-      QgsCachedFeature *cachedFeature = new QgsCachedFeature( feat, this );
+      QgsCachedFeature *cachedFeature = new QgsCachedFeature( feat, this, allAttributesFetched );
       mCache.insert( feat.id(), cachedFeature );
       if ( mCacheUnorderedKeys.find( feat.id() ) == mCacheUnorderedKeys.end() )
       {

src/gui/attributetable/qgsattributetablemodel.cpp

@@ -91,6 +91,22 @@ bool QgsAttributeTableModel::loadFeatureAtId( QgsFeatureId fid ) const
   return mLayerCache->featureAtId( fid, mFeat );
 }
 
+bool QgsAttributeTableModel::loadFeatureAtId( QgsFeatureId fid, int fieldIdx ) const
+{
+  QgsDebugMsgLevel( QStringLiteral( "loading feature %1 with field %2" ).arg( fid, fieldIdx ), 3 );
+
+  if ( mLayerCache->cacheSubsetOfAttributes().contains( fieldIdx ) )
+  {
+    return loadFeatureAtId( fid );
+  }
+
+  if ( fid == std::numeric_limits<int>::min() )
+  {
+    return false;
+  }
+  return mLayerCache->featureAtIdWithAllAttributes( fid, mFeat );
+}
+
 int QgsAttributeTableModel::extraColumns() const
 {
   return mExtraColumns;
@@ -702,9 +718,9 @@ QVariant QgsAttributeTableModel::data( const QModelIndex &index, int role ) cons
     return static_cast<Qt::Alignment::Int>( widgetData.fieldFormatter->alignmentFlag( mLayer, fieldId, widgetData.config ) | Qt::AlignVCenter );
   }
 
-  if ( mFeat.id() != rowId || !mFeat.isValid() )
+  if ( mFeat.id() != rowId || !mFeat.isValid() || ! mLayerCache->cacheSubsetOfAttributes().contains( fieldId ) )
   {
-    if ( !loadFeatureAtId( rowId ) )
+    if ( !loadFeatureAtId( rowId, fieldId ) )
       return QVariant( "ERROR" );
 
     if ( mFeat.id() != rowId )