Load layers in parallel threads when opening project (#53069)

* load layers in parallel threads when opening project

* parallel loading by default and settingsin advanced settings

* fix freezing when no layer supports parallel loading

* clean leftover

* fix unwanted remove and typo

* typo

* fix flake8

* load OGR metadata under global mutex

* Revert "load OGR metadata under global mutex"

This reverts commit adcaf9e.

* remove parallel loading capability for OGR

* start application in server test

* import start_app

* fix flake8

python/core/auto_generated/providers/qgsdataprovider.sip.in

@@ -78,6 +78,7 @@ Abstract base class for spatial data provider implementations.
       SkipGetExtent,
       SkipFullScan,
       ForceReadOnly,
+      SkipCredentialsRequest,
     };
     typedef QFlags<QgsDataProvider::ReadFlag> ReadFlags;
 

python/core/auto_generated/providers/qgsprovidermetadata.sip.in

@@ -157,6 +157,7 @@ library object.
     {
       FileBasedUris,
       SaveLayerMetadata,
+      ParallelCreateProvider,
     };
     typedef QFlags<QgsProviderMetadata::ProviderCapability> ProviderCapabilities;
 

python/core/auto_generated/qgscredentials.sip.in

@@ -35,12 +35,12 @@ signal :py:func:`~destroyed` to be notified of the deletion
 
     virtual ~QgsCredentials();
 
-    bool get( const QString &realm, QString &username /In,Out/, QString &password /In,Out/, const QString &message = QString() );
+    bool get( const QString &realm, QString &username /In,Out/, QString &password /In,Out/, const QString &message = QString(), bool requestCredentials = true );
 %Docstring
 Requests credentials for the specified ``realm``.
 
 If existing credentials exist for the given ``realm``, these will be returned. Otherwise the credential
-handler will prompt for the correct username and password.
+handler will prompt for the correct username and password if ``requestCredentials`` is set to True.
 
 The retrieved or user-entered details will be stored in ``username`` and ``password``.
 

python/core/auto_generated/qgslayerdefinition.sip.in

@@ -108,14 +108,14 @@ Class used to work with layer dependencies stored in a XML project or layer defi
 %End
       public:
 
-        DependencySorter( const QDomDocument &doc );
+        explicit DependencySorter( const QDomDocument &doc );
 %Docstring
 Constructor
 
 :param doc: The XML document containing maplayer elements
 %End
 
-        DependencySorter( const QString &fileName );
+        explicit DependencySorter( const QString &fileName );
 %Docstring
 Constructor
 
@@ -140,6 +140,13 @@ Whether some cyclic dependency has been detected
         bool hasMissingDependency() const;
 %Docstring
 Whether some dependency is missing
+%End
+
+        bool isLayerDependent( const QString &layerId ) const;
+%Docstring
+Returns whether the layer associated with the``layerId`` is dependent from another layer
+
+.. versionadded:: 3.32
 %End
 
     };

python/core/auto_generated/qgsmaplayer.sip.in

@@ -633,13 +633,15 @@ or layer with temporary data (as temporary mesh layer dataset)
     typedef QFlags<QgsMapLayer::ReadFlag> ReadFlags;
 
 
-    bool readLayerXml( const QDomElement &layerElement, QgsReadWriteContext &context, QgsMapLayer::ReadFlags flags = QgsMapLayer::ReadFlags() );
+    bool readLayerXml( const QDomElement &layerElement, QgsReadWriteContext &context,
+                       QgsMapLayer::ReadFlags flags = QgsMapLayer::ReadFlags(), QgsDataProvider *preloadedProvider /Transfer/ = 0 );
 %Docstring
 Sets state from DOM document
 
 :param layerElement: The DOM element corresponding to ``maplayer'' tag
 :param context: writing context (e.g. for conversion between relative and absolute paths)
 :param flags: optional argument which can be used to control layer reading behavior.
+:param preloadedProvider: optional preloaded data provider that will be used as data provider for this layer, takes ownership (since QGIS 3.32)
 \note
 
 The DOM node corresponds to a DOM document project file XML element read
@@ -1576,6 +1578,15 @@ Enable or disable map tips for this layer
 %Docstring
 Returns true if map tips are enabled for this layer
 
+.. versionadded:: 3.32
+%End
+
+    static QgsDataProvider::ReadFlags providerReadFlags( const QDomNode &layerNode, QgsMapLayer::ReadFlags layerReadFlags );
+%Docstring
+Returns provider read flag deduced from layer read flags ``layerReadFlags`` and a dom node ``layerNode``
+that describes a layer (corresponding to ``maplayer'' tag in a DOM document project file read by :py:class:`QgsProject`).
+This static method is used when loading a project.
+
 .. versionadded:: 3.32
 %End
 
@@ -2091,6 +2102,7 @@ this method is now deprecated and always return ``False``, because circular depe
 
 
 
+
 };
 
 QFlags<QgsMapLayer::LayerFlag> operator|(QgsMapLayer::LayerFlag f1, QFlags<QgsMapLayer::LayerFlag> f2);

src/core/CMakeLists.txt

@@ -280,6 +280,7 @@ set(QGIS_CORE_SRCS
   providers/qgsprovidersublayermodel.cpp
   providers/qgsprovidersublayertask.cpp
   providers/qgsproviderutils.cpp
+  providers/qgsrunnableprovidercreator.cpp
 
   providers/arcgis/qgsarcgisportalutils.cpp
   providers/arcgis/qgsarcgisrestquery.cpp
@@ -1728,6 +1729,7 @@ set(QGIS_CORE_HDRS
   providers/qgsprovidersublayermodel.h
   providers/qgsprovidersublayertask.h
   providers/qgsproviderutils.h
+  providers/qgsrunnableprovidercreator.h
 
   providers/arcgis/qgsarcgisportalutils.h
   providers/arcgis/qgsarcgisrestquery.h

src/core/mesh/qgsmeshlayer.cpp

@@ -45,6 +45,8 @@
 #include "qgsmeshlayerprofilegenerator.h"
 #include "qgsmeshlayerelevationproperties.h"
 #include "qgsthreadingutils.h"
+#include "qgsapplication.h"
+#include "qgsruntimeprofiler.h"
 
 QgsMeshLayer::QgsMeshLayer( const QString &meshLayerPath,
                             const QString &baseName,
@@ -1836,15 +1838,7 @@ bool QgsMeshLayer::readXml( const QDomNode &layer_node, QgsReadWriteContext &con
   }
 
   const QgsDataProvider::ProviderOptions providerOptions;
-  QgsDataProvider::ReadFlags flags = QgsDataProvider::ReadFlags();
-  if ( mReadFlags & QgsMapLayer::FlagTrustLayerMetadata )
-  {
-    flags |= QgsDataProvider::FlagTrustDataSource;
-  }
-  if ( mReadFlags & QgsMapLayer::FlagForceReadOnly )
-  {
-    flags |= QgsDataProvider::ForceReadOnly;
-  }
+  QgsDataProvider::ReadFlags flags = providerReadFlags( layer_node, mReadFlags );
 
   const QDomElement elemExtraDatasets = layer_node.firstChildElement( QStringLiteral( "extra-datasets" ) );
   if ( !elemExtraDatasets.isNull() )
@@ -2077,7 +2071,18 @@ bool QgsMeshLayer::setDataProvider( QString const &provider, const QgsDataProvid
   mProviderKey = provider;
   const QString dataSource = mDataSource;
 
-  mDataProvider = qobject_cast<QgsMeshDataProvider *>( QgsProviderRegistry::instance()->createProvider( provider, dataSource, options, flags ) );
+  if ( mPreloadedProvider )
+  {
+    mDataProvider = qobject_cast< QgsMeshDataProvider * >( mPreloadedProvider.release() );
+  }
+  else
+  {
+    std::unique_ptr< QgsScopedRuntimeProfile > profile;
+    if ( QgsApplication::profiler()->groupIsActive( QStringLiteral( "projectload" ) ) )
+      profile = std::make_unique< QgsScopedRuntimeProfile >( tr( "Create %1 provider" ).arg( provider ), QStringLiteral( "projectload" ) );
+
+    mDataProvider = qobject_cast<QgsMeshDataProvider *>( QgsProviderRegistry::instance()->createProvider( provider, dataSource, options, flags ) );
+  }
 
   if ( !mDataProvider )
   {

src/core/pointcloud/qgspointcloudlayer.cpp

@@ -148,7 +148,7 @@ bool QgsPointCloudLayer::readXml( const QDomNode &layerNode, QgsReadWriteContext
   if ( !( mReadFlags & QgsMapLayer::FlagDontResolveLayers ) )
   {
     const QgsDataProvider::ProviderOptions providerOptions { context.transformContext() };
-    QgsDataProvider::ReadFlags flags = QgsDataProvider::ReadFlags();
+    QgsDataProvider::ReadFlags flags = providerReadFlags( layerNode, mReadFlags );
     // read extent
     if ( mReadFlags & QgsMapLayer::FlagReadExtentFromXml )
     {
@@ -160,19 +160,9 @@ bool QgsPointCloudLayer::readXml( const QDomNode &layerNode, QgsReadWriteContext
 
         // store the extent
         setExtent( mbr );
-
-        // skip get extent
-        flags |= QgsDataProvider::SkipGetExtent;
       }
     }
-    if ( mReadFlags & QgsMapLayer::FlagTrustLayerMetadata )
-    {
-      flags |= QgsDataProvider::FlagTrustDataSource;
-    }
-    if ( mReadFlags & QgsMapLayer::FlagForceReadOnly )
-    {
-      flags |= QgsDataProvider::ForceReadOnly;
-    }
+
     setDataSource( mDataSource, mLayerName, mProviderKey, providerOptions, flags );
     const QDomNode subset = layerNode.namedItem( QStringLiteral( "subset" ) );
     const QString subsetText = subset.toElement().text();
@@ -400,7 +390,18 @@ void QgsPointCloudLayer::setDataSourcePrivate( const QString &dataSource, const
   mProviderKey = provider;
   mDataSource = dataSource;
 
-  mDataProvider.reset( qobject_cast<QgsPointCloudDataProvider *>( QgsProviderRegistry::instance()->createProvider( provider, dataSource, options, flags ) ) );
+  if ( mPreloadedProvider )
+  {
+    mDataProvider.reset( qobject_cast< QgsPointCloudDataProvider * >( mPreloadedProvider.release() ) );
+  }
+  else
+  {
+    std::unique_ptr< QgsScopedRuntimeProfile > profile;
+    if ( QgsApplication::profiler()->groupIsActive( QStringLiteral( "projectload" ) ) )
+      profile = std::make_unique< QgsScopedRuntimeProfile >( tr( "Create %1 provider" ).arg( provider ), QStringLiteral( "projectload" ) );
+    mDataProvider.reset( qobject_cast<QgsPointCloudDataProvider *>( QgsProviderRegistry::instance()->createProvider( provider, dataSource, options, flags ) ) );
+  }
+
   if ( !mDataProvider )
   {
     QgsDebugMsg( QStringLiteral( "Unable to get point cloud data provider" ) );