Connections API: executeSqlWithNames, return column names

This is mainly useful for Python, this way we can probably
avoid calling fields() and all related cost when all we need
is colum names for display.

I'm not very happy with the method name and I would probably just
drop the old signature for executeSql and replace it with the new
one, unfortunately is public API.

But if everybody is happy we could consider an API break.

python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in

@@ -45,6 +45,18 @@ is not supported or cannot be performed without errors.
     typedef QFlags<QgsAbstractDatabaseProviderConnection::TableFlag> TableFlags;
 
 
+    struct QueryResult
+    {
+      SIP_PYOBJECT __repr__();
+%MethodCode
+      QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rows.size() );
+      sipRes = PyUnicode_FromString( str.toUtf8().constData() );
+%End
+      QStringList columnns;
+      QList<QList<QVariant>> rows;
+    };
+
+
     struct TableProperty
     {
 
@@ -434,6 +446,16 @@ Executes raw ``sql`` and returns the (possibly empty) list of results in a multi
 Raises a QgsProviderConnectionException if any errors are encountered.
 
 :raises :: py:class:`QgsProviderConnectionException`
+%End
+
+    virtual QueryResult executeSqlWithNames( const QString &sql, QgsFeedback *feedback = 0 ) const throw( QgsProviderConnectionException );
+%Docstring
+Executes raw ``sql`` and returns the (possibly empty) query results, optionally ``feedback`` can be provided.
+Raises a QgsProviderConnectionException if any errors are encountered.
+
+:raises :: py:class:`QgsProviderConnectionException`
+
+.. versionadded:: 3.18
 %End
 
     virtual void vacuum( const QString &schema, const QString &name ) const throw( QgsProviderConnectionException );

src/core/providers/ogr/qgsgeopackageproviderconnection.cpp

@@ -164,7 +164,7 @@ void QgsGeoPackageProviderConnection::renameVectorTable( const QString &schema,
   }
 }
 
-QList<QList<QVariant>> QgsGeoPackageProviderConnection::executeSql( const QString &sql, QgsFeedback *feedback ) const
+QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnection::executeSqlWithNames( const QString &sql, QgsFeedback *feedback ) const
 {
   checkCapability( Capability::ExecuteSql );
   return executeGdalSqlPrivate( sql, feedback );
@@ -220,8 +220,8 @@ bool QgsGeoPackageProviderConnection::spatialIndexExists( const QString &schema,
   {
     QgsMessageLog::logMessage( QStringLiteral( "Schema is not supported by GPKG, ignoring" ), QStringLiteral( "OGR" ), Qgis::Info );
   }
-  const QList<QVariantList> res = executeGdalSqlPrivate( QStringLiteral( "SELECT HasSpatialIndex(%1, %2)" ).arg( QgsSqliteUtils::quotedString( name ),
-                                  QgsSqliteUtils::quotedString( geometryColumn ) ) );
+  const auto res { executeGdalSqlPrivate( QStringLiteral( "SELECT HasSpatialIndex(%1, %2)" ).arg( QgsSqliteUtils::quotedString( name ),
+                                          QgsSqliteUtils::quotedString( geometryColumn ) ) ).rows };
   return !res.isEmpty() && !res.at( 0 ).isEmpty() && res.at( 0 ).at( 0 ).toBool();
 }
 
@@ -354,9 +354,9 @@ void QgsGeoPackageProviderConnection::setDefaultCapabilities()
   };
 }
 
-QList<QVariantList> QgsGeoPackageProviderConnection::executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback ) const
+QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnection::executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback ) const
 {
-  QList<QVariantList> results;
+  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
@@ -392,6 +392,10 @@ QList<QVariantList> QgsGeoPackageProviderConnection::executeGdalSqlPrivate( cons
         if ( fields.isEmpty() )
         {
           fields = QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) );
+          for ( const auto &f : qgis::as_const( fields ) )
+          {
+            results.columnns.push_back( f.name() );
+          }
         }
 
         if ( ! fields.isEmpty() )
@@ -411,7 +415,7 @@ QList<QVariantList> QgsGeoPackageProviderConnection::executeGdalSqlPrivate( cons
           }
         }
 
-        results.push_back( row );
+        results.rows.push_back( row );
       }
       GDALDatasetReleaseResultSet( hDS.get(), ogrLayer );
     }

src/core/providers/ogr/qgsgeopackageproviderconnection.h

@@ -38,7 +38,7 @@ class QgsGeoPackageProviderConnection : public QgsAbstractDatabaseProviderConnec
     void dropVectorTable( const QString &schema, const QString &name ) const override;
     void dropRasterTable( const QString &schema, const QString &name ) const override;
     void renameVectorTable( const QString &schema, const QString &name, const QString &newName ) const override;
-    QList<QList<QVariant>> executeSql( const QString &sql, QgsFeedback *feedback = nullptr ) const override;
+    QueryResult executeSqlWithNames( const QString &sql, QgsFeedback *feedback = nullptr ) const override;
     void vacuum( const QString &schema, const QString &name ) const override;
     void createSpatialIndex( const QString &schema, const QString &name, const QgsAbstractDatabaseProviderConnection::SpatialIndexOptions &options = QgsAbstractDatabaseProviderConnection::SpatialIndexOptions() ) const override;
     bool spatialIndexExists( const QString &schema, const QString &name, const QString &geometryColumn ) const override;
@@ -52,7 +52,8 @@ class QgsGeoPackageProviderConnection : public QgsAbstractDatabaseProviderConnec
 
     void setDefaultCapabilities();
     //! Use GDAL to execute SQL
-    QList<QVariantList> executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback = nullptr ) const;
+    QueryResult executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback = nullptr ) const;
+
 
 };
 

src/core/qgsabstractdatabaseproviderconnection.cpp

@@ -133,12 +133,19 @@ void QgsAbstractDatabaseProviderConnection::renameSchema( const QString &, const
   checkCapability( Capability::RenameSchema );
 }
 
-QList<QList<QVariant>> QgsAbstractDatabaseProviderConnection::executeSql( const QString &, QgsFeedback * ) const
+QList<QList<QVariant>> QgsAbstractDatabaseProviderConnection::executeSql( const QString &sql, QgsFeedback *feedback ) const
+{
+  return executeSqlWithNames( sql, feedback ).rows;
+}
+
+
+QgsAbstractDatabaseProviderConnection::QueryResult QgsAbstractDatabaseProviderConnection::executeSqlWithNames( const QString &, QgsFeedback * ) const
 {
   checkCapability( Capability::ExecuteSql );
-  return QList<QList<QVariant>>();
+  return QueryResult();
 }
 
+
 void QgsAbstractDatabaseProviderConnection::vacuum( const QString &, const QString & ) const
 {
   checkCapability( Capability::Vacuum );

src/core/qgsabstractdatabaseproviderconnection.h

@@ -67,6 +67,27 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
     Q_DECLARE_FLAGS( TableFlags, TableFlag )
     Q_FLAG( TableFlags )
 
+    /**
+     * The QueryResult class represents the result of a query executed by executeSqlWithNames()
+     *
+     * It encapsulates the result rows and a list of the column names.
+     * The query result may be empty in case the query returns nothing.
+     * \since QGIS 3.18
+     */
+    struct CORE_EXPORT QueryResult
+    {
+#ifdef SIP_RUN
+      SIP_PYOBJECT __repr__();
+      % MethodCode
+      QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rows.size() );
+      sipRes = PyUnicode_FromString( str.toUtf8().constData() );
+      % End
+#endif
+      QStringList columnns;
+      QList<QList<QVariant>> rows;
+    };
+
+
     /**
      * The TableProperty class represents a database table or view.
      *
@@ -460,6 +481,14 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
      */
     virtual QList<QList<QVariant>> executeSql( const QString &sql, QgsFeedback *feedback = nullptr ) const SIP_THROW( QgsProviderConnectionException );
 
+    /**
+     * Executes raw \a sql and returns the (possibly empty) query results, optionally \a feedback can be provided.
+     * Raises a QgsProviderConnectionException if any errors are encountered.
+     * \throws QgsProviderConnectionException
+     * \since QGIS 3.18
+     */
+    virtual QueryResult executeSqlWithNames( const QString &sql, QgsFeedback *feedback = nullptr ) const SIP_THROW( QgsProviderConnectionException );
+
     /**
      * Vacuum the database table with given \a schema and \a name (schema is ignored if not supported by the backend).
      * Raises a QgsProviderConnectionException if any errors are encountered.

src/providers/mssql/qgsmssqlproviderconnection.cpp

@@ -15,6 +15,7 @@
  ***************************************************************************/
 
 #include <QSqlRecord>
+#include <QSqlField>
 
 #include "qgsmssqlproviderconnection.h"
 #include "qgsmssqlconnection.h"
@@ -213,15 +214,15 @@ void QgsMssqlProviderConnection::dropSchema( const QString &schemaName,  bool fo
                      .arg( QgsMssqlProvider::quotedIdentifier( schemaName ) ) );
 }
 
-QList<QVariantList> QgsMssqlProviderConnection::executeSql( const QString &sql, QgsFeedback *feedback ) const
+QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::executeSqlWithNames( const QString &sql, QgsFeedback *feedback ) const
 {
   checkCapability( Capability::ExecuteSql );
   return executeSqlPrivate( sql, true, feedback );
 }
 
-QList<QVariantList> QgsMssqlProviderConnection::executeSqlPrivate( const QString &sql, bool resolveTypes, QgsFeedback *feedback ) const
+QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::executeSqlPrivate( const QString &sql, bool resolveTypes, QgsFeedback *feedback ) const
 {
-  QList<QVariantList> results;
+  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
@@ -264,6 +265,10 @@ QList<QVariantList> QgsMssqlProviderConnection::executeSqlPrivate( const QString
     {
       const QSqlRecord rec { q.record() };
       const int numCols { rec.count() };
+      for ( int idx = 0; idx < numCols; ++idx )
+      {
+        results.columnns.push_back( rec.field( idx ).name() );
+      }
       while ( q.next() && ( ! feedback || ! feedback->isCanceled() ) )
       {
         QVariantList row;
@@ -278,7 +283,7 @@ QList<QVariantList> QgsMssqlProviderConnection::executeSqlPrivate( const QString
             row.push_back( q.value( col ).toString() );
           }
         }
-        results.push_back( row );
+        results.rows.push_back( row );
       }
     }
 
@@ -360,7 +365,7 @@ QList<QgsMssqlProviderConnection::TableProperty> QgsMssqlProviderConnection::tab
              .arg( QgsMssqlProvider::quotedValue( schema ) );
   }
 
-  const QList<QVariantList> results { executeSqlPrivate( query, false ) };
+  const QList<QVariantList> results { executeSqlPrivate( query, false ).rows };
   for ( const auto &row : results )
   {
     Q_ASSERT( row.count( ) == 6 );
@@ -394,7 +399,7 @@ QList<QgsMssqlProviderConnection::TableProperty> QgsMssqlProviderConnection::tab
       // This may fail for invalid geometries
       try
       {
-        const auto geomColResults { executeSqlPrivate( geomColSql ) };
+        const auto geomColResults { executeSqlPrivate( geomColSql ).rows };
         for ( const auto &row : geomColResults )
         {
           table.addGeometryColumnType( QgsWkbTypes::parseType( row[0].toString() ),
@@ -443,7 +448,7 @@ QStringList QgsMssqlProviderConnection::schemas( ) const
      WHERE u.issqluser = 1
         AND u.name NOT IN ('sys', 'guest', 'INFORMATION_SCHEMA')
     )raw" )};
-  const QList<QVariantList> result { executeSqlPrivate( sql, false ) };
+  const QList<QVariantList> result { executeSqlPrivate( sql, false ).rows };
   for ( const auto &row : result )
   {
     if ( row.size() > 0 )

src/providers/mssql/qgsmssqlproviderconnection.h

@@ -41,7 +41,7 @@ class QgsMssqlProviderConnection : public QgsAbstractDatabaseProviderConnection
     void dropVectorTable( const QString &schema, const QString &name ) const override;
     void createSchema( const QString &name ) const override;
     void dropSchema( const QString &name, bool force = false ) const override;
-    QList<QVariantList> executeSql( const QString &sql, QgsFeedback *feedback ) const override;
+    QgsAbstractDatabaseProviderConnection::QueryResult executeSqlWithNames( const QString &sql, QgsFeedback *feedback ) const override;
     QList<QgsAbstractDatabaseProviderConnection::TableProperty> tables( const QString &schema,
         const TableFlags &flags = TableFlags() ) const override;
     QStringList schemas( ) const override;
@@ -52,7 +52,7 @@ class QgsMssqlProviderConnection : public QgsAbstractDatabaseProviderConnection
 
   private:
 
-    QList<QVariantList> executeSqlPrivate( const QString &sql, bool resolveTypes = true, QgsFeedback *feedback = nullptr ) const;
+    QgsAbstractDatabaseProviderConnection::QueryResult executeSqlPrivate( const QString &sql, bool resolveTypes = true, QgsFeedback *feedback = nullptr ) const;
     void setDefaultCapabilities();
     void dropTablePrivate( const QString &schema, const QString &name ) const;
     void renameTablePrivate( const QString &schema, const QString &name, const QString &newName ) const;

src/providers/postgres/qgspostgresproviderconnection.cpp

@@ -192,15 +192,21 @@ void QgsPostgresProviderConnection::renameSchema( const QString &name, const QSt
                      .arg( QgsPostgresConn::quotedIdentifier( newName ) ) );
 }
 
-QList<QVariantList> QgsPostgresProviderConnection::executeSql( const QString &sql, QgsFeedback *feedback ) const
+QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection::executeSqlWithNames( const QString &sql, QgsFeedback *feedback ) const
 {
   checkCapability( Capability::ExecuteSql );
-  return executeSqlPrivate( sql, true, feedback );
+  return executeSqlPrivateWithNames( sql, true, feedback );
 }
 
 QList<QVariantList> QgsPostgresProviderConnection::executeSqlPrivate( const QString &sql, bool resolveTypes, QgsFeedback *feedback, std::shared_ptr<QgsPoolPostgresConn> pgconn ) const
 {
-  QList<QVariantList> results;
+  QStringList columnNames;
+  return executeSqlPrivateWithNames( sql, resolveTypes, feedback, pgconn ).rows;
+}
+
+QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection::executeSqlPrivateWithNames( const QString &sql, bool resolveTypes, QgsFeedback *feedback, std::shared_ptr<QgsPoolPostgresConn> pgconn ) const
+{
+  QueryResult results;
 
   // Check feedback first!
   if ( feedback && feedback->isCanceled() )
@@ -257,8 +263,16 @@ QList<QVariantList> QgsPostgresProviderConnection::executeSqlPrivate( const QStr
                    .arg( err );
       }
     }
+
     if ( res.PQntuples() > 0 )
     {
+
+      // Get column names
+      for ( int rowIdx = 0; rowIdx < res.PQnfields(); rowIdx++ )
+      {
+        results.columnns.push_back( res.PQfname( rowIdx ) );
+      }
+
       // Try to convert value types at least for basic simple types that can be directly mapped to Python
       QMap<int, QVariant::Type> typeMap;
       if ( resolveTypes )
@@ -355,7 +369,7 @@ QList<QVariantList> QgsPostgresProviderConnection::executeSqlPrivate( const QStr
             row.push_back( res.PQgetvalue( rowIdx, colIdx ) );
           }
         }
-        results.push_back( row );
+        results.rows.push_back( row );
       }
     }
     if ( ! errCause.isEmpty() )
@@ -693,7 +707,7 @@ QList<QgsVectorDataProvider::NativeType> QgsPostgresProviderConnection::nativeTy
 
 QgsFields QgsPostgresProviderConnection::fields( const QString &schema, const QString &tableName ) const
 {
-  // Try the base implementation first and fall back to a more complex approch for the
+  // Try the base implementation first and fall back to a more complex approach for the
   // few PG-specific corner cases that do not work with the base implementation.
   try
   {

src/providers/postgres/qgspostgresproviderconnection.h

@@ -46,7 +46,7 @@ class QgsPostgresProviderConnection : public QgsAbstractDatabaseProviderConnecti
     void createSchema( const QString &name ) const override;
     void dropSchema( const QString &name, bool force = false ) const override;
     void renameSchema( const QString &name, const QString &newName ) const override;
-    QList<QVariantList> executeSql( const QString &sql, QgsFeedback *feedback = nullptr ) const override;
+    QgsAbstractDatabaseProviderConnection::QueryResult executeSqlWithNames( const QString &sql, QgsFeedback *feedback = nullptr ) const override;
     void vacuum( const QString &schema, const QString &name ) const override;
     void createSpatialIndex( const QString &schema, const QString &name, const QgsAbstractDatabaseProviderConnection::SpatialIndexOptions &options = QgsAbstractDatabaseProviderConnection::SpatialIndexOptions() ) const override;
     bool spatialIndexExists( const QString &schema, const QString &name, const QString &geometryColumn ) const override;
@@ -62,6 +62,7 @@ class QgsPostgresProviderConnection : public QgsAbstractDatabaseProviderConnecti
   private:
 
     QList<QVariantList> executeSqlPrivate( const QString &sql, bool resolveTypes = true, QgsFeedback *feedback = nullptr, std::shared_ptr< class QgsPoolPostgresConn > pgconn = nullptr ) const;
+    QgsAbstractDatabaseProviderConnection::QueryResult executeSqlPrivateWithNames( const QString &sql, bool resolveTypes = true, QgsFeedback *feedback = nullptr, std::shared_ptr< class QgsPoolPostgresConn > pgconn = nullptr ) const;
     void setDefaultCapabilities();
     void dropTablePrivate( const QString &schema, const QString &name ) const;
     void renameTablePrivate( const QString &schema, const QString &name, const QString &newName ) const;