[FEATURE] Add method to get list of unique strings matching

a substring from a vector data provider

Base implementation iterates through all features, but
providers can override with optimised versions. Native
implementations for postgres, spatialite and OGR included.

Author: nyalldawson

python/core/qgsvectordataprovider.sip

@@ -148,6 +148,18 @@ class QgsVectorDataProvider : QgsDataProvider
      */
     virtual void uniqueValues( int index, QList<QVariant> &uniqueValues /Out/, int limit = -1 ) const;
 
+    /**
+     * Returns unique string values of an attribute which contain a specified subset string. Subset
+     * matching is done in a case-insensitive manner.
+     * @param index the index of the attribute
+     * @param substring substring to match (case insensitive)
+     * @param limit maxmum number of the values to return, or -1 to return all unique values
+     * @param feedback optional feedback object for cancelling request
+     * @returns list of unique strings containg substring
+     */
+    virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
+                                               QgsFeedback* feedback = nullptr ) const;
+
     /** Calculates an aggregated value from the layer's features. The base implementation does nothing,
      * but subclasses can override this method to handoff calculation of aggregates to the provider.
      * @param aggregate aggregate to calculate

src/core/qgsvectordataprovider.cpp

@@ -26,6 +26,7 @@
 #include "qgsfeature.h"
 #include "qgsfeatureiterator.h"
 #include "qgsfeaturerequest.h"
+#include "qgsfeedback.h"
 #include "qgsfields.h"
 #include "qgsgeometry.h"
 #include "qgsgeometrycollection.h"
@@ -432,6 +433,37 @@ void QgsVectorDataProvider::uniqueValues( int index, QList<QVariant> &values, in
   }
 }
 
+QStringList QgsVectorDataProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
+{
+  QgsFeature f;
+  QgsAttributeList keys;
+  keys.append( index );
+
+  QgsFeatureRequest request;
+  request.setSubsetOfAttributes( keys );
+  request.setFlags( QgsFeatureRequest::NoGeometry );
+  QString fieldName = fields().at( index ).name();
+  request.setFilterExpression( QStringLiteral( "\"%1\" ILIKE '%%2%'" ).arg( fieldName, substring ) );
+  QgsFeatureIterator fi = getFeatures( request );
+
+  QSet<QString> set;
+  QStringList results;
+
+  while ( fi.nextFeature( f ) )
+  {
+    QString value = f.attribute( index ).toString();
+    if ( !set.contains( value ) )
+    {
+      results.append( value );
+      set.insert( value );
+    }
+
+    if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
+      break;
+  }
+  return results;
+}
+
 QVariant QgsVectorDataProvider::aggregate( QgsAggregateCalculator::Aggregate aggregate, int index,
     const QgsAggregateCalculator::AggregateParameters& parameters, QgsExpressionContext* context, bool& ok ) const
 {

src/core/qgsvectordataprovider.h

@@ -36,6 +36,7 @@ typedef QHash<int, QString> QgsAttrPalIndexNameHash;
 
 class QgsFeatureIterator;
 class QgsTransaction;
+class QgsFeedback;
 
 #include "qgsfeaturerequest.h"
 
@@ -201,6 +202,18 @@ class CORE_EXPORT QgsVectorDataProvider : public QgsDataProvider
      */
     virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const;
 
+    /**
+     * Returns unique string values of an attribute which contain a specified subset string. Subset
+     * matching is done in a case-insensitive manner.
+     * @param index the index of the attribute
+     * @param substring substring to match (case insensitive)
+     * @param limit maxmum number of the values to return, or -1 to return all unique values
+     * @param feedback optional feedback object for cancelling request
+     * @returns list of unique strings containg substring
+     */
+    virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
+        QgsFeedback* feedback = nullptr ) const;
+
     /** Calculates an aggregated value from the layer's features. The base implementation does nothing,
      * but subclasses can override this method to handoff calculation of aggregates to the provider.
      * @param aggregate aggregate to calculate

src/providers/ogr/qgsogrprovider.cpp

@@ -20,6 +20,7 @@ email                : sherman at mrcc.com
 #include "qgslogger.h"
 #include "qgsmessagelog.h"
 #include "qgslocalec.h"
+#include "qgsfeedback.h"
 
 #define CPL_SUPRESS_CPLUSPLUS
 #include <gdal.h>         // to collect version information
@@ -2924,6 +2925,59 @@ void QgsOgrProvider::uniqueValues( int index, QList<QVariant> &uniqueValues, int
 #endif
 }
 
+QStringList QgsOgrProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
+{
+  QStringList results;
+
+  if ( !mValid || index < 0 || index >= mAttributeFields.count() )
+    return results;
+
+  QgsField fld = mAttributeFields.at( index );
+  if ( fld.name().isNull() )
+  {
+    return results; //not a provider field
+  }
+
+#if defined(GDAL_VERSION_NUM) && GDAL_VERSION_NUM < 1910
+  // avoid GDAL #4509
+  return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
+#else
+  QByteArray sql = "SELECT DISTINCT " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) );
+  sql += " FROM " + quotedIdentifier( OGR_FD_GetName( OGR_L_GetLayerDefn( ogrLayer ) ) );
+
+  sql += " WHERE " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) ) + " LIKE '%" +  textEncoding()->fromUnicode( substring ) + "%'";
+
+  if ( !mSubsetString.isEmpty() )
+  {
+    sql += " AND (" + textEncoding()->fromUnicode( mSubsetString ) + ')';
+  }
+
+  sql += " ORDER BY " + textEncoding()->fromUnicode( fld.name() ) + " ASC";  // quoting of fieldname produces a syntax error
+
+  QgsDebugMsg( QString( "SQL: %1" ).arg( textEncoding()->toUnicode( sql ) ) );
+  OGRLayerH l = OGR_DS_ExecuteSQL( ogrDataSource, sql.constData(), nullptr, nullptr );
+  if ( !l )
+  {
+    QgsDebugMsg( "Failed to execute SQL" );
+    return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
+  }
+
+  OGRFeatureH f;
+  while (( f = OGR_L_GetNextFeature( l ) ) )
+  {
+    if ( OGR_F_IsFieldSet( f, 0 ) )
+      results << textEncoding()->toUnicode( OGR_F_GetFieldAsString( f, 0 ) );
+    OGR_F_Destroy( f );
+
+    if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
+      break;
+  }
+
+  OGR_DS_ReleaseResultSet( ogrDataSource, l );
+  return results;
+#endif
+}
+
 QVariant QgsOgrProvider::minimumValue( int index ) const
 {
   if ( !mValid || index < 0 || index >= mAttributeFields.count() )

src/providers/ogr/qgsogrprovider.h

@@ -207,6 +207,9 @@ class QgsOgrProvider : public QgsVectorDataProvider
      */
     virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;
 
+    virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
+        QgsFeedback* feedback = nullptr ) const override;
+
     /** Return a provider name
      *
      * Essentially just returns the provider key.  Should be used to build file

src/providers/postgres/qgspostgresprovider.cpp

@@ -37,6 +37,7 @@
 #include "qgspostgresfeatureiterator.h"
 #include "qgspostgrestransaction.h"
 #include "qgslogger.h"
+#include "qgsfeedback.h"
 
 const QString POSTGRES_KEY = QStringLiteral( "postgres" );
 const QString POSTGRES_DESCRIPTION = QStringLiteral( "PostgreSQL/PostGIS data provider" );
@@ -1573,6 +1574,52 @@ void QgsPostgresProvider::uniqueValues( int index, QList<QVariant> &uniqueValues
   }
 }
 
+QStringList QgsPostgresProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
+{
+  QStringList results;
+
+  try
+  {
+    // get the field name
+    QgsField fld = field( index );
+    QString sql = QString( "SELECT DISTINCT %1 FROM %2 WHERE" )
+                  .arg( quotedIdentifier( fld.name() ),
+                        mQuery );
+
+    if ( !mSqlWhereClause.isEmpty() )
+    {
+      sql += QString( " ( %1 ) AND " ).arg( mSqlWhereClause );
+    }
+
+    sql += QString( " %1 ILIKE '%%2%'" ).arg( quotedIdentifier( fld.name() ), substring );
+
+
+    sql +=  QString( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );
+
+    if ( limit >= 0 )
+    {
+      sql += QString( " LIMIT %1" ).arg( limit );
+    }
+
+    sql = QString( "SELECT %1 FROM (%2) foo" ).arg( connectionRO()->fieldExpression( fld ), sql );
+
+    QgsPostgresResult res( connectionRO()->PQexec( sql ) );
+    if ( res.PQresultStatus() == PGRES_TUPLES_OK )
+    {
+      for ( int i = 0; i < res.PQntuples(); i++ )
+      {
+        results << ( convertValue( fld.type(), fld.subType(), res.PQgetvalue( i, 0 ) ) ).toString();
+        if ( feedback && feedback->isCancelled() )
+          break;
+      }
+    }
+  }
+  catch ( PGFieldNotFound )
+  {
+  }
+  return results;
+}
+
 void QgsPostgresProvider::enumValues( int index, QStringList& enumList ) const
 {
   enumList.clear();

src/providers/postgres/qgspostgresprovider.h

@@ -155,6 +155,8 @@ class QgsPostgresProvider : public QgsVectorDataProvider
     QVariant minimumValue( int index ) const override;
     QVariant maximumValue( int index ) const override;
     virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;
+    virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
+        QgsFeedback* feedback = nullptr ) const override;
     virtual void enumValues( int index, QStringList& enumList ) const override;
     bool isValid() const override;
     virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }

src/providers/spatialite/qgsspatialiteprovider.cpp

@@ -29,6 +29,7 @@ email                : a.furieri@lqt.it
 #include "qgsspatialiteprovider.h"
 #include "qgsspatialiteconnpool.h"
 #include "qgsspatialitefeatureiterator.h"
+#include "qgsfeedback.h"
 
 #include <qgsjsonutils.h>
 #include <qgsvectorlayer.h>
@@ -3702,6 +3703,79 @@ void QgsSpatiaLiteProvider::uniqueValues( int index, QList < QVariant > &uniqueV
   return;
 }
 
+QStringList QgsSpatiaLiteProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
+{
+  QStringList results;
+
+  sqlite3_stmt *stmt = nullptr;
+  QString sql;
+
+  // get the field name
+  if ( index < 0 || index >= mAttributeFields.count() )
+  {
+    return results; //invalid field
+  }
+  QgsField fld = mAttributeFields.at( index );
+
+  sql = QStringLiteral( "SELECT DISTINCT %1 FROM %2 " ).arg( quotedIdentifier( fld.name() ), mQuery );
+  sql += QStringLiteral( " WHERE " ) + quotedIdentifier( fld.name() ) + QStringLiteral( " LIKE '%" ) + substring + QStringLiteral( "%'" );
+
+  if ( !mSubsetString.isEmpty() )
+  {
+    sql += QStringLiteral( " AND ( " ) + mSubsetString + ')';
+  }
+
+  sql += QStringLiteral( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );
+
+  if ( limit >= 0 )
+  {
+    sql += QStringLiteral( " LIMIT %1" ).arg( limit );
+  }
+
+  // SQLite prepared statement
+  if ( sqlite3_prepare_v2( mSqliteHandle, sql.toUtf8().constData(), -1, &stmt, nullptr ) != SQLITE_OK )
+  {
+    // some error occurred
+    QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
+    return results;
+  }
+
+  while (( limit < 0 || results.size() < limit ) && ( !feedback || !feedback->isCancelled() ) )
+  {
+    // this one is an infinitive loop, intended to fetch any row
+    int ret = sqlite3_step( stmt );
+
+    if ( ret == SQLITE_DONE )
+    {
+      // there are no more rows to fetch - we can stop looping
+      break;
+    }
+
+    if ( ret == SQLITE_ROW )
+    {
+      // fetching one column value
+      switch ( sqlite3_column_type( stmt, 0 ) )
+      {
+        case SQLITE_TEXT:
+          results.append( QString::fromUtf8(( const char * ) sqlite3_column_text( stmt, 0 ) ) );
+          break;
+        default:
+          break;
+      }
+    }
+    else
+    {
+      QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
+      sqlite3_finalize( stmt );
+      return results;
+    }
+  }
+
+  sqlite3_finalize( stmt );
+
+  return results;
+}
+
 QString QgsSpatiaLiteProvider::geomParam() const
 {
   QString geometry;

src/providers/spatialite/qgsspatialiteprovider.h

@@ -131,6 +131,9 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
     QVariant maximumValue( int index ) const override;
     virtual void uniqueValues( int index, QList < QVariant > &uniqueValues, int limit = -1 ) const override;
 
+    virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
+        QgsFeedback* feedback = nullptr ) const override;
+
     bool isValid() const override;
     virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }
 

tests/src/python/providertestbase.py

@@ -566,6 +566,25 @@ def testUnique(self):
         self.provider.setSubsetString(None)
         self.assertEqual(set(values), set([200, 300]))
 
+    def testUniqueStringsMatching(self):
+        self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'a')), set(['Pear', 'Orange', 'Apple']))
+        # test case insensitive
+        self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'A')), set(['Pear', 'Orange', 'Apple']))
+        # test string ending in substring
+        self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'ney')), set(['Honey']))
+        # test limit
+        result = set(self.provider.uniqueStringsMatching(2, 'a', 2))
+        self.assertEqual(len(result), 2)
+        self.assertTrue(result.issubset(set(['Pear', 'Orange', 'Apple'])))
+
+        assert set([u'Apple', u'Honey', u'Orange', u'Pear', NULL]) == set(self.provider.uniqueValues(2)), 'Got {}'.format(set(self.provider.uniqueValues(2)))
+
+        subset = self.getSubsetString2()
+        self.provider.setSubsetString(subset)
+        values = self.provider.uniqueStringsMatching(2, 'a')
+        self.provider.setSubsetString(None)
+        self.assertEqual(set(values), set(['Pear', 'Apple']))
+
     def testFeatureCount(self):
         assert self.provider.featureCount() == 5, 'Got {}'.format(self.provider.featureCount())