Skip to content

Commit 3242321

Browse files
nyalldawsonm-kuhn
authored andcommitted
[FEATURE] Add method to get list of unique strings matching
a substring from a vector data provider Base implementation iterates through all features, but providers can override with optimised versions. Native implementations for postgres, spatialite and OGR included.
1 parent ff691a6 commit 3242321

File tree

10 files changed

+259
-0
lines changed

10 files changed

+259
-0
lines changed

python/core/qgsvectordataprovider.sip

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,18 @@ class QgsVectorDataProvider : QgsDataProvider
148148
*/
149149
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues /Out/, int limit = -1 ) const;
150150

151+
/**
152+
* Returns unique string values of an attribute which contain a specified subset string. Subset
153+
* matching is done in a case-insensitive manner.
154+
* @param index the index of the attribute
155+
* @param substring substring to match (case insensitive)
156+
* @param limit maxmum number of the values to return, or -1 to return all unique values
157+
* @param feedback optional feedback object for cancelling request
158+
* @returns list of unique strings containg substring
159+
*/
160+
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
161+
QgsFeedback* feedback = nullptr ) const;
162+
151163
/** Calculates an aggregated value from the layer's features. The base implementation does nothing,
152164
* but subclasses can override this method to handoff calculation of aggregates to the provider.
153165
* @param aggregate aggregate to calculate

src/core/qgsvectordataprovider.cpp

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "qgsfeature.h"
2727
#include "qgsfeatureiterator.h"
2828
#include "qgsfeaturerequest.h"
29+
#include "qgsfeedback.h"
2930
#include "qgsfields.h"
3031
#include "qgsgeometry.h"
3132
#include "qgsgeometrycollection.h"
@@ -432,6 +433,37 @@ void QgsVectorDataProvider::uniqueValues( int index, QList<QVariant> &values, in
432433
}
433434
}
434435

436+
QStringList QgsVectorDataProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
437+
{
438+
QgsFeature f;
439+
QgsAttributeList keys;
440+
keys.append( index );
441+
442+
QgsFeatureRequest request;
443+
request.setSubsetOfAttributes( keys );
444+
request.setFlags( QgsFeatureRequest::NoGeometry );
445+
QString fieldName = fields().at( index ).name();
446+
request.setFilterExpression( QStringLiteral( "\"%1\" ILIKE '%%2%'" ).arg( fieldName, substring ) );
447+
QgsFeatureIterator fi = getFeatures( request );
448+
449+
QSet<QString> set;
450+
QStringList results;
451+
452+
while ( fi.nextFeature( f ) )
453+
{
454+
QString value = f.attribute( index ).toString();
455+
if ( !set.contains( value ) )
456+
{
457+
results.append( value );
458+
set.insert( value );
459+
}
460+
461+
if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
462+
break;
463+
}
464+
return results;
465+
}
466+
435467
QVariant QgsVectorDataProvider::aggregate( QgsAggregateCalculator::Aggregate aggregate, int index,
436468
const QgsAggregateCalculator::AggregateParameters& parameters, QgsExpressionContext* context, bool& ok ) const
437469
{

src/core/qgsvectordataprovider.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ typedef QHash<int, QString> QgsAttrPalIndexNameHash;
3636

3737
class QgsFeatureIterator;
3838
class QgsTransaction;
39+
class QgsFeedback;
3940

4041
#include "qgsfeaturerequest.h"
4142

@@ -201,6 +202,18 @@ class CORE_EXPORT QgsVectorDataProvider : public QgsDataProvider
201202
*/
202203
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const;
203204

205+
/**
206+
* Returns unique string values of an attribute which contain a specified subset string. Subset
207+
* matching is done in a case-insensitive manner.
208+
* @param index the index of the attribute
209+
* @param substring substring to match (case insensitive)
210+
* @param limit maxmum number of the values to return, or -1 to return all unique values
211+
* @param feedback optional feedback object for cancelling request
212+
* @returns list of unique strings containg substring
213+
*/
214+
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
215+
QgsFeedback* feedback = nullptr ) const;
216+
204217
/** Calculates an aggregated value from the layer's features. The base implementation does nothing,
205218
* but subclasses can override this method to handoff calculation of aggregates to the provider.
206219
* @param aggregate aggregate to calculate

src/providers/ogr/qgsogrprovider.cpp

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ email : sherman at mrcc.com
2020
#include "qgslogger.h"
2121
#include "qgsmessagelog.h"
2222
#include "qgslocalec.h"
23+
#include "qgsfeedback.h"
2324

2425
#define CPL_SUPRESS_CPLUSPLUS
2526
#include <gdal.h> // to collect version information
@@ -2924,6 +2925,59 @@ void QgsOgrProvider::uniqueValues( int index, QList<QVariant> &uniqueValues, int
29242925
#endif
29252926
}
29262927

2928+
QStringList QgsOgrProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
2929+
{
2930+
QStringList results;
2931+
2932+
if ( !mValid || index < 0 || index >= mAttributeFields.count() )
2933+
return results;
2934+
2935+
QgsField fld = mAttributeFields.at( index );
2936+
if ( fld.name().isNull() )
2937+
{
2938+
return results; //not a provider field
2939+
}
2940+
2941+
#if defined(GDAL_VERSION_NUM) && GDAL_VERSION_NUM < 1910
2942+
// avoid GDAL #4509
2943+
return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
2944+
#else
2945+
QByteArray sql = "SELECT DISTINCT " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) );
2946+
sql += " FROM " + quotedIdentifier( OGR_FD_GetName( OGR_L_GetLayerDefn( ogrLayer ) ) );
2947+
2948+
sql += " WHERE " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) ) + " LIKE '%" + textEncoding()->fromUnicode( substring ) + "%'";
2949+
2950+
if ( !mSubsetString.isEmpty() )
2951+
{
2952+
sql += " AND (" + textEncoding()->fromUnicode( mSubsetString ) + ')';
2953+
}
2954+
2955+
sql += " ORDER BY " + textEncoding()->fromUnicode( fld.name() ) + " ASC"; // quoting of fieldname produces a syntax error
2956+
2957+
QgsDebugMsg( QString( "SQL: %1" ).arg( textEncoding()->toUnicode( sql ) ) );
2958+
OGRLayerH l = OGR_DS_ExecuteSQL( ogrDataSource, sql.constData(), nullptr, nullptr );
2959+
if ( !l )
2960+
{
2961+
QgsDebugMsg( "Failed to execute SQL" );
2962+
return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
2963+
}
2964+
2965+
OGRFeatureH f;
2966+
while (( f = OGR_L_GetNextFeature( l ) ) )
2967+
{
2968+
if ( OGR_F_IsFieldSet( f, 0 ) )
2969+
results << textEncoding()->toUnicode( OGR_F_GetFieldAsString( f, 0 ) );
2970+
OGR_F_Destroy( f );
2971+
2972+
if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
2973+
break;
2974+
}
2975+
2976+
OGR_DS_ReleaseResultSet( ogrDataSource, l );
2977+
return results;
2978+
#endif
2979+
}
2980+
29272981
QVariant QgsOgrProvider::minimumValue( int index ) const
29282982
{
29292983
if ( !mValid || index < 0 || index >= mAttributeFields.count() )

src/providers/ogr/qgsogrprovider.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,9 @@ class QgsOgrProvider : public QgsVectorDataProvider
207207
*/
208208
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;
209209

210+
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
211+
QgsFeedback* feedback = nullptr ) const override;
212+
210213
/** Return a provider name
211214
*
212215
* Essentially just returns the provider key. Should be used to build file

src/providers/postgres/qgspostgresprovider.cpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "qgspostgresfeatureiterator.h"
3838
#include "qgspostgrestransaction.h"
3939
#include "qgslogger.h"
40+
#include "qgsfeedback.h"
4041

4142
const QString POSTGRES_KEY = QStringLiteral( "postgres" );
4243
const QString POSTGRES_DESCRIPTION = QStringLiteral( "PostgreSQL/PostGIS data provider" );
@@ -1573,6 +1574,52 @@ void QgsPostgresProvider::uniqueValues( int index, QList<QVariant> &uniqueValues
15731574
}
15741575
}
15751576

1577+
QStringList QgsPostgresProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
1578+
{
1579+
QStringList results;
1580+
1581+
try
1582+
{
1583+
// get the field name
1584+
QgsField fld = field( index );
1585+
QString sql = QString( "SELECT DISTINCT %1 FROM %2 WHERE" )
1586+
.arg( quotedIdentifier( fld.name() ),
1587+
mQuery );
1588+
1589+
if ( !mSqlWhereClause.isEmpty() )
1590+
{
1591+
sql += QString( " ( %1 ) AND " ).arg( mSqlWhereClause );
1592+
}
1593+
1594+
sql += QString( " %1 ILIKE '%%2%'" ).arg( quotedIdentifier( fld.name() ), substring );
1595+
1596+
1597+
sql += QString( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );
1598+
1599+
if ( limit >= 0 )
1600+
{
1601+
sql += QString( " LIMIT %1" ).arg( limit );
1602+
}
1603+
1604+
sql = QString( "SELECT %1 FROM (%2) foo" ).arg( connectionRO()->fieldExpression( fld ), sql );
1605+
1606+
QgsPostgresResult res( connectionRO()->PQexec( sql ) );
1607+
if ( res.PQresultStatus() == PGRES_TUPLES_OK )
1608+
{
1609+
for ( int i = 0; i < res.PQntuples(); i++ )
1610+
{
1611+
results << ( convertValue( fld.type(), fld.subType(), res.PQgetvalue( i, 0 ) ) ).toString();
1612+
if ( feedback && feedback->isCancelled() )
1613+
break;
1614+
}
1615+
}
1616+
}
1617+
catch ( PGFieldNotFound )
1618+
{
1619+
}
1620+
return results;
1621+
}
1622+
15761623
void QgsPostgresProvider::enumValues( int index, QStringList& enumList ) const
15771624
{
15781625
enumList.clear();

src/providers/postgres/qgspostgresprovider.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ class QgsPostgresProvider : public QgsVectorDataProvider
155155
QVariant minimumValue( int index ) const override;
156156
QVariant maximumValue( int index ) const override;
157157
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;
158+
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
159+
QgsFeedback* feedback = nullptr ) const override;
158160
virtual void enumValues( int index, QStringList& enumList ) const override;
159161
bool isValid() const override;
160162
virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }

src/providers/spatialite/qgsspatialiteprovider.cpp

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ email : a.furieri@lqt.it
2929
#include "qgsspatialiteprovider.h"
3030
#include "qgsspatialiteconnpool.h"
3131
#include "qgsspatialitefeatureiterator.h"
32+
#include "qgsfeedback.h"
3233

3334
#include <qgsjsonutils.h>
3435
#include <qgsvectorlayer.h>
@@ -3702,6 +3703,79 @@ void QgsSpatiaLiteProvider::uniqueValues( int index, QList < QVariant > &uniqueV
37023703
return;
37033704
}
37043705

3706+
QStringList QgsSpatiaLiteProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
3707+
{
3708+
QStringList results;
3709+
3710+
sqlite3_stmt *stmt = nullptr;
3711+
QString sql;
3712+
3713+
// get the field name
3714+
if ( index < 0 || index >= mAttributeFields.count() )
3715+
{
3716+
return results; //invalid field
3717+
}
3718+
QgsField fld = mAttributeFields.at( index );
3719+
3720+
sql = QStringLiteral( "SELECT DISTINCT %1 FROM %2 " ).arg( quotedIdentifier( fld.name() ), mQuery );
3721+
sql += QStringLiteral( " WHERE " ) + quotedIdentifier( fld.name() ) + QStringLiteral( " LIKE '%" ) + substring + QStringLiteral( "%'" );
3722+
3723+
if ( !mSubsetString.isEmpty() )
3724+
{
3725+
sql += QStringLiteral( " AND ( " ) + mSubsetString + ')';
3726+
}
3727+
3728+
sql += QStringLiteral( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );
3729+
3730+
if ( limit >= 0 )
3731+
{
3732+
sql += QStringLiteral( " LIMIT %1" ).arg( limit );
3733+
}
3734+
3735+
// SQLite prepared statement
3736+
if ( sqlite3_prepare_v2( mSqliteHandle, sql.toUtf8().constData(), -1, &stmt, nullptr ) != SQLITE_OK )
3737+
{
3738+
// some error occurred
3739+
QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
3740+
return results;
3741+
}
3742+
3743+
while (( limit < 0 || results.size() < limit ) && ( !feedback || !feedback->isCancelled() ) )
3744+
{
3745+
// this one is an infinitive loop, intended to fetch any row
3746+
int ret = sqlite3_step( stmt );
3747+
3748+
if ( ret == SQLITE_DONE )
3749+
{
3750+
// there are no more rows to fetch - we can stop looping
3751+
break;
3752+
}
3753+
3754+
if ( ret == SQLITE_ROW )
3755+
{
3756+
// fetching one column value
3757+
switch ( sqlite3_column_type( stmt, 0 ) )
3758+
{
3759+
case SQLITE_TEXT:
3760+
results.append( QString::fromUtf8(( const char * ) sqlite3_column_text( stmt, 0 ) ) );
3761+
break;
3762+
default:
3763+
break;
3764+
}
3765+
}
3766+
else
3767+
{
3768+
QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
3769+
sqlite3_finalize( stmt );
3770+
return results;
3771+
}
3772+
}
3773+
3774+
sqlite3_finalize( stmt );
3775+
3776+
return results;
3777+
}
3778+
37053779
QString QgsSpatiaLiteProvider::geomParam() const
37063780
{
37073781
QString geometry;

src/providers/spatialite/qgsspatialiteprovider.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
131131
QVariant maximumValue( int index ) const override;
132132
virtual void uniqueValues( int index, QList < QVariant > &uniqueValues, int limit = -1 ) const override;
133133

134+
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
135+
QgsFeedback* feedback = nullptr ) const override;
136+
134137
bool isValid() const override;
135138
virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }
136139

tests/src/python/providertestbase.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,25 @@ def testUnique(self):
566566
self.provider.setSubsetString(None)
567567
self.assertEqual(set(values), set([200, 300]))
568568

569+
def testUniqueStringsMatching(self):
570+
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'a')), set(['Pear', 'Orange', 'Apple']))
571+
# test case insensitive
572+
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'A')), set(['Pear', 'Orange', 'Apple']))
573+
# test string ending in substring
574+
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'ney')), set(['Honey']))
575+
# test limit
576+
result = set(self.provider.uniqueStringsMatching(2, 'a', 2))
577+
self.assertEqual(len(result), 2)
578+
self.assertTrue(result.issubset(set(['Pear', 'Orange', 'Apple'])))
579+
580+
assert set([u'Apple', u'Honey', u'Orange', u'Pear', NULL]) == set(self.provider.uniqueValues(2)), 'Got {}'.format(set(self.provider.uniqueValues(2)))
581+
582+
subset = self.getSubsetString2()
583+
self.provider.setSubsetString(subset)
584+
values = self.provider.uniqueStringsMatching(2, 'a')
585+
self.provider.setSubsetString(None)
586+
self.assertEqual(set(values), set(['Pear', 'Apple']))
587+
569588
def testFeatureCount(self):
570589
assert self.provider.featureCount() == 5, 'Got {}'.format(self.provider.featureCount())
571590

0 commit comments

Comments
 (0)