Skip to content
Permalink
Browse files

[FEATURE] Add method to get list of unique strings matching

a substring from a vector data provider

Base implementation iterates through all features, but
providers can override with optimised versions. Native
implementations for postgres, spatialite and OGR included.
  • Loading branch information
nyalldawson authored and m-kuhn committed Nov 16, 2016
1 parent ff691a6 commit 3242321aa9a3bc81d3db369edeadea4922ea861d
@@ -148,6 +148,18 @@ class QgsVectorDataProvider : QgsDataProvider
*/
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues /Out/, int limit = -1 ) const;

/**
* Returns unique string values of an attribute which contain a specified subset string. Subset
* matching is done in a case-insensitive manner.
* @param index the index of the attribute
* @param substring substring to match (case insensitive)
* @param limit maxmum number of the values to return, or -1 to return all unique values
* @param feedback optional feedback object for cancelling request
* @returns list of unique strings containg substring
*/
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
QgsFeedback* feedback = nullptr ) const;

/** Calculates an aggregated value from the layer's features. The base implementation does nothing,
* but subclasses can override this method to handoff calculation of aggregates to the provider.
* @param aggregate aggregate to calculate
@@ -26,6 +26,7 @@
#include "qgsfeature.h"
#include "qgsfeatureiterator.h"
#include "qgsfeaturerequest.h"
#include "qgsfeedback.h"
#include "qgsfields.h"
#include "qgsgeometry.h"
#include "qgsgeometrycollection.h"
@@ -432,6 +433,37 @@ void QgsVectorDataProvider::uniqueValues( int index, QList<QVariant> &values, in
}
}

QStringList QgsVectorDataProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
{
QgsFeature f;
QgsAttributeList keys;
keys.append( index );

QgsFeatureRequest request;
request.setSubsetOfAttributes( keys );
request.setFlags( QgsFeatureRequest::NoGeometry );
QString fieldName = fields().at( index ).name();
request.setFilterExpression( QStringLiteral( "\"%1\" ILIKE '%%2%'" ).arg( fieldName, substring ) );
QgsFeatureIterator fi = getFeatures( request );

QSet<QString> set;
QStringList results;

while ( fi.nextFeature( f ) )
{
QString value = f.attribute( index ).toString();
if ( !set.contains( value ) )
{
results.append( value );
set.insert( value );
}

if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
break;
}
return results;
}

QVariant QgsVectorDataProvider::aggregate( QgsAggregateCalculator::Aggregate aggregate, int index,
const QgsAggregateCalculator::AggregateParameters& parameters, QgsExpressionContext* context, bool& ok ) const
{
@@ -36,6 +36,7 @@ typedef QHash<int, QString> QgsAttrPalIndexNameHash;

class QgsFeatureIterator;
class QgsTransaction;
class QgsFeedback;

#include "qgsfeaturerequest.h"

@@ -201,6 +202,18 @@ class CORE_EXPORT QgsVectorDataProvider : public QgsDataProvider
*/
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const;

/**
* Returns unique string values of an attribute which contain a specified subset string. Subset
* matching is done in a case-insensitive manner.
* @param index the index of the attribute
* @param substring substring to match (case insensitive)
* @param limit maxmum number of the values to return, or -1 to return all unique values
* @param feedback optional feedback object for cancelling request
* @returns list of unique strings containg substring
*/
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
QgsFeedback* feedback = nullptr ) const;

/** Calculates an aggregated value from the layer's features. The base implementation does nothing,
* but subclasses can override this method to handoff calculation of aggregates to the provider.
* @param aggregate aggregate to calculate
@@ -20,6 +20,7 @@ email : sherman at mrcc.com
#include "qgslogger.h"
#include "qgsmessagelog.h"
#include "qgslocalec.h"
#include "qgsfeedback.h"

#define CPL_SUPRESS_CPLUSPLUS
#include <gdal.h> // to collect version information
@@ -2924,6 +2925,59 @@ void QgsOgrProvider::uniqueValues( int index, QList<QVariant> &uniqueValues, int
#endif
}

QStringList QgsOgrProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
{
QStringList results;

if ( !mValid || index < 0 || index >= mAttributeFields.count() )
return results;

QgsField fld = mAttributeFields.at( index );
if ( fld.name().isNull() )
{
return results; //not a provider field
}

#if defined(GDAL_VERSION_NUM) && GDAL_VERSION_NUM < 1910
// avoid GDAL #4509
return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
#else
QByteArray sql = "SELECT DISTINCT " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) );
sql += " FROM " + quotedIdentifier( OGR_FD_GetName( OGR_L_GetLayerDefn( ogrLayer ) ) );

sql += " WHERE " + quotedIdentifier( textEncoding()->fromUnicode( fld.name() ) ) + " LIKE '%" + textEncoding()->fromUnicode( substring ) + "%'";

if ( !mSubsetString.isEmpty() )
{
sql += " AND (" + textEncoding()->fromUnicode( mSubsetString ) + ')';
}

sql += " ORDER BY " + textEncoding()->fromUnicode( fld.name() ) + " ASC"; // quoting of fieldname produces a syntax error

QgsDebugMsg( QString( "SQL: %1" ).arg( textEncoding()->toUnicode( sql ) ) );
OGRLayerH l = OGR_DS_ExecuteSQL( ogrDataSource, sql.constData(), nullptr, nullptr );
if ( !l )
{
QgsDebugMsg( "Failed to execute SQL" );
return QgsVectorDataProvider::uniqueStringsMatching( index, substring, limit, feedback );
}

OGRFeatureH f;
while (( f = OGR_L_GetNextFeature( l ) ) )
{
if ( OGR_F_IsFieldSet( f, 0 ) )
results << textEncoding()->toUnicode( OGR_F_GetFieldAsString( f, 0 ) );
OGR_F_Destroy( f );

if (( limit >= 0 && results.size() >= limit ) || ( feedback && feedback->isCancelled() ) )
break;
}

OGR_DS_ReleaseResultSet( ogrDataSource, l );
return results;
#endif
}

QVariant QgsOgrProvider::minimumValue( int index ) const
{
if ( !mValid || index < 0 || index >= mAttributeFields.count() )
@@ -207,6 +207,9 @@ class QgsOgrProvider : public QgsVectorDataProvider
*/
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;

virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
QgsFeedback* feedback = nullptr ) const override;

/** Return a provider name
*
* Essentially just returns the provider key. Should be used to build file
@@ -37,6 +37,7 @@
#include "qgspostgresfeatureiterator.h"
#include "qgspostgrestransaction.h"
#include "qgslogger.h"
#include "qgsfeedback.h"

const QString POSTGRES_KEY = QStringLiteral( "postgres" );
const QString POSTGRES_DESCRIPTION = QStringLiteral( "PostgreSQL/PostGIS data provider" );
@@ -1573,6 +1574,52 @@ void QgsPostgresProvider::uniqueValues( int index, QList<QVariant> &uniqueValues
}
}

QStringList QgsPostgresProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
{
QStringList results;

try
{
// get the field name
QgsField fld = field( index );
QString sql = QString( "SELECT DISTINCT %1 FROM %2 WHERE" )
.arg( quotedIdentifier( fld.name() ),
mQuery );

if ( !mSqlWhereClause.isEmpty() )
{
sql += QString( " ( %1 ) AND " ).arg( mSqlWhereClause );
}

sql += QString( " %1 ILIKE '%%2%'" ).arg( quotedIdentifier( fld.name() ), substring );


sql += QString( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );

if ( limit >= 0 )
{
sql += QString( " LIMIT %1" ).arg( limit );
}

sql = QString( "SELECT %1 FROM (%2) foo" ).arg( connectionRO()->fieldExpression( fld ), sql );

QgsPostgresResult res( connectionRO()->PQexec( sql ) );
if ( res.PQresultStatus() == PGRES_TUPLES_OK )
{
for ( int i = 0; i < res.PQntuples(); i++ )
{
results << ( convertValue( fld.type(), fld.subType(), res.PQgetvalue( i, 0 ) ) ).toString();
if ( feedback && feedback->isCancelled() )
break;
}
}
}
catch ( PGFieldNotFound )
{
}
return results;
}

void QgsPostgresProvider::enumValues( int index, QStringList& enumList ) const
{
enumList.clear();
@@ -155,6 +155,8 @@ class QgsPostgresProvider : public QgsVectorDataProvider
QVariant minimumValue( int index ) const override;
QVariant maximumValue( int index ) const override;
virtual void uniqueValues( int index, QList<QVariant> &uniqueValues, int limit = -1 ) const override;
virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
QgsFeedback* feedback = nullptr ) const override;
virtual void enumValues( int index, QStringList& enumList ) const override;
bool isValid() const override;
virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }
@@ -29,6 +29,7 @@ email : a.furieri@lqt.it
#include "qgsspatialiteprovider.h"
#include "qgsspatialiteconnpool.h"
#include "qgsspatialitefeatureiterator.h"
#include "qgsfeedback.h"

#include <qgsjsonutils.h>
#include <qgsvectorlayer.h>
@@ -3702,6 +3703,79 @@ void QgsSpatiaLiteProvider::uniqueValues( int index, QList < QVariant > &uniqueV
return;
}

QStringList QgsSpatiaLiteProvider::uniqueStringsMatching( int index, const QString& substring, int limit, QgsFeedback* feedback ) const
{
QStringList results;

sqlite3_stmt *stmt = nullptr;
QString sql;

// get the field name
if ( index < 0 || index >= mAttributeFields.count() )
{
return results; //invalid field
}
QgsField fld = mAttributeFields.at( index );

sql = QStringLiteral( "SELECT DISTINCT %1 FROM %2 " ).arg( quotedIdentifier( fld.name() ), mQuery );
sql += QStringLiteral( " WHERE " ) + quotedIdentifier( fld.name() ) + QStringLiteral( " LIKE '%" ) + substring + QStringLiteral( "%'" );

if ( !mSubsetString.isEmpty() )
{
sql += QStringLiteral( " AND ( " ) + mSubsetString + ')';
}

sql += QStringLiteral( " ORDER BY %1" ).arg( quotedIdentifier( fld.name() ) );

if ( limit >= 0 )
{
sql += QStringLiteral( " LIMIT %1" ).arg( limit );
}

// SQLite prepared statement
if ( sqlite3_prepare_v2( mSqliteHandle, sql.toUtf8().constData(), -1, &stmt, nullptr ) != SQLITE_OK )
{
// some error occurred
QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
return results;
}

while (( limit < 0 || results.size() < limit ) && ( !feedback || !feedback->isCancelled() ) )
{
// this one is an infinitive loop, intended to fetch any row
int ret = sqlite3_step( stmt );

if ( ret == SQLITE_DONE )
{
// there are no more rows to fetch - we can stop looping
break;
}

if ( ret == SQLITE_ROW )
{
// fetching one column value
switch ( sqlite3_column_type( stmt, 0 ) )
{
case SQLITE_TEXT:
results.append( QString::fromUtf8(( const char * ) sqlite3_column_text( stmt, 0 ) ) );
break;
default:
break;
}
}
else
{
QgsMessageLog::logMessage( tr( "SQLite error: %2\nSQL: %1" ).arg( sql, sqlite3_errmsg( mSqliteHandle ) ), tr( "SpatiaLite" ) );
sqlite3_finalize( stmt );
return results;
}
}

sqlite3_finalize( stmt );

return results;
}

QString QgsSpatiaLiteProvider::geomParam() const
{
QString geometry;
@@ -131,6 +131,9 @@ class QgsSpatiaLiteProvider: public QgsVectorDataProvider
QVariant maximumValue( int index ) const override;
virtual void uniqueValues( int index, QList < QVariant > &uniqueValues, int limit = -1 ) const override;

virtual QStringList uniqueStringsMatching( int index, const QString& substring, int limit = -1,
QgsFeedback* feedback = nullptr ) const override;

bool isValid() const override;
virtual bool isSaveAndLoadStyleToDBSupported() const override { return true; }

@@ -566,6 +566,25 @@ def testUnique(self):
self.provider.setSubsetString(None)
self.assertEqual(set(values), set([200, 300]))

def testUniqueStringsMatching(self):
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'a')), set(['Pear', 'Orange', 'Apple']))
# test case insensitive
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'A')), set(['Pear', 'Orange', 'Apple']))
# test string ending in substring
self.assertEqual(set(self.provider.uniqueStringsMatching(2, 'ney')), set(['Honey']))
# test limit
result = set(self.provider.uniqueStringsMatching(2, 'a', 2))
self.assertEqual(len(result), 2)
self.assertTrue(result.issubset(set(['Pear', 'Orange', 'Apple'])))

assert set([u'Apple', u'Honey', u'Orange', u'Pear', NULL]) == set(self.provider.uniqueValues(2)), 'Got {}'.format(set(self.provider.uniqueValues(2)))

subset = self.getSubsetString2()
self.provider.setSubsetString(subset)
values = self.provider.uniqueStringsMatching(2, 'a')
self.provider.setSubsetString(None)
self.assertEqual(set(values), set(['Pear', 'Apple']))

def testFeatureCount(self):
assert self.provider.featureCount() == 5, 'Got {}'.format(self.provider.featureCount())

0 comments on commit 3242321

Please sign in to comment.
You can’t perform that action at this time.