Skip to content
Permalink
Browse files
Merge pull request #44347 from domi4484/backport-44099-to-queued_ltr_…
…backports

Backport "Added missing type names to uri parsing regex for memory provider" ltr
  • Loading branch information
m-kuhn committed Aug 27, 2021
2 parents 9f19631 + e1a20d1 commit d5b2217768683e448aaad411a82cd7ebfa9559cc
@@ -25,7 +25,7 @@

#include <QUrl>
#include <QUrlQuery>
#include <QRegExp>
#include <QRegularExpression>

///@cond PRIVATE

@@ -103,7 +103,7 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "text" ), QVariant::String, -1, -1, -1, -1 )

// boolean
<< QgsVectorDataProvider::NativeType( tr( "Boolean" ), QStringLiteral( "bool" ), QVariant::Bool )
<< QgsVectorDataProvider::NativeType( tr( "Boolean" ), QStringLiteral( "boolean" ), QVariant::Bool )

// blob
<< QgsVectorDataProvider::NativeType( tr( "Binary object (BLOB)" ), QStringLiteral( "binary" ), QVariant::ByteArray )
@@ -113,94 +113,101 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
if ( query.hasQueryItem( QStringLiteral( "field" ) ) )
{
QList<QgsField> attributes;
QRegExp reFieldDef( "\\:"
"(int|integer|long|int8|real|double|string|date|time|datetime|binary|bool|boolean)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$", Qt::CaseInsensitive );
const thread_local QRegularExpression reFieldDef( "\\:"
"([\\w\\s]+)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$",
QRegularExpression::CaseInsensitiveOption );
QStringList fields = query.allQueryItemValues( QStringLiteral( "field" ) );
for ( int i = 0; i < fields.size(); i++ )
{
QString name = QUrl::fromPercentEncoding( fields.at( i ).toUtf8() );
QRegularExpressionMatch regularExpressionMatch = reFieldDef.match( name );

// If no match -> use string as type
QVariant::Type type = QVariant::String;
QVariant::Type subType = QVariant::Invalid;
QString typeName( QStringLiteral( "string" ) );
int length = 255;
int precision = 0;

int pos = reFieldDef.indexIn( name );
if ( pos >= 0 )
if ( regularExpressionMatch.hasMatch() )
{
name = name.mid( 0, pos );
typeName = reFieldDef.cap( 1 ).toLower();
if ( typeName == QLatin1String( "int" ) || typeName == QLatin1String( "integer" ) )
name = name.mid( 0, regularExpressionMatch.capturedStart() );
typeName = regularExpressionMatch.captured( 1 ).toLower();

// Search typeName correspondence in native types
bool isNativeType = false;
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
{
type = QVariant::Int;
typeName = QStringLiteral( "integer" );
length = -1;
if ( nativeType.mTypeName.toLower() == typeName )
{
isNativeType = true;
type = nativeType.mType;
subType = nativeType.mSubType;
typeName = nativeType.mTypeName;
break;
}
}
else if ( typeName == QLatin1String( "int8" ) || typeName == QLatin1String( "long" ) )

// Not a native type -> check other supported types:
if ( isNativeType == false )
{
type = QVariant::LongLong;
typeName = QStringLiteral( "int8" );
length = -1;
if ( typeName == QLatin1String( "int" ) )
{
type = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( typeName == QLatin1String( "long" ) )
{
type = QVariant::LongLong;
typeName = QStringLiteral( "int8" );
}
else if ( typeName == QLatin1String( "bool" ) )
{
type = QVariant::Bool;
typeName = QStringLiteral( "boolean" );
}
else
{
QgsLogger::warning( tr( "Unsupported typeName '%1'. Will be handled as string." ).arg( typeName ) );
type = QVariant::String;
typeName = QStringLiteral( "string" );
}
}
else if ( typeName == QLatin1String( "real" ) || typeName == QLatin1String( "double" ) )

// Set default length/precision for double/real
if ( typeName == QLatin1String( "real" ) || typeName == QLatin1String( "double" ) )
{
type = QVariant::Double;
typeName = QStringLiteral( "double" );
length = 20;
precision = 5;
}
else if ( typeName == QLatin1String( "date" ) )
{
type = QVariant::Date;
typeName = QStringLiteral( "date" );
length = -1;
}
else if ( typeName == QLatin1String( "time" ) )
{
type = QVariant::Time;
typeName = QStringLiteral( "time" );
length = -1;
}
else if ( typeName == QLatin1String( "datetime" ) )
{
type = QVariant::DateTime;
typeName = QStringLiteral( "datetime" );
length = -1;
}
else if ( typeName == QLatin1String( "bool" ) || typeName == QLatin1String( "boolean" ) )
{
type = QVariant::Bool;
typeName = QStringLiteral( "boolean" );
length = -1;
}
else if ( typeName == QLatin1String( "binary" ) )
{
type = QVariant::ByteArray;
typeName = QStringLiteral( "binary" );
length = -1;
}

if ( !reFieldDef.cap( 2 ).isEmpty() )
{
length = reFieldDef.cap( 2 ).toInt();
}
if ( !reFieldDef.cap( 3 ).isEmpty() )
{
precision = reFieldDef.cap( 3 ).toInt();
}
if ( !reFieldDef.cap( 4 ).isEmpty() )
if ( !regularExpressionMatch.captured( 2 ).isEmpty() )
length = regularExpressionMatch.captured( 2 ).toInt();

if ( !regularExpressionMatch.captured( 3 ).isEmpty() )
precision = regularExpressionMatch.captured( 3 ).toInt();

// Array
if ( !regularExpressionMatch.captured( 4 ).isEmpty() )
{
//array
subType = type;
type = ( subType == QVariant::String ? QVariant::StringList : QVariant::List );
if ( subType == QVariant::Invalid )
subType = type;

if ( type != QVariant::List && type != QVariant::StringList )
type = type == QVariant::String ? QVariant::StringList : QVariant::List;

const QLatin1String listSuffix( "list" );
if ( !typeName.endsWith( listSuffix ) )
typeName += QLatin1String( "list" );
}
}
if ( !name.isEmpty() )
attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );

attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );
}
addAttributes( attributes );
}
@@ -270,9 +277,38 @@ QString QgsMemoryProvider::dataSourceUri( bool expandAuthConfig ) const
QgsAttributeList attrs = const_cast<QgsMemoryProvider *>( this )->attributeIndexes();
for ( int i = 0; i < attrs.size(); i++ )
{
QgsField field = mFields.at( attrs[i] );
const QgsField field = mFields.at( attrs[i] );
QString fieldDef = field.name();
fieldDef.append( QStringLiteral( ":%2(%3,%4)" ).arg( field.typeName() ).arg( field.length() ).arg( field.precision() ) );

QString typeName = field.typeName();
bool isList = false;
if ( field.type() == QVariant::List || field.type() == QVariant::StringList )
{
switch ( field.subType() )
{
case QVariant::Int:
typeName = QStringLiteral( "integer" );
break;

case QVariant::LongLong:
typeName = QStringLiteral( "long" );
break;

case QVariant::Double:
typeName = QStringLiteral( "double" );
break;

case QVariant::String:
typeName = QStringLiteral( "string" );
break;

default:
break;
}
isList = true;
}

fieldDef.append( QStringLiteral( ":%2(%3,%4)%5" ).arg( typeName ).arg( field.length() ).arg( field.precision() ).arg( isList ? QStringLiteral( "[]" ) : QString() ) );
query.addQueryItem( QStringLiteral( "field" ), fieldDef );
}
uri.setQuery( query );
@@ -520,9 +556,9 @@ bool QgsMemoryProvider::deleteFeatures( const QgsFeatureIds &id )

bool QgsMemoryProvider::addAttributes( const QList<QgsField> &attributes )
{
for ( QList<QgsField>::const_iterator it = attributes.begin(); it != attributes.end(); ++it )
for ( QgsField field : attributes )
{
switch ( it->type() )
switch ( field.type() )
{
case QVariant::Int:
case QVariant::Double:
@@ -537,11 +573,36 @@ bool QgsMemoryProvider::addAttributes( const QList<QgsField> &attributes )
case QVariant::ByteArray:
break;
default:
QgsDebugMsg( "Field type not supported: " + it->typeName() );
QgsDebugMsg( "Field type not supported: " + field.typeName() );
continue;
}

// Make sure added attributes typeName correspond to a native type name
bool isNativeTypeName = false;
NativeType nativeTypeCandidate( QString(), QString(), QVariant::Invalid );
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
{
if ( nativeType.mTypeName.toLower() == field.typeName().toLower() )
{
isNativeTypeName = true;
break;
}

if ( nativeType.mType == field.type()
&& nativeTypeCandidate.mType == QVariant::Invalid )
nativeTypeCandidate = nativeType;
}
if ( !isNativeTypeName )
{
if ( nativeTypeCandidate.mType != QVariant::Invalid )
{
field.setTypeName( nativeTypeCandidate.mTypeName );
}
}

// add new field as a last one
mFields.append( *it );
mFields.append( field );

for ( QgsFeatureMap::iterator fit = mFeatures.begin(); fit != mFeatures.end(); ++fit )
{
@@ -312,7 +312,7 @@ QString QgsField::displayString( const QVariant &v ) const
}
}
// Default for doubles with precision
else if ( d->type == QVariant::Double && d->precision > 0 )
else if ( d->precision > 0 )
{
if ( -1 < v.toDouble() && v.toDouble() < 1 )
{
@@ -695,7 +695,7 @@ def testSpatialIndex(self):
vl.dataProvider().createSpatialIndex()
self.assertEqual(vl.hasSpatialIndex(), QgsFeatureSource.SpatialIndexPresent)

def testClone(self):
def testCloneId(self):
"""Test that a cloned layer has a single new id and
the same fields as the source layer"""

@@ -709,7 +709,7 @@ def testClone(self):
QgsField("size", QVariant.Double)]))
vl2 = vl.clone()
self.assertTrue(
'memory?geometry=Point&crs=EPSG:4326&field=name:(0,0)&field=age:(0,0)&field=size:(0,0)' in vl2.publicSource())
'memory?geometry=Point&crs=EPSG:4326&field=name:string(0,0)&field=age:integer(0,0)&field=size:double(0,0)' in vl2.publicSource())
self.assertEqual(len(parse_qs(vl.publicSource())['uid']), 1)
self.assertEqual(len(parse_qs(vl2.publicSource())['uid']), 1)
self.assertNotEqual(parse_qs(vl2.publicSource())['uid'][0], parse_qs(vl.publicSource())['uid'][0])
@@ -826,6 +826,55 @@ def testTypeValidation(self):
f = vl.getFeature(1)
self.assertEqual(f.attribute('int'), 123)

def testAddAttributes(self):
"""Test that fields with empty/invalid typenames are updated to native type names"""

vl = QgsVectorLayer("Point", "temporary_points", "memory")
pr = vl.dataProvider()

# add fields
pr.addAttributes([QgsField("name", QVariant.String),
QgsField("age", QVariant.Int, "invalidInteger"),
QgsField("size", QVariant.Double),
QgsField("mytext", QVariant.String, "text"),
QgsField("size2", QVariant.Double, "double precision"),
QgsField("short", QVariant.Int, "int2"),
QgsField("lessshort", QVariant.Int, "int4"),
QgsField("numericfield", QVariant.Double, "numeric"),
QgsField("decimalfield", QVariant.Double, "decimal")])

self.assertEqual(pr.fields()[0].typeName(), "string")
self.assertEqual(pr.fields()[1].typeName(), "integer")
self.assertEqual(pr.fields()[2].typeName(), "double")
self.assertEqual(pr.fields()[3].typeName(), "text")
self.assertEqual(pr.fields()[4].typeName(), "double precision")
self.assertEqual(pr.fields()[5].typeName(), "int2")
self.assertEqual(pr.fields()[6].typeName(), "int4")
self.assertEqual(pr.fields()[7].typeName(), "numeric")
self.assertEqual(pr.fields()[8].typeName(), "decimal")

vl2 = vl.clone()

self.assertEqual(pr.fields()[0].name(), vl2.fields()[0].name())
self.assertEqual(pr.fields()[1].name(), vl2.fields()[1].name())
self.assertEqual(pr.fields()[2].name(), vl2.fields()[2].name())
self.assertEqual(pr.fields()[3].name(), vl2.fields()[3].name())
self.assertEqual(pr.fields()[4].name(), vl2.fields()[4].name())
self.assertEqual(pr.fields()[5].name(), vl2.fields()[5].name())
self.assertEqual(pr.fields()[6].name(), vl2.fields()[6].name())
self.assertEqual(pr.fields()[7].name(), vl2.fields()[7].name())
self.assertEqual(pr.fields()[8].name(), vl2.fields()[8].name())

self.assertEqual(pr.fields()[0].typeName(), vl2.fields()[0].typeName())
self.assertEqual(pr.fields()[1].typeName(), vl2.fields()[1].typeName())
self.assertEqual(pr.fields()[2].typeName(), vl2.fields()[2].typeName())
self.assertEqual(pr.fields()[3].typeName(), vl2.fields()[3].typeName())
self.assertEqual(pr.fields()[4].typeName(), vl2.fields()[4].typeName())
self.assertEqual(pr.fields()[5].typeName(), vl2.fields()[5].typeName())
self.assertEqual(pr.fields()[6].typeName(), vl2.fields()[6].typeName())
self.assertEqual(pr.fields()[7].typeName(), vl2.fields()[7].typeName())
self.assertEqual(pr.fields()[8].typeName(), vl2.fields()[8].typeName())


class TestPyQgsMemoryProviderIndexed(unittest.TestCase, ProviderTestCase):
"""Runs the provider test suite against an indexed memory layer"""
@@ -493,7 +493,7 @@ def testInvalidPrimaryKey(self):

# create auxiliary storage based on the invalid field
s = QgsAuxiliaryStorage()
pkf = vl.fields().field(vl.fields().indexOf('invalid_pk'))
pkf = field
al = s.createAuxiliaryLayer(pkf, vl)

self.assertEqual(al, None)

0 comments on commit d5b2217

Please sign in to comment.