Skip to content
Permalink
Browse files
Added missing type names to uri parsing regex for memory provider
- This was causing strange field names when cloning layers. fixes #44087
- Some code cleanup + QRegexp replaced with QRegularExpression
  • Loading branch information
domi4484 committed Aug 17, 2021
1 parent 84f5d18 commit 847d4cc4a50c83c8709e6335378cb78386bb41ac
Showing with 160 additions and 79 deletions.
  1. +99 −79 src/core/providers/memory/qgsmemoryprovider.cpp
  2. +61 −0 tests/src/python/test_provider_memory.py
@@ -25,7 +25,7 @@

#include <QUrl>
#include <QUrlQuery>
#include <QRegExp>
#include <QRegularExpression>

///@cond PRIVATE

@@ -112,95 +112,108 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions

if ( query.hasQueryItem( QStringLiteral( "field" ) ) )
{
QStringList parseTypes;
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
parseTypes.append( nativeType.mTypeName );

// Additional supported types (will be converted to native)
parseTypes.append( QLatin1String( "int" ) );
parseTypes.append( QLatin1String( "long" ) );
parseTypes.append( QLatin1String( "bool" ) );

QList<QgsField> attributes;
QRegExp reFieldDef( "\\:"
"(int|integer|long|int8|real|double|string|date|time|datetime|binary|bool|boolean)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$", Qt::CaseInsensitive );
QRegularExpression reFieldDef( QString( "\\:"
"(%1)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$" ).arg( parseTypes.join( '|' ) ),
QRegularExpression::CaseInsensitiveOption );
QStringList fields = query.allQueryItemValues( QStringLiteral( "field" ) );
for ( int i = 0; i < fields.size(); i++ )
{
QString name = QUrl::fromPercentEncoding( fields.at( i ).toUtf8() );
QRegularExpressionMatch regularExpressionMatch = reFieldDef.match( name );
if ( !regularExpressionMatch.hasMatch() )
{
QgsLogger::warning( tr( "Could not correctly parse field '%1' from uri '%2'" ).arg( name, uri ) );
continue;
}

name = name.mid( 0, regularExpressionMatch.capturedStart() );

QVariant::Type type = QVariant::String;
QVariant::Type subType = QVariant::Invalid;
QString typeName( QStringLiteral( "string" ) );
int length = 255;
int precision = 0;
QString typeName( regularExpressionMatch.captured( 1 ).toLower() );

// Search typname correspondance in native types
bool isNativeType = false;
for ( const NativeType &nativeType : nativeTypesList )
{
if ( nativeType.mTypeName.toLower() == typeName )
{
isNativeType = true;
type = nativeType.mType;
subType = nativeType.mSubType;
typeName = nativeType.mTypeName;
break;
}
}

int pos = reFieldDef.indexIn( name );
if ( pos >= 0 )
// Not a native type -> check other supported types:
if ( isNativeType == false )
{
name = name.mid( 0, pos );
typeName = reFieldDef.cap( 1 ).toLower();
if ( typeName == QLatin1String( "int" ) || typeName == QLatin1String( "integer" ) )
if ( typeName == QLatin1String( "int" ) )
{
type = QVariant::Int;
typeName = QStringLiteral( "integer" );
length = -1;
}
else if ( typeName == QLatin1String( "int8" ) || typeName == QLatin1String( "long" ) )
else if ( typeName == QLatin1String( "long" ) )
{
type = QVariant::LongLong;
typeName = QStringLiteral( "int8" );
length = -1;
}
else if ( typeName == QLatin1String( "real" ) || typeName == QLatin1String( "double" ) )
{
type = QVariant::Double;
typeName = QStringLiteral( "double" );
length = 20;
precision = 5;
}
else if ( typeName == QLatin1String( "date" ) )
{
type = QVariant::Date;
typeName = QStringLiteral( "date" );
length = -1;
}
else if ( typeName == QLatin1String( "time" ) )
{
type = QVariant::Time;
typeName = QStringLiteral( "time" );
length = -1;
}
else if ( typeName == QLatin1String( "datetime" ) )
{
type = QVariant::DateTime;
typeName = QStringLiteral( "datetime" );
length = -1;
}
else if ( typeName == QLatin1String( "bool" ) || typeName == QLatin1String( "boolean" ) )
else if ( typeName == QLatin1String( "bool" ) )
{
type = QVariant::Bool;
typeName = QStringLiteral( "boolean" );
length = -1;
}
else if ( typeName == QLatin1String( "binary" ) )
else
{
type = QVariant::ByteArray;
typeName = QStringLiteral( "binary" );
length = -1;
QgsLogger::warning( tr( "Unsupported typeName '%1'. Will be handled as string." ).arg( typeName ) );
type = QVariant::String;
typeName = QStringLiteral( "string" );
}
}

if ( !reFieldDef.cap( 2 ).isEmpty() )
{
length = reFieldDef.cap( 2 ).toInt();
}
if ( !reFieldDef.cap( 3 ).isEmpty() )
{
precision = reFieldDef.cap( 3 ).toInt();
}
if ( !reFieldDef.cap( 4 ).isEmpty() )
{
//array
int length = -1;
if ( !regularExpressionMatch.captured( 2 ).isEmpty() )
{
length = regularExpressionMatch.captured( 2 ).toInt();
}

int precision = 0;
if ( !regularExpressionMatch.captured( 3 ).isEmpty() )
{
precision = regularExpressionMatch.captured( 3 ).toInt();
}

// Array
if ( !regularExpressionMatch.captured( 4 ).isEmpty() )
{
if ( subType == QVariant::Invalid )
subType = type;
type = ( subType == QVariant::String ? QVariant::StringList : QVariant::List );
}

if ( type != QVariant::List && type != QVariant::StringList )
type = type == QVariant::String ? QVariant::StringList : QVariant::List;

const QLatin1String listSuffix( "list" );
if ( !typeName.endsWith( listSuffix ) )
typeName += QLatin1String( "list" );
}
if ( !name.isEmpty() )
attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );

attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );
}
addAttributes( attributes );
}
@@ -522,24 +535,31 @@ bool QgsMemoryProvider::addAttributes( const QList<QgsField> &attributes )
{
for ( QList<QgsField>::const_iterator it = attributes.begin(); it != attributes.end(); ++it )
{
switch ( it->type() )
if ( !supportedType( field ) )
continue;

// Make sure added attributes typeName correspond to a native type name
bool isNativeTypeName = false;
NativeType nativeTypeCandidate( QString(), QString(), QVariant::Invalid );
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
{
case QVariant::Int:
case QVariant::Double:
case QVariant::String:
case QVariant::Date:
case QVariant::Time:
case QVariant::DateTime:
case QVariant::LongLong:
case QVariant::StringList:
case QVariant::List:
case QVariant::Bool:
case QVariant::ByteArray:
break;
default:
QgsDebugMsg( "Field type not supported: " + it->typeName() );
if ( nativeType.mType == field.type()
&& nativeTypeCandidate.mType == QVariant::Invalid )
nativeTypeCandidate = nativeType;
}

if ( !isNativeTypeName )
{
if ( nativeTypeCandidate.mType == QVariant::Invalid )
{
QgsLogger::warning( "Field type not supported: " + field.typeName() );
continue;
}

field.setTypeName( nativeTypeCandidate.mTypeName );
}

// add new field as a last one
mFields.append( *it );

@@ -826,6 +826,67 @@ def testTypeValidation(self):
f = vl.getFeature(1)
self.assertEqual(f.attribute('int'), 123)

def testAddAttributes(self):
"""Test that fields with empty/invalid typenames are updated to native type names"""

vl = QgsVectorLayer("Point", "temporary_points", "memory")
pr = vl.dataProvider()

# add fields
pr.addAttributes([QgsField("name", QVariant.String),
QgsField("age", QVariant.Int, "invalidInteger"),
QgsField("size", QVariant.Double),
QgsField("mytext", QVariant.String, "text"),
QgsField("size2", QVariant.Double, "double precision"),
QgsField("short", QVariant.Int, "int2"),
QgsField("lessshort", QVariant.Int, "int4"),
QgsField("numericfield", QVariant.Double, "numeric"),
QgsField("decimalfield", QVariant.Double, "decimal"),
QgsField("stringlistfield", QVariant.StringList, "stringlist"),
QgsField("integerlistfield", QVariant.List, "integerlist"),
QgsField("doublelistfield", QVariant.List, "doublelist")])

self.assertEqual(pr.fields()[0].typeName(), "string")
self.assertEqual(pr.fields()[1].typeName(), "integer")
self.assertEqual(pr.fields()[2].typeName(), "double")
self.assertEqual(pr.fields()[3].typeName(), "text")
self.assertEqual(pr.fields()[4].typeName(), "double precision")
self.assertEqual(pr.fields()[5].typeName(), "int2")
self.assertEqual(pr.fields()[6].typeName(), "int4")
self.assertEqual(pr.fields()[7].typeName(), "numeric")
self.assertEqual(pr.fields()[8].typeName(), "decimal")
self.assertEqual(pr.fields()[9].typeName(), "stringlist")
self.assertEqual(pr.fields()[10].typeName(), "integerlist")
self.assertEqual(pr.fields()[11].typeName(), "doublelist")

vl2 = vl.clone()

self.assertEqual(pr.fields()[0].name(), vl2.fields()[0].name())
self.assertEqual(pr.fields()[1].name(), vl2.fields()[1].name())
self.assertEqual(pr.fields()[2].name(), vl2.fields()[2].name())
self.assertEqual(pr.fields()[3].name(), vl2.fields()[3].name())
self.assertEqual(pr.fields()[4].name(), vl2.fields()[4].name())
self.assertEqual(pr.fields()[5].name(), vl2.fields()[5].name())
self.assertEqual(pr.fields()[6].name(), vl2.fields()[6].name())
self.assertEqual(pr.fields()[7].name(), vl2.fields()[7].name())
self.assertEqual(pr.fields()[8].name(), vl2.fields()[8].name())
self.assertEqual(pr.fields()[9].name(), vl2.fields()[9].name())
self.assertEqual(pr.fields()[10].name(), vl2.fields()[10].name())
self.assertEqual(pr.fields()[11].name(), vl2.fields()[11].name())

self.assertEqual(pr.fields()[0].typeName(), vl2.fields()[0].typeName())
self.assertEqual(pr.fields()[1].typeName(), vl2.fields()[1].typeName())
self.assertEqual(pr.fields()[2].typeName(), vl2.fields()[2].typeName())
self.assertEqual(pr.fields()[3].typeName(), vl2.fields()[3].typeName())
self.assertEqual(pr.fields()[4].typeName(), vl2.fields()[4].typeName())
self.assertEqual(pr.fields()[5].typeName(), vl2.fields()[5].typeName())
self.assertEqual(pr.fields()[6].typeName(), vl2.fields()[6].typeName())
self.assertEqual(pr.fields()[7].typeName(), vl2.fields()[7].typeName())
self.assertEqual(pr.fields()[8].typeName(), vl2.fields()[8].typeName())
self.assertEqual(pr.fields()[9].typeName(), vl2.fields()[9].typeName())
self.assertEqual(pr.fields()[10].typeName(), vl2.fields()[10].typeName())
self.assertEqual(pr.fields()[11].typeName(), vl2.fields()[11].typeName())


class TestPyQgsMemoryProviderIndexed(unittest.TestCase, ProviderTestCase):
"""Runs the provider test suite against an indexed memory layer"""

0 comments on commit 847d4cc

Please sign in to comment.