Skip to content
Permalink
Browse files

Delimited text detectTypes flag - fixes #18601

[FEATURE] Adds a detectTypes flag to the delimited text provider url. If
set to "no" then type detection is not done and all attributes are
treated as text fields.  Otherwise the original behaviour of
detecting field types is preserved.

[needs-docs] Adds a "Detect field types" check box to the record and
field options section of the delimited text provider GUI.

This addresses (at least partially) issue #18601.  A more complete
solution would be to allow users to set field types.
  • Loading branch information
ccrook authored and nyalldawson committed Jul 5, 2018
1 parent 8f5d968 commit b4f2069c6fd1fc6a583918327a7fbc9eccabe2ff
@@ -120,6 +120,10 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
}
}

mDetectTypes=true;
if ( url.hasQueryItem( QStringLiteral( "detectTypes" ) ) )
mDetectTypes = ! url.queryItemValue( QStringLiteral( "detectTypes" ) ).toLower().startsWith( 'n' );

if ( url.hasQueryItem( QStringLiteral( "decimalPoint" ) ) )
mDecimalPoint = url.queryItemValue( QStringLiteral( "decimalPoint" ) );

@@ -225,7 +229,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
QgsDebugMsg( QString( "Field type string: %1" ).arg( strTypeList ) );

int pos = 0;
QRegExp reType( "(integer|real|string|date|datetime|time)" );
QRegExp reType( "(integer|real|double|string|date|datetime|time)" );

while ( ( pos = reType.indexIn( strTypeList, pos ) ) != -1 )
{
@@ -240,10 +244,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
// *message=tr("Reading field types from %1").arg(csvtInfo.fileName());
}


return types;


}

void QgsDelimitedTextProvider::resetCachedSubset() const
@@ -556,6 +557,11 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
couldBeDouble[i] = true;
}

if( ! mDetectTypes )
{
continue;
}

// Now test for still valid possible types for the field
// Types are possible until first record which cannot be parsed

@@ -603,40 +609,40 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
QString typeName = QStringLiteral( "text" );
if ( i < csvtTypes.size() )
{
if ( csvtTypes[i] == QLatin1String( "integer" ) )
{
fieldType = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( csvtTypes[i] == QLatin1String( "long" ) || csvtTypes[i] == QLatin1String( "longlong" ) || csvtTypes[i] == QLatin1String( "int8" ) )
{
fieldType = QVariant::LongLong; //QVariant doesn't support long
typeName = QStringLiteral( "longlong" );
}
else if ( csvtTypes[i] == QLatin1String( "real" ) || csvtTypes[i] == QLatin1String( "double" ) )
{
fieldType = QVariant::Double;
typeName = QStringLiteral( "double" );
}
typeName = csvtTypes[i];
}
else if ( i < couldBeInt.size() )
else if ( mDetectTypes && i < couldBeInt.size() )
{
if ( couldBeInt[i] )
{
fieldType = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( couldBeLongLong[i] )
{
fieldType = QVariant::LongLong;
typeName = QStringLiteral( "longlong" );
}
else if ( couldBeDouble[i] )
{
fieldType = QVariant::Double;
typeName = QStringLiteral( "double" );
}
}
if( typeName == QStringLiteral( "integer" ) )
{
fieldType = QVariant::Int;
}
else if ( typeName == QStringLiteral( "longlong" ) )
{
fieldType = QVariant::LongLong;
}
else if( typeName == QStringLiteral( "real" ) || typeName == QStringLiteral( "double" ) )
{
typeName = QStringLiteral( "double" );
fieldType = QVariant::Double;
}
else
{
typeName = QStringLiteral( "text" );
}
attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
}

@@ -227,6 +227,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QString mWktFieldName;
QString mXFieldName;
QString mYFieldName;
bool mDetectTypes;

mutable int mXFieldIndex = -1;
mutable int mYFieldIndex = -1;
@@ -145,6 +145,8 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()

QUrl url = mFile->url();

url.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );

if ( cbxPointIsComma->isChecked() )
{
url.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
@@ -192,9 +194,9 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()

}

if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? "yes" : "no" );
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? "yes" : "no" );
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? "yes" : "no" );
if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );

// store the settings
saveSettings();
@@ -284,6 +286,7 @@ void QgsDelimitedTextSourceSelect::loadSettings( const QString &subkey, bool loa

rowCounter->setValue( settings.value( key + "/startFrom", 0 ).toInt() );
cbxUseHeader->setChecked( settings.value( key + "/useHeader", "true" ) != "false" );
cbxDetectTypes->setChecked( settings.value( key + "/detectTypes", "true" ) != "false" );
cbxTrimFields->setChecked( settings.value( key + "/trimFields", "false" ) == "true" );
cbxSkipEmptyFields->setChecked( settings.value( key + "/skipEmptyFields", "false" ) == "true" );
cbxPointIsComma->setChecked( settings.value( key + "/decimalPoint", "." ).toString().contains( ',' ) );
@@ -329,6 +332,7 @@ void QgsDelimitedTextSourceSelect::saveSettings( const QString &subkey, bool sav
settings.setValue( key + "/delimiterRegexp", txtDelimiterRegexp->text() );
settings.setValue( key + "/startFrom", rowCounter->value() );
settings.setValue( key + "/useHeader", cbxUseHeader->isChecked() ? "true" : "false" );
settings.setValue( key + "/detectTypes", cbxDetectTypes->isChecked() ? "true" : "false" );
settings.setValue( key + "/trimFields", cbxTrimFields->isChecked() ? "true" : "false" );
settings.setValue( key + "/skipEmptyFields", cbxSkipEmptyFields->isChecked() ? "true" : "false" );
settings.setValue( key + "/decimalPoint", cbxPointIsComma->isChecked() ? "," : "." );
@@ -45,16 +45,7 @@
<bool>true</bool>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
@@ -295,16 +286,7 @@
</widget>
<widget class="QWidget" name="swpDelimOptions">
<layout class="QVBoxLayout" name="verticalLayout_11">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
@@ -325,16 +307,7 @@
<property name="spacing">
<number>0</number>
</property>
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>2</number>
</property>
<item>
@@ -593,16 +566,7 @@
</widget>
<widget class="QWidget" name="swpRegexpOptions">
<layout class="QVBoxLayout" name="verticalLayout_12">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
@@ -811,6 +775,16 @@
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QCheckBox" name="cbxDetectTypes">
<property name="text">
<string>Detect field types</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
</item>
@@ -900,16 +874,7 @@
</property>
<widget class="QWidget" name="swpGeomXY">
<layout class="QVBoxLayout" name="verticalLayout_8">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
@@ -1033,16 +998,7 @@
</widget>
<widget class="QWidget" name="swpGeomWKT">
<layout class="QVBoxLayout" name="verticalLayout_6">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
@@ -1332,6 +1288,7 @@
<tabstop>recordOptionsGroupBox</tabstop>
<tabstop>rowCounter</tabstop>
<tabstop>cbxUseHeader</tabstop>
<tabstop>cbxDetectTypes</tabstop>
<tabstop>cbxPointIsComma</tabstop>
<tabstop>cbxTrimFields</tabstop>
<tabstop>cbxSkipEmptyFields</tabstop>
@@ -337,7 +337,7 @@ def printWanted(self, testname, result):
print((prefix + ' ' + repr(msg) + ','))
print((prefix + ' ]'))
print(' return wanted')
print()
print('',flush=True)

def recordDifference(self, record1, record2):
# Compare a record defined as a dictionary
@@ -804,6 +804,21 @@ def test_040_issue_14666(self):
params = {'yField': 'y', 'xField': 'x', 'type': 'csv', 'delimiter': '\\t'}
requests = None
self.runTest(filename, requests, **params)

def test_041_no_detect_type(self):
# CSV file parsing
# Skip lines
filename = 'testtypes.csv'
params = {'yField': 'lat', 'xField': 'lon', 'type': 'csv', 'detectTypes': 'no'}
requests = None
self.runTest(filename, requests, **params)

def test_042_no_detect_types_csvt(self):
# CSVT field types
filename = 'testcsvt.csv'
params = {'geomType': 'none', 'type': 'csv', 'detectTypes': 'no' }
requests = None
self.runTest(filename, requests, **params)


if __name__ == '__main__':

0 comments on commit b4f2069

Please sign in to comment.
You can’t perform that action at this time.