387 changes: 188 additions & 199 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/providers/delimitedtext/qgsdelimitedtextprovider.h
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider

void clearInvalidLines();
void recordInvalidLine( QString message );
void handleInvalidLines();
void reportErrors( QStringList messages = QStringList() );
void resetStream();
bool recordIsEmpty( QStringList &record );

Expand Down
86 changes: 42 additions & 44 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,8 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
QDialog( parent, fl ),
mFile( new QgsDelimitedTextFile() ),
mExampleRowCount( 20 ),
mColumnNamePrefix( "Column_" ),
mPluginKey( "/Plugin-DelimitedText" ),
mLastFileType("")
mLastFileType( "" )
{

setupUi( this );
Expand All @@ -51,7 +50,13 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
}

cmbEncoding->clear();
QStringList codecs;
foreach ( QByteArray codec, QTextCodec::availableCodecs() )
{
codecs.append( codec );
}
codecs.sort();
foreach( QString codec, codecs )
{
cmbEncoding->addItem( codec );
}
Expand All @@ -62,6 +67,7 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt

connect( txtFilePath, SIGNAL( textChanged( QString ) ), this, SLOT( updateFileName() ) );
connect( txtLayerName, SIGNAL( textChanged( QString ) ), this, SLOT( enableAccept() ) );
connect( cmbEncoding, SIGNAL( currentIndexChanged( int ) ), this, SLOT( updateFieldsAndEnable() ) );

connect( delimiterCSV, SIGNAL( toggled( bool ) ), this, SLOT( updateFieldsAndEnable() ) );
connect( delimiterChars, SIGNAL( toggled( bool ) ), this, SLOT( updateFieldsAndEnable() ) );
Expand Down Expand Up @@ -153,10 +159,8 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
{
QString field = cmbXField->currentText();
if ( ! useHeader ) field.remove( mColumnNamePrefix );
url.addQueryItem( "xField", field );
field = cmbYField->currentText();
if ( ! useHeader ) field.remove( mColumnNamePrefix );
url.addQueryItem( "yField", field );
}
}
Expand All @@ -165,7 +169,6 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
if ( ! cmbWktField->currentText().isEmpty() )
{
QString field = cmbWktField->currentText();
if ( ! useHeader ) field.remove( mColumnNamePrefix );
url.addQueryItem( "wktField", field );
}
if ( cmbGeometryType->currentIndex() > 0 )
Expand Down Expand Up @@ -314,9 +317,9 @@ void QgsDelimitedTextSourceSelect::loadSettingsForFile( QString filename )
{
if ( filename.isEmpty() ) return;
QFileInfo fi( filename );
QString filetype=fi.suffix();
QString filetype = fi.suffix();
// Don't expect to change settings if not changing file type
if( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
if ( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
mLastFileType = filetype;
}

Expand Down Expand Up @@ -383,27 +386,13 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
if ( ! loadDelimitedFileDefinition() )
return;

bool useHeader = mFile->useHeader();
QStringList fieldList;
QList<bool> isValidNumber;
QList<bool> isValidWkt;
QList<bool> isEmpty;
// Put a sample set of records into the sample box. Also while scanning assess suitability of
// fields for use as coordinate and WKT fields

if ( useHeader )
{
fieldList = mFile->columnNames();
tblSample->setColumnCount( fieldList.size() );
tblSample->resizeColumnsToContents();
for ( int i = 0; i < fieldList.size(); i++ )
{
isValidNumber.append( false );
isValidWkt.append( false );
isEmpty.append( true );
}
}

// put a lines into the sample box

QList<bool> isValidCoordinate;
QList<bool> isValidWkt;
QList<bool> isEmpty;
int counter = 0;
QStringList values;
QRegExp wktre( "^\\s*(?:MULTI)?(?:POINT|LINESTRING|POLYGON)\\s*Z?\\s*M?\\(", Qt::CaseInsensitive );
Expand All @@ -415,24 +404,20 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
if ( status != QgsDelimitedTextFile::RecordOk ) continue;
counter++;

// If don't have headers, then check column count and expand if necessary
// Don't count blank columns
// Look at count of non-blank fields

int nv = values.size();
while ( nv > 0 && values[nv-1].isEmpty() ) nv--;

if ( nv > fieldList.size() )
if ( isEmpty.size() < nv )
{
while ( fieldList.size() < nv )
while ( isEmpty.size() < nv )
{
int nc = fieldList.size();
QString column = mColumnNamePrefix + QString::number( nc + 1 );
fieldList.append( column );
isEmpty.append( true );
isValidNumber.append( false );
isValidCoordinate.append( false );
isValidWkt.append( false );
}
tblSample->setColumnCount( fieldList.size() );
tblSample->setColumnCount( nv );
}

tblSample->setRowCount( counter );
Expand All @@ -449,10 +434,10 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
if ( isEmpty[i] )
{
isEmpty[i] = false;
isValidNumber[i] = true;
isValidCoordinate[i] = true;
isValidWkt[i] = true;
}
if ( isValidNumber[i] )
if ( isValidCoordinate[i] )
{
bool ok = true;
if ( cbxPointIsComma->isChecked() )
Expand All @@ -467,7 +452,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
{
value.toDouble( &ok );
}
isValidNumber[i] = ok;
isValidCoordinate[i] = ok;
}
if ( isValidWkt[i] )
{
Expand All @@ -478,6 +463,19 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
}
}

QStringList fieldList = mFile->fieldNames();

if ( isEmpty.size() < fieldList.size() )
{
while ( isEmpty.size() < fieldList.size() )
{
isEmpty.append( true );
isValidCoordinate.append( false );
isValidWkt.append( false );
}
tblSample->setColumnCount( fieldList.size() );
}

tblSample->setHorizontalHeaderLabels( fieldList );
tblSample->resizeColumnsToContents();
tblSample->resizeRowsToContents();
Expand Down Expand Up @@ -508,11 +506,11 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
// Now try setting optional X,Y fields - will only reset the fields if
// not already set.

trySetXYField( fieldList, isValidNumber, "longitude", "latitude" );
trySetXYField( fieldList, isValidNumber, "lon", "lat" );
trySetXYField( fieldList, isValidNumber, "east", "north" );
trySetXYField( fieldList, isValidNumber, "x", "y" );
trySetXYField( fieldList, isValidNumber, "e", "n" );
trySetXYField( fieldList, isValidCoordinate, "longitude", "latitude" );
trySetXYField( fieldList, isValidCoordinate, "lon", "lat" );
trySetXYField( fieldList, isValidCoordinate, "east", "north" );
trySetXYField( fieldList, isValidCoordinate, "x", "y" );
trySetXYField( fieldList, isValidCoordinate, "e", "n" );

// And also a WKT field if there is one

Expand Down Expand Up @@ -581,7 +579,7 @@ bool QgsDelimitedTextSourceSelect::trySetXYField( QStringList &fields, QList<boo
if ( ! fields.contains( yfield, Qt::CaseInsensitive ) ) continue;
for ( int iy = 0; iy < fields.size(); iy++ )
{
if ( ! isValidNumber[i] ) continue;
if ( ! isValidNumber[iy] ) continue;
if ( iy == i ) continue;
if ( fields[iy].compare( yfield, Qt::CaseInsensitive ) == 0 )
{
Expand Down
1 change: 0 additions & 1 deletion src/providers/delimitedtext/qgsdelimitedtextsourceselect.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
private:
QgsDelimitedTextFile *mFile;
int mExampleRowCount;
QString mColumnNamePrefix;
QString mPluginKey;
QString mLastFileType;

Expand Down
3 changes: 3 additions & 0 deletions src/ui/qgsdelimitedtextsourceselectbase.ui
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,9 @@
<property name="whatsThis">
<string>Select the file encoding</string>
</property>
<property name="insertPolicy">
<enum>QComboBox::InsertAtTop</enum>
</property>
</widget>
</item>
</layout>
Expand Down
292 changes: 265 additions & 27 deletions tests/src/python/test_qgsdelimitedtextprovider.py

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions tests/testdata/delimitedtext/test.badquote
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
id,description,data,info
1,Unclosed quotes 1,"Quoted,data1
2,Unclosed quotes 2,"Quoted,data2",info2
3,Recovered after unclosed quore,"Data ok",inf3
4,Unclosed quotes to end of file,"Never ending field ...
3 changes: 3 additions & 0 deletions tests/testdata/delimitedtext/testfields.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
id,"description",data,,data,28,24.5,field_3,data_1
1,Generation of field names,Some data,Some info,,,,,,,,last data

2 changes: 2 additions & 0 deletions tests/testdata/delimitedtext/testlatin1.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
id|description|name
1|Correctly read latin1 encoding|This test is �
3 changes: 3 additions & 0 deletions tests/testdata/delimitedtext/testre.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
idREGEXPdescriptionREGEXPdataREGEXPinfo
1REGEXP Basic regular expression test REGEXP data1 REGEXP info
2REGEXP Basic regular expression test 2 RE data2 RE info2
4 changes: 4 additions & 0 deletions tests/testdata/delimitedtext/testre2.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
id description information
1 Anchored regexp Some data
2 Anchored regexp invalid
3 Anchored regexp recovered Some data
2 changes: 2 additions & 0 deletions tests/testdata/delimitedtext/testutf8.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
id|description|name
1|Correctly read UTF8 encoding|Field has āccèntéd text