298 changes: 296 additions & 2 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

#include "qgsapplication.h"
#include "qgsdataprovider.h"
#include "qgsexpression.h"
#include "qgsfeature.h"
#include "qgsfield.h"
#include "qgsgeometry.h"
Expand Down Expand Up @@ -60,6 +61,8 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
, mWktHasZM( false )
, mWktHasPrefix( false )
, mXyDms( false )
, mSubsetString("")
, mSubsetExpression( 0 )
, mMaxInvalidLines( 50 )
, mShowInvalidLines( true )
, mCrs()
Expand All @@ -77,6 +80,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
QString wktField;
QString xField;
QString yField;
QString subset;

if ( url.hasQueryItem( "geomType" ) )
{
Expand Down Expand Up @@ -121,6 +125,12 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
QgsDebugMsg( "xField is: " + xField );
QgsDebugMsg( "yField is: " + yField );

if( url.hasQueryItem( "subset" ))
{
subset = url.queryItemValue("subset");
QgsDebugMsg( "subset is: " + subset );
}

if ( url.hasQueryItem( "quiet" ) ) mShowInvalidLines = false;

// assume the layer is invalid until proven otherwise
Expand Down Expand Up @@ -390,6 +400,11 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )

reportErrors( warnings );
mValid = mGeometryType != QGis::UnknownGeometry;

if( ! subset.isEmpty())
{
setSubsetString( subset );
}
}


Expand Down Expand Up @@ -495,6 +510,12 @@ QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
delete mFile;
mFile = 0;
}

if( mSubsetExpression )
{
delete mSubsetExpression;
mSubsetExpression = 0;
}
}


Expand Down Expand Up @@ -548,7 +569,7 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
QgsMessageLog::logMessage( tr( "Errors in file %1" ).arg( mFile->fileName() ), tag );
foreach ( QString message, messages )
{
QgsMessageLog::logMessage( message );
QgsMessageLog::logMessage( message, tag );
}
if ( ! mInvalidLines.isEmpty() )
{
Expand All @@ -559,7 +580,6 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
QgsMessageLog::logMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ), tag );
}


// Display errors in a dialog...
if ( mShowInvalidLines )
{
Expand All @@ -586,7 +606,281 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
}
}

//

bool QgsDelimitedTextProvider::setSubsetString( QString subset, bool updateFeatureCount )
{

bool valid = true;

// If there is a new subset string then encode it..

QgsExpression *expression = 0;
if( ! subset.isEmpty() )
{

expression = new QgsExpression( subset );
QString error;
if( expression->hasParserError())
{
error = expression->parserErrorString();
}
else
{
expression->prepare( fields() );
if( expression->hasEvalError() )
{
error = expression->evalErrorString();
}
}
if( ! error.isEmpty() )
{
valid = false;
delete expression;
expression = 0;
QString tag( "DelimitedText" );
QgsMessageLog::logMessage( tr( "Invalid subset string %1 for %2" ).arg(subset).arg( mFile->fileName() ), tag );
}
}


// if the expression is valid, then reset the subset string and data source Uri

if( valid )
{

if( mSubsetExpression ) delete mSubsetExpression;
mSubsetString = subset;
mSubsetExpression = expression;

// Encode the subset string into the data source URI.

QUrl url = QUrl::fromEncoded( dataSourceUri().toAscii() );
if( url.hasQueryItem("subset")) url.removeAllQueryItems("subset");
if( ! subset.isEmpty()) url.addQueryItem("subset",subset);
setDataSourceUri( QString::fromAscii( url.toEncoded() ) );

// Update the feature count and extents if requested
if( updateFeatureCount )
{
resetDataSummary();
}
}

return valid;
}

void QgsDelimitedTextProvider::resetDataSummary()
{
QgsFeatureIterator fi = getFeatures(QgsFeatureRequest());
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while( fi.nextFeature(f))
{
if( mGeometryType != QGis::NoGeometry )
{
if( mNumberFeatures == 0 )
{
mExtent = f.geometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.geometry()->boundingBox());
mExtent.combineExtentWith( &bbox);
}
}
mNumberFeatures++;
}
}


bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request )
{
QStringList tokens;
while ( true )
{
// before we do anything else, assume that there's something wrong with
// the feature
feature.setValid( false );
QgsDelimitedTextFile::Status status = file->nextRecord( tokens );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;

int fid = file->recordLineNumber();
if( request.filterType() == QgsFeatureRequest::FilterFid && fid != request.filterFid()) continue;
if ( recordIsEmpty( tokens ) ) continue;

while ( tokens.size() < mFieldCount )
tokens.append( QString::null );

QgsGeometry *geom = 0;

// Note: Always need to load geometry even if request has NoGeometry set
// and subset string doesn't need geometry, as only by loading geometry
// do we know that this is a valid record in the data set.
if ( mWktFieldIndex >= 0 )
{
geom = loadGeometryWkt( tokens, request );
}
else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
{
geom = loadGeometryXY( tokens,request );
}

if ( !geom && mWkbType != QGis::WKBNoGeometry )
{
// Already dealt with invalid lines in provider - no need to repeat
// removed code (CC 2013-04-13) ...
// mInvalidLines << line;
// In any case it may be a valid line that is excluded because of
// bounds check...
continue;
}

// At this point the current feature values are valid

feature.setValid( true );
feature.setFields( &attributeFields ); // allow name-based attribute lookups
feature.setFeatureId( fid );
feature.initAttributes( attributeFields.count() );

if ( geom )
feature.setGeometry( geom );

// If we have subset expression, then ened attributes
if ( ! mSubsetExpression && request.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
const QgsAttributeList& attrs = request.subsetOfAttributes();
for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
{
int fieldIdx = *i;
fetchAttribute( feature, fieldIdx, tokens );
}
}
else
{
for ( int idx = 0; idx < attributeFields.count(); ++idx )
fetchAttribute( feature, idx, tokens );
}

// Are we using a subset expression, if so try and evaluate
// and accept result if passes.

if( mSubsetExpression )
{
QVariant isOk = mSubsetExpression->evaluate( &feature );
if( mSubsetExpression->hasEvalError() ) continue;
if( ! isOk.toBool() ) continue;
}

// We have a good line, so return
return true;

} // !mStream->atEnd()

return false;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request )
{
QgsGeometry* geom = 0;
QString sWkt = tokens[mWktFieldIndex];

geom = geomFromWkt( sWkt );

if ( geom && geom->type() != mGeometryType )
{
delete geom;
geom = 0;
}
if ( geom && !boundsCheck( geom, request ) )
{
delete geom;
geom = 0;
}
return geom;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request )
{
QString sX = tokens[mXFieldIndex];
QString sY = tokens[mYFieldIndex];
QgsPoint pt;
bool ok = pointFromXY( sX, sY, pt );

if ( ok && boundsCheck( pt, request ) )
{
return QgsGeometry::fromPoint( pt );
}
return 0;
}

/**
* Check to see if the point is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

return request.filterRect().contains( pt );
}

/**
* Check to see if the geometry is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

if ( request.flags() & QgsFeatureRequest::ExactIntersect )
return geom->intersects( request.filterRect() );
else
return geom->boundingBox().intersects( request.filterRect() );
}


void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
{
if( fieldIdx < 0 || fieldIdx >= attributeColumns.count()) return;
int column = attributeColumns[fieldIdx];
if( column < 0 || column >= tokens.count()) return;
const QString &value = tokens[column];
QVariant val;
switch ( attributeFields[fieldIdx].type() )
{
case QVariant::Int:
if ( value.isEmpty() )
val = QVariant( attributeFields[fieldIdx].type() );
else
val = QVariant( value );
break;
case QVariant::Double:
if ( value.isEmpty() )
{
val = QVariant( attributeFields[fieldIdx].type() );
}
else if ( mDecimalPoint.isEmpty() )
{
val = QVariant( value.toDouble() );
}
else
{
val = QVariant( QString( value ).replace( mDecimalPoint, "." ).toDouble() );
}
break;
default:
val = QVariant( value );
break;
}
feature.setAttribute( fieldIdx, val );
}

// Return the extent of the layer
QgsRectangle QgsDelimitedTextProvider::extent()
Expand Down
38 changes: 37 additions & 1 deletion src/providers/delimitedtext/qgsdelimitedtextprovider.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class QTextStream;

class QgsDelimitedTextFeatureIterator;
class QgsDelimitedTextFile;

class QgsExpression;

/**
\class QgsDelimitedTextProvider
Expand Down Expand Up @@ -143,7 +143,28 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
bool isValid();

virtual QgsCoordinateReferenceSystem crs();
/**
* Set the subset string used to create a subset of features in
* the layer.
*/
virtual bool setSubsetString( QString subset, bool updateFeatureCount = true );

/**
* provider supports setting of subset strings
*/
virtual bool supportsSubsetString() { return true; }

/**
* Returns the subset definition string (typically sql) currently in
* use by the layer and used by the provider to limit the feature set.
* Must be overridden in the dataprovider, otherwise returns a null
* QString.
*/
virtual QString subsetString()
{
return mSubsetString;
}
/* new functions */

/**
Expand Down Expand Up @@ -175,10 +196,22 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
void reportErrors( QStringList messages = QStringList() );
void resetStream();
bool recordIsEmpty( QStringList &record );
bool nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request );
QgsGeometry* loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request);
QgsGeometry* loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request );
bool boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request );
bool boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request );
void fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens );
void resetDataSummary();


QgsGeometry *geomFromWkt( QString &sWkt );
bool pointFromXY( QString &sX, QString &sY, QgsPoint &point );
double dmsStringToDouble( const QString &sX, bool *xOk );


QString mUri;

//! Text file
QgsDelimitedTextFile *mFile;

Expand Down Expand Up @@ -212,6 +245,9 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QString mDecimalPoint;
bool mXyDms;

QString mSubsetString;
QgsExpression *mSubsetExpression;

//! Storage for any lines in the file that couldn't be loaded
int mMaxInvalidLines;
int mNExtraInvalidLines;
Expand Down
29 changes: 17 additions & 12 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include "qgisinterface.h"
#include "qgscontexthelp.h"
#include "qgslogger.h"

#include "qgsvectordataprovider.h"
#include "qgsdelimitedtextprovider.h"
#include "qgsdelimitedtextfile.h"

Expand All @@ -30,10 +30,13 @@
#include <QTextCodec>
#include <QUrl>

const int MAX_SAMPLE_LENGTH=200;

QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt::WFlags fl, bool embedded ):
QDialog( parent, fl ),
mFile( new QgsDelimitedTextFile() ),
mExampleRowCount( 20 ),
mBadRowCount( 0 ),
mPluginKey( "/Plugin-DelimitedText" ),
mLastFileType( "" )
{
Expand All @@ -50,16 +53,7 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
}

cmbEncoding->clear();
QStringList codecs;
foreach ( QByteArray codec, QTextCodec::availableCodecs() )
{
codecs.append( codec );
}
codecs.sort();
foreach ( QString codec, codecs )
{
cmbEncoding->addItem( codec );
}
cmbEncoding->addItems( QgsVectorDataProvider::availableEncodings());
cmbEncoding->setCurrentIndex( cmbEncoding->findText( "UTF-8" ) );
loadSettings();

Expand Down Expand Up @@ -392,14 +386,15 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
QList<bool> isValidWkt;
QList<bool> isEmpty;
int counter = 0;
mBadRowCount = 0;
QStringList values;
QRegExp wktre( "^\\s*(?:MULTI)?(?:POINT|LINESTRING|POLYGON)\\s*Z?\\s*M?\\(", Qt::CaseInsensitive );

while ( counter < mExampleRowCount )
{
QgsDelimitedTextFile::Status status = mFile->nextRecord( values );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;
if ( status != QgsDelimitedTextFile::RecordOk ) { mBadRowCount++; continue; }
counter++;

// Look at count of non-blank fields
Expand All @@ -425,6 +420,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
for ( int i = 0; i < tblSample->columnCount(); i++ )
{
QString value = i < nv ? values[i] : "";
if( value.length() > MAX_SAMPLE_LENGTH ) value = value.mid(0,MAX_SAMPLE_LENGTH)+"...";
QTableWidgetItem *item = new QTableWidgetItem( value );
tblSample->setItem( counter - 1, i, item );
if ( ! value.isEmpty() )
Expand Down Expand Up @@ -689,6 +685,10 @@ bool QgsDelimitedTextSourceSelect::validate()
else if ( tblSample->rowCount() == 0 )
{
message = tr( "No data found in file" );
if( mBadRowCount > 0 )
{
message = message + " (" + tr("%1 badly formatted records discarded").arg(mBadRowCount)+")";
}
}
else if ( geomTypeXY->isChecked() && ( cmbXField->currentText().isEmpty() || cmbYField->currentText().isEmpty() ) )
{
Expand All @@ -705,6 +705,11 @@ bool QgsDelimitedTextSourceSelect::validate()
else
{
enabled = true;
if( mBadRowCount > 0 )
{
message = tr("%1 badly formatted records discarded from sample data").arg(mBadRowCount);
}

}
lblStatus->setText( message );
return enabled;
Expand Down
1 change: 1 addition & 0 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
private:
QgsDelimitedTextFile *mFile;
int mExampleRowCount;
int mBadRowCount;
QString mPluginKey;
QString mLastFileType;

Expand Down
952 changes: 702 additions & 250 deletions tests/src/python/test_qgsdelimitedtextprovider.py

Large diffs are not rendered by default.